Configurable Repository Timeout (#25)

* feat: implement correct prometheus metrics

* feat: remove nodeID from metrics due to high cardinality

* feat: add configurable timeouts to contentserver

client/client_test.go

@@ -63,6 +63,7 @@ func initTestServer(t testing.TB) (socketAddr, webserverAddr string) {
 			webserverAddr,
 			pathContentserver,
 			varDir,
+			server.DefaultRepositoryTimeout,
 		)
 		if err != nil {
 			t.Fatal("test server crashed: ", err)

contentserver.go

@@ -25,14 +25,16 @@ const (
 
 	ServiceName                  = "Content Server"
 	DefaultHealthzHandlerAddress = ":8080"
-	DefaultPrometheusListener    = "127.0.0.1:9200"
+	DefaultPrometheusListener    = ":9200"
 )
 
 var (
-	flagAddress          = flag.String("address", "", "address to bind socket server host:port")
+	flagAddress                   = flag.String("address", "", "address to bind socket server host:port")
-	flagWebserverAddress = flag.String("webserver-address", "", "address to bind web server host:port, when empty no webserver will be spawned")
+	flagWebserverAddress          = flag.String("webserver-address", "", "address to bind web server host:port, when empty no webserver will be spawned")
-	flagWebserverPath    = flag.String("webserver-path", "/contentserver", "path to export the webserver on - useful when behind a proxy")
+	flagWebserverPath             = flag.String("webserver-path", "/contentserver", "path to export the webserver on - useful when behind a proxy")
-	flagVarDir           = flag.String("var-dir", "/var/lib/contentserver", "where to put my data")
+	flagVarDir                    = flag.String("var-dir", "/var/lib/contentserver", "where to put my data")
+	flagPrometheusListener        = flag.String("prometheus-listener", getenv("PROMETHEUS_LISTENER", DefaultPrometheusListener), "address for the prometheus listener")
+	flagRepositoryTimeoutDuration = flag.Duration("repository-timeout-duration", server.DefaultRepositoryTimeout, "timeout duration for the contentserver")
 
 	// debugging / profiling
 	flagDebug     = flag.Bool("debug", false, "toggle debug mode")
@@ -40,6 +42,13 @@ var (
 	flagHeapDump  = flag.Int("heap-dump", 0, "dump heap every X minutes")
 )
 
+func getenv(env, fallback string) string {
+	if value, ok := os.LookupEnv(env); ok {
+		return value
+	}
+	return fallback
+}
+
 func exitUsage(code int) {
 	fmt.Println("Usage:", os.Args[0], "http(s)://your-content-server/path/to/content.json")
 	flag.PrintDefaults()
@@ -89,10 +98,10 @@ func main() {
 		fmt.Println(*flagAddress, flag.Arg(0))
 
 		// kickoff metric handlers
-		go metrics.RunPrometheusHandler(DefaultPrometheusListener)
+		go metrics.RunPrometheusHandler(*flagPrometheusListener)
 		go status.RunHealthzHandlerListener(DefaultHealthzHandlerAddress, ServiceName)
 
-		err := server.RunServerSocketAndWebServer(flag.Arg(0), *flagAddress, *flagWebserverAddress, *flagWebserverPath, *flagVarDir)
+		err := server.RunServerSocketAndWebServer(flag.Arg(0), *flagAddress, *flagWebserverAddress, *flagWebserverPath, *flagVarDir, *flagRepositoryTimeoutDuration)
 		if err != nil {
 			fmt.Println("exiting with error", err)
 			os.Exit(1)

repo/repo.go

@@ -54,7 +54,7 @@ type repoDimension struct {
 }
 
 // NewRepo constructor
-func NewRepo(server string, varDir string) *Repo {
+func NewRepo(server string, varDir string, repositoryTimeout time.Duration) *Repo {
 
 	logger.Log.Info("creating new repo",
 		zap.String("server", server),
@@ -67,7 +67,7 @@ func NewRepo(server string, varDir string) *Repo {
 		history:                    newHistory(varDir),
 		dimensionUpdateChannel:     make(chan *repoDimension),
 		dimensionUpdateDoneChannel: make(chan error),
-		httpClient:                 getDefaultHTTPClient(2 * time.Minute),
+		httpClient:                 getDefaultHTTPClient(repositoryTimeout),
 		updateInProgressChannel:    make(chan chan updateResponse, 0),
 	}
 
@@ -160,7 +160,7 @@ func (repo *Repo) getNodes(nodeRequests map[string]*requests.Node, env *requests
 				zap.String("nodeName", nodeName),
 				zap.String("nodeID", nodeRequest.ID),
 			)
-			status.M.InvalidNodeTreeRequests.WithLabelValues(nodeRequest.ID).Inc()
+			status.M.InvalidNodeTreeRequests.WithLabelValues().Inc()
 			continue
 		}
 		nodes[nodeName] = repo.getNode(treeNode, nodeRequest.Expand, nodeRequest.MimeTypes, path, 0, groups, nodeRequest.DataFields, nodeRequest.ExposeHiddenNodes)
@@ -249,12 +249,12 @@ func (repo *Repo) WriteRepoBytes(w io.Writer) {
 		logger.Log.Error("Failed to serve Repo JSON", zap.Error(err))
 	}
 
-	w.Write([]byte("{\"reply\":"))
+	_, _ = w.Write([]byte("{\"reply\":"))
 	_, err = io.Copy(w, f)
 	if err != nil {
 		logger.Log.Error("Failed to serve Repo JSON", zap.Error(err))
 	}
-	w.Write([]byte("}"))
+	_, _ = w.Write([]byte("}"))
 }
 
 // Update - reload contents of repository with json from repo.server

repo/repo_test.go

@@ -18,7 +18,7 @@ func init() {
 
 func NewTestRepo(server, varDir string) *Repo {
 
-	r := NewRepo(server, varDir)
+	r := NewRepo(server, varDir, 2*time.Minute)
 
 	// because the travis CI VMs are very slow,
 	// we need to add some delay to allow the server to startup

server/server.go

@@ -3,13 +3,15 @@ package server
 import (
 	"errors"
 	"fmt"
+	"net"
+	"net/http"
+	"os"
+	"time"
+
 	. "github.com/foomo/contentserver/logger"
 	"github.com/foomo/contentserver/repo"
 	jsoniter "github.com/json-iterator/go"
 	"go.uber.org/zap"
-	"net"
-	"net/http"
-	"os"
 )
 
 var (
@@ -30,11 +32,14 @@ const (
 	HandlerUpdate = "update"
 	// HandlerGetRepo get the whole repo
 	HandlerGetRepo = "getRepo"
+
+	// DefaultRepositoryTimeout for the HTTP client towards the repo
+	DefaultRepositoryTimeout = 2 * time.Minute
 )
 
 // Run - let it run and enjoy on a socket near you
 func Run(server string, address string, varDir string) error {
-	return RunServerSocketAndWebServer(server, address, "", "", varDir)
+	return RunServerSocketAndWebServer(server, address, "", "", varDir, DefaultRepositoryTimeout)
 }
 
 func RunServerSocketAndWebServer(
@@ -43,13 +48,14 @@ func RunServerSocketAndWebServer(
 	webserverAddress string,
 	webserverPath string,
 	varDir string,
+	repositoryTimeout time.Duration,
 ) error {
 	if address == "" && webserverAddress == "" {
 		return errors.New("one of the addresses needs to be set")
 	}
 	Log.Info("building repo with content", zap.String("server", server))
 
-	r := repo.NewRepo(server, varDir)
+	r := repo.NewRepo(server, varDir, repositoryTimeout)
 
 	// start initial update and handle error
 	go func() {

status/metrics.go

@@ -38,8 +38,7 @@ type Metrics struct {
 func newMetrics() *Metrics {
 	return &Metrics{
 		InvalidNodeTreeRequests: newCounterVec("invalid_node_tree_request_count",
-			"Counts the number of invalid tree nodes for a specific node ID",
+			"Counts the number of invalid tree nodes for a specific node"),
-			"nodeID"),
 		ServiceRequestCounter: newCounterVec(
 			"service_request_count",
 			"Count of requests for each handler",