Merge branch 'feature/memdebug' of github.com:foomo/contentserver into feature/memdebug

Makefile

@@ -27,6 +27,12 @@ build-docker: clean build-arch
 	echo "# tagged container `cat .image_id` as $(IMAGE):$(TAG)"
 	rm -vf .image_id .cacert.pem
 
+build-testclient:
+	go build -o bin/testclient -i github.com/foomo/contentserver/testing/client
+
+build-testserver:
+	go build -o bin/testserver -i github.com/foomo/contentserver/testing/server
+
 package: build
 	pkg/build.sh
 
@@ -46,6 +52,15 @@ test:
 bench:
 	go test -run=none -bench=. ./...
 
+run-testserver:
+	bin/testserver -json-file var/cse-globus-stage-b-with-main-section.json
+
+run-contentserver:
+	contentserver -var-dir var -webserver-address :9191 -address :9999 -log-level debug http://127.0.0.1:1234
+
+clean-var:
+	rm var/contentserver-repo-2019*
+
 # Profiling
 
 test-cpu-profile:

content/reponode.go

@@ -22,13 +22,13 @@ type RepoNode struct {
 	// published from - to is going to be an array of fromTos
 }
 
-// NewRepoNode constructor
+// // NewRepoNode constructor
-func NewRepoNode() *RepoNode {
+// func NewRepoNode() *RepoNode {
-	return &RepoNode{
+// 	return &RepoNode{
-		Data:  make(map[string]interface{}),
+// 		Data:  make(map[string]interface{}, 0), // set initial size to zero explicitely?
-		Nodes: make(map[string]*RepoNode),
+// 		Nodes: make(map[string]*RepoNode, 0),
-	}
+// 	}
-}
+// }
 
 // WireParents helper method to reference from child to parent in a tree
 // recursively

contentserver.go

@@ -11,6 +11,9 @@ import (
 	"github.com/foomo/contentserver/metrics"
 	"github.com/foomo/contentserver/status"
 
+	"net/http"
+	_ "net/http/pprof"
+
 	"github.com/foomo/contentserver/log"
 	"github.com/foomo/contentserver/server"
 )
@@ -66,6 +69,10 @@ func exitUsage(code int) {
 func main() {
 	flag.Parse()
 
+	go func() {
+		fmt.Println(http.ListenAndServe("localhost:6060", nil))
+	}()
+
 	if *flagShowVersionFlag {
 		fmt.Printf("%v\n", uniqushPushVersion)
 		return
@@ -74,9 +81,12 @@ func main() {
 	if *flagFreeOSMem > 0 {
 		log.Notice("[INFO] freeing OS memory every ", *flagFreeOSMem, " minutes!")
 		go func() {
-			for _ = range time.After(time.Duration(*flagFreeOSMem) * time.Minute) {
+			for {
-				log.Notice("FreeOSMemory")
+				select {
-				debug.FreeOSMemory()
+				case <-time.After(time.Duration(*flagFreeOSMem) * time.Minute):
+					log.Notice("FreeOSMemory")
+					debug.FreeOSMemory()
+				}
 			}
 		}()
 	}
@@ -84,16 +94,19 @@ func main() {
 	if *flagHeapDump > 0 {
 		log.Notice("[INFO] dumping heap every ", *flagHeapDump, " minutes!")
 		go func() {
-			for _ = range time.After(time.Duration(*flagFreeOSMem) * time.Minute) {
+			for {
-				log.Notice("HeapDump")
+				select {
-				f, err := os.Create("heapdump")
+				case <-time.After(time.Duration(*flagFreeOSMem) * time.Minute):
-				if err != nil {
+					log.Notice("HeapDump")
-					panic("failed to create heap dump file")
+					f, err := os.Create("heapdump")
-				}
+					if err != nil {
-				debug.WriteHeapDump(f.Fd())
+						panic("failed to create heap dump file")
-				err = f.Close()
+					}
-				if err != nil {
+					debug.WriteHeapDump(f.Fd())
-					panic("failed to create heap dump file")
+					err = f.Close()
+					if err != nil {
+						panic("failed to create heap dump file")
+					}
 				}
 			}
 		}()

repo/loader.go

@@ -143,7 +143,7 @@ func (repo *Repo) update() (repoRuntime int64, jsonBytes []byte, err error) {
 		log.Debug("we have no json to load - the repo server did not reply", err)
 		return repoRuntime, jsonBytes, err
 	}
-	log.Debug("loading json from: "+repo.server, string(jsonBytes))
+	log.Debug("loading json from: "+repo.server, "length:", len(jsonBytes))
 	nodes, err := loadNodesFromJSON(jsonBytes)
 	if err != nil {
 		// could not load nodes from json

server/socketserver.go

@@ -40,7 +40,8 @@ func extractHandlerAndJSONLentgh(header string) (handler Handler, jsonLength int
 
 func (s *socketServer) execute(handler Handler, jsonBytes []byte) (reply []byte) {
 	if log.SelectedLevel == log.LevelDebug {
-		log.Debug("  incoming json buffer:", string(jsonBytes))
+		log.Debug("  incoming json buffer of length: ", len(jsonBytes))
+		// log.Debug("  incoming json buffer:", string(jsonBytes))
 	}
 	reply, handlingError := handleRequest(s.repo, handler, jsonBytes, s.metrics)
 	if handlingError != nil {
@@ -127,7 +128,8 @@ func (s *socketServer) handleConnection(conn net.Conn) {
 				}
 
 				if log.SelectedLevel == log.LevelDebug {
-					log.Debug("  read json: " + string(jsonBytes))
+					log.Debug("  read json, length: ", len(jsonBytes))
+					// log.Debug("  read json: " + string(jsonBytes))
 				}
 				s.writeResponse(conn, s.execute(handler, jsonBytes))
 				// note: connection remains open

testing/client/client.go

@@ -0,0 +1,32 @@
+package main
+
+import (
+	"log"
+	"time"
+
+	"github.com/davecgh/go-spew/spew"
+	"github.com/foomo/contentserver/client"
+)
+
+func main() {
+	serverAdr := "http://127.0.0.1:9191/contentserver"
+	c, errClient := client.NewHTTPClient(serverAdr)
+	if errClient != nil {
+		log.Fatal(errClient)
+	}
+
+	for i := 1; i <= 50; i++ {
+		go func() {
+			log.Println("start update")
+			resp, errUpdate := c.Update()
+			if errUpdate != nil {
+				spew.Dump(resp)
+				log.Fatal(errUpdate)
+			}
+			log.Println(i, "update done", resp)
+		}()
+		time.Sleep(5 * time.Second)
+	}
+
+	log.Println("done")
+}

testing/server/server.go

@@ -0,0 +1,35 @@
+package main
+
+import (
+	"flag"
+	"log"
+	"net/http"
+)
+
+type testServer struct {
+	file string
+}
+
+func main() {
+
+	var (
+		flagJSONFile = flag.String("json-file", "", "provide a json source file")
+		flagAddress  = flag.String("addr", ":1234", "set the webserver address")
+	)
+	flag.Parse()
+
+	if *flagJSONFile == "" {
+		log.Fatal("js source file must be provided")
+	}
+
+	ts := &testServer{
+		file: *flagJSONFile,
+	}
+
+	log.Println("start test server at", *flagAddress, "serving file:", ts.file)
+	log.Fatal(http.ListenAndServe(*flagAddress, ts))
+}
+
+func (ts *testServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	http.ServeFile(w, r, ts.file)
+}