From 3eac4bd1b7bb22664b7bee1707c8eb51fa7cf82f Mon Sep 17 00:00:00 2001 From: Stefan Martinov Date: Wed, 8 May 2019 17:51:51 +0200 Subject: [PATCH 01/79] feat: add prometheus handler and go-modules --- .travis.yml | 2 -- Dockerfile | 32 ++++++++++++++++++++++++-------- go.mod | 3 +++ go.sum | 10 ++++++++++ metrics/prometheus.go | 27 +++++++++++++++++++++++++++ 5 files changed, 64 insertions(+), 10 deletions(-) create mode 100644 go.mod create mode 100644 go.sum create mode 100644 metrics/prometheus.go diff --git a/.travis.yml b/.travis.yml index d622f77..1a0bbea 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,3 @@ language: go go: - - 1.10 - - 1.11 - tip diff --git a/Dockerfile b/Dockerfile index 4d39c56..755a58d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,20 +1,36 @@ -FROM scratch +############################## +###### STAGE: BUILD ###### +############################## +FROM golang:latest AS build-env -COPY bin/contentserver-linux-amd64 /usr/sbin/contentserver +WORKDIR /src -# install ca root certificates -# https://curl.haxx.se/docs/caextract.html -# http://blog.codeship.com/building-minimal-docker-containers-for-go-applications/ -# does not work on docker for mac :( -# ADD https://curl.haxx.se/ca/cacert.pem /etc/ssl/certs/ca-certificates.crt -ADD .cacert.pem /etc/ssl/certs/ca-certificates.crt +COPY ./go.mod ./go.sum ./ +RUN go mod download && go mod vendor && go install -i ./vendor/... + +# Import the code from the context. +COPY ./ ./ + +RUN GOARCH=amd64 GOOS=linux CGO_ENABLED=0 go build -o /contentserver + +############################## +###### STAGE: PACKAGE ###### +############################## +FROM alpine ENV CONTENT_SERVER_LOG_LEVEL=error ENV CONTENT_SERVER_ADDR=0.0.0.0:80 ENV CONTENT_SERVER_VAR_DIR=/var/lib/contentserver +RUN apk add --update --no-cache ca-certificates curl bash && rm -rf /var/cache/apk/* + +COPY --from=build-env /contentserver /usr/sbin/contentserver + + VOLUME $CONTENT_SERVER_VAR_DIR + EXPOSE 80 +EXPOSE 9200 ## Prometheus Listener ENTRYPOINT ["/usr/sbin/contentserver"] diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..55a367c --- /dev/null +++ b/go.mod @@ -0,0 +1,3 @@ +module github.com/foomo/contentserver + +require github.com/prometheus/client_golang v0.9.2 // indirect diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..86f674a --- /dev/null +++ b/go.sum @@ -0,0 +1,10 @@ +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740= +github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= diff --git a/metrics/prometheus.go b/metrics/prometheus.go new file mode 100644 index 0000000..5fbfabf --- /dev/null +++ b/metrics/prometheus.go @@ -0,0 +1,27 @@ +package metrics + +import ( + "fmt" + "github.com/foomo/contentserver/log" + "github.com/prometheus/client_golang/prometheus/promhttp" + "net/http" +) + +const ( + DefaultPrometheusListener = ":9200" +) + +func PrometheusHandler() http.Handler { + h := http.NewServeMux() + h.Handle("/metrics", promhttp.Handler()) + return h +} + +func RunPrometheusHandler(listener string) { + log.Notice(fmt.Sprintf("starting prometheus handler on address '%s'", DefaultPrometheusListener)) + log.Error(http.ListenAndServe(listener, PrometheusHandler())) +} + +func RunPrometheusHandlerOnDefaultAddress() { + RunPrometheusHandler(DefaultPrometheusListener) +} From 6b891e7f0fdb2fe1514c1eec1b45ff3e569a3182 Mon Sep 17 00:00:00 2001 From: Stefan Martinov Date: Wed, 8 May 2019 17:57:08 +0200 Subject: [PATCH 02/79] chore: update modules & testing --- go.mod | 2 +- go.sum | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 55a367c..1fe38f3 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,3 @@ module github.com/foomo/contentserver -require github.com/prometheus/client_golang v0.9.2 // indirect +require github.com/prometheus/client_golang v0.9.2 diff --git a/go.sum b/go.sum index 86f674a..904fd75 100644 --- a/go.sum +++ b/go.sum @@ -1,10 +1,16 @@ +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740= github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jOqq0gIVUe6Yk0/QMZ640k6NvkxcBf+8= github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= From d0e9a2966a799bee58c532dbd96d5167eaa80390 Mon Sep 17 00:00:00 2001 From: Stefan Martinov Date: Wed, 8 May 2019 18:04:58 +0200 Subject: [PATCH 03/79] chore: add correct expose file and better makefile --- Dockerfile | 6 +++--- Makefile | 13 ++++++++++--- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index 755a58d..5d7aa1f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -29,9 +29,9 @@ COPY --from=build-env /contentserver /usr/sbin/contentserver VOLUME $CONTENT_SERVER_VAR_DIR -EXPOSE 80 -EXPOSE 9200 ## Prometheus Listener - ENTRYPOINT ["/usr/sbin/contentserver"] CMD ["-address=$CONTENT_SERVER_ADDR", "-log-level=$CONTENT_SERVER_LOG_LEVEL", "-var-dir=$CONTENT_SERVER_VAR_DIR"] + +EXPOSE 80 +EXPOSE 9200 \ No newline at end of file diff --git a/Makefile b/Makefile index 416f315..063c6f8 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,7 @@ SHELL := /bin/bash -TAG=`git describe --exact-match --tags $(git log -n1 --pretty='%h') 2>/dev/null || git rev-parse --abbrev-ref HEAD` +TAG?=latest +IMAGE=docker-registry.bestbytes.net/contentserver all: build test tag: @@ -15,11 +16,17 @@ build-arch: clean build-docker: clean build-arch curl https://curl.haxx.se/ca/cacert.pem > .cacert.pem docker build -q . > .image_id - docker tag `cat .image_id` docker-registry.bestbytes.net/contentserver:$(TAG) - echo "# tagged container `cat .image_id` as docker-registry.bestbytes.net/contentserver:$(TAG)" + docker tag `cat .image_id` $(IMAGE):$(TAG) + echo "# tagged container `cat .image_id` as $(IMAGE):$(TAG)" rm -vf .image_id .cacert.pem package: build pkg/build.sh test: go test ./... + +docker-build: + docker build -t $(IMAGE):$(TAG) . + +docker-push: + docker push $(IMAGE):$(TAG) \ No newline at end of file From ff28d6670fab1710770722e2b525adf98f71a504 Mon Sep 17 00:00:00 2001 From: Stefan Martinov Date: Wed, 8 May 2019 18:21:20 +0200 Subject: [PATCH 04/79] feat: add healthz handler to contentserver --- contentserver.go | 14 +++++++++++++- metrics/prometheus.go | 10 +--------- status/healthz.go | 26 ++++++++++++++++++++++++++ 3 files changed, 40 insertions(+), 10 deletions(-) create mode 100644 status/healthz.go diff --git a/contentserver.go b/contentserver.go index b2fc356..425b1a2 100644 --- a/contentserver.go +++ b/contentserver.go @@ -3,6 +3,8 @@ package main import ( "flag" "fmt" + "github.com/foomo/contentserver/metrics" + "github.com/foomo/contentserver/status" "os" "strings" @@ -18,8 +20,15 @@ const ( logLevelError = "error" ) +const ( + ServiceName = "Content Server" + + DefaultHealthzHandlerAddress = ":8080" + DefaultPrometheusListener = ":9200" +) + var ( - uniqushPushVersion = "content-server 1.4.1" + uniqushPushVersion = "content-server 1.5.0" showVersionFlag = flag.Bool("version", false, "version info") address = flag.String("address", "", "address to bind socket server host:port") webserverAddress = flag.String("webserver-address", "", "address to bind web server host:port, when empty no webserver will be spawned") @@ -70,6 +79,9 @@ func main() { level = log.LevelDebug } log.SelectedLevel = level + go metrics.RunPrometheusHandler(DefaultPrometheusListener) + go status.RunHealthzHandlerListener(DefaultHealthzHandlerAddress, ServiceName) + err := server.RunServerSocketAndWebServer(flag.Arg(0), *address, *webserverAddress, *webserverPath, *varDir) if err != nil { fmt.Println("exiting with error", err) diff --git a/metrics/prometheus.go b/metrics/prometheus.go index 5fbfabf..984a933 100644 --- a/metrics/prometheus.go +++ b/metrics/prometheus.go @@ -7,10 +7,6 @@ import ( "net/http" ) -const ( - DefaultPrometheusListener = ":9200" -) - func PrometheusHandler() http.Handler { h := http.NewServeMux() h.Handle("/metrics", promhttp.Handler()) @@ -18,10 +14,6 @@ func PrometheusHandler() http.Handler { } func RunPrometheusHandler(listener string) { - log.Notice(fmt.Sprintf("starting prometheus handler on address '%s'", DefaultPrometheusListener)) + log.Notice(fmt.Sprintf("starting prometheus handler on address '%s'", listener)) log.Error(http.ListenAndServe(listener, PrometheusHandler())) } - -func RunPrometheusHandlerOnDefaultAddress() { - RunPrometheusHandler(DefaultPrometheusListener) -} diff --git a/status/healthz.go b/status/healthz.go new file mode 100644 index 0000000..21acbbb --- /dev/null +++ b/status/healthz.go @@ -0,0 +1,26 @@ +package status + +import ( + "encoding/json" + "fmt" + "github.com/foomo/contentserver/log" + "net/http" +) + +func RunHealthzHandlerListener(address string, serviceName string) { + log.Notice(fmt.Sprintf("starting healthz handler on '%s'" + address)) + log.Error(http.ListenAndServe(address, HealthzHandler(serviceName))) +} + +func HealthzHandler(serviceName string) http.Handler { + data := map[string]string{ + "service": serviceName, + } + status, _ := json.Marshal(data) + h := http.NewServeMux() + h.Handle("/healthz", http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + w.Write(status) + })) + + return h +} From b7f10ed673fcbfd736a784f155a5b9844ec43bf8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Frederik=20L=C3=B6ffert?= Date: Tue, 14 May 2019 16:40:25 +0200 Subject: [PATCH 05/79] add prometheus request metrics --- server/handlerequest.go | 39 +++++++++++++++++++++++++++++++++- server/server.go | 26 +++++++++++++---------- server/socketserver.go | 46 ++++++++++++----------------------------- server/webserver.go | 14 ++++++++----- status/metrics.go | 43 ++++++++++++++++++++++++++++++++++++++ 5 files changed, 118 insertions(+), 50 deletions(-) create mode 100644 status/metrics.go diff --git a/server/handlerequest.go b/server/handlerequest.go index b03e817..f3be21f 100644 --- a/server/handlerequest.go +++ b/server/handlerequest.go @@ -4,19 +4,26 @@ import ( "encoding/json" "errors" "fmt" + "time" "github.com/foomo/contentserver/log" "github.com/foomo/contentserver/repo" "github.com/foomo/contentserver/requests" "github.com/foomo/contentserver/responses" + "github.com/foomo/contentserver/status" + "github.com/prometheus/client_golang/prometheus" ) -func handleRequest(r *repo.Repo, handler Handler, jsonBytes []byte) (replyBytes []byte, err error) { +func handleRequest(r *repo.Repo, handler Handler, jsonBytes []byte, metrics *status.Metrics) (replyBytes []byte, err error) { + // variables var reply interface{} var apiErr error var jsonErr error + start := time.Now() + + // helper processor processIfJSONIsOk := func(err error, processingFunc func()) { if err != nil { jsonErr = err @@ -25,37 +32,46 @@ func handleRequest(r *repo.Repo, handler Handler, jsonBytes []byte) (replyBytes processingFunc() } + // handle and process switch handler { case HandlerGetURIs: getURIRequest := &requests.URIs{} processIfJSONIsOk(json.Unmarshal(jsonBytes, &getURIRequest), func() { reply = r.GetURIs(getURIRequest.Dimension, getURIRequest.IDs) }) + addMetrics(metrics, HandlerGetURIs, start, jsonErr, apiErr) case HandlerGetContent: contentRequest := &requests.Content{} processIfJSONIsOk(json.Unmarshal(jsonBytes, &contentRequest), func() { reply, apiErr = r.GetContent(contentRequest) }) + addMetrics(metrics, HandlerGetContent, start, jsonErr, apiErr) case HandlerGetNodes: nodesRequest := &requests.Nodes{} processIfJSONIsOk(json.Unmarshal(jsonBytes, &nodesRequest), func() { reply = r.GetNodes(nodesRequest) }) + addMetrics(metrics, HandlerGetNodes, start, jsonErr, apiErr) case HandlerUpdate: updateRequest := &requests.Update{} processIfJSONIsOk(json.Unmarshal(jsonBytes, &updateRequest), func() { reply = r.Update() }) + addMetrics(metrics, HandlerUpdate, start, jsonErr, apiErr) case HandlerGetRepo: repoRequest := &requests.Repo{} processIfJSONIsOk(json.Unmarshal(jsonBytes, &repoRequest), func() { reply = r.GetRepo() }) + addMetrics(metrics, HandlerGetRepo, start, jsonErr, apiErr) default: err = errors.New(log.Error(" can not handle this one " + handler)) errorResponse := responses.NewError(1, "unknown handler") reply = errorResponse + addMetrics(metrics, "default", start, jsonErr, apiErr) } + + // error handling if jsonErr != nil { err = jsonErr log.Error(" could not read incoming json:", jsonErr) @@ -66,9 +82,30 @@ func handleRequest(r *repo.Repo, handler Handler, jsonBytes []byte) (replyBytes err = apiErr reply = responses.NewError(3, "internal error "+apiErr.Error()) } + return encodeReply(reply) } +func addMetrics(metrics *status.Metrics, handlerName Handler, start time.Time, errJSON error, errAPI error) { + + duration := time.Since(start) + + s := "succeeded" + if errJSON != nil || errAPI != nil { + s = "failed" + } + + metrics.ServiceRequestCounter.With(prometheus.Labels{ + status.MetricLabelHandler: string(handlerName), + status.MetricLabelStatus: s, + }).Inc() + + metrics.ServiceRequestDuration.With(prometheus.Labels{ + status.MetricLabelHandler: string(handlerName), + status.MetricLabelStatus: s, + }).Observe(float64(duration.Nanoseconds())) +} + func encodeReply(reply interface{}) (replyBytes []byte, err error) { encodedBytes, jsonReplyErr := json.MarshalIndent(map[string]interface{}{ "reply": reply, diff --git a/server/server.go b/server/server.go index 06f1329..60e51de 100644 --- a/server/server.go +++ b/server/server.go @@ -74,19 +74,23 @@ func runSocketServer( address string, chanErr chan error, ) { - s := &socketServer{ - stats: newStats(), - repo: repo, - } - ln, err := net.Listen("tcp", address) - if err != nil { - err = errors.New("RunSocketServer: could not start the on \"" + address + "\" - error: " + fmt.Sprint(err)) - // failed to create socket - log.Error(err) - chanErr <- err + // create socket server + s, errSocketServer := newSocketServer(repo) + if errSocketServer != nil { + log.Error(errSocketServer) + chanErr <- errSocketServer return } - // there we go + + // listen on socket + ln, errListen := net.Listen("tcp", address) + if errListen != nil { + errListenSocket := errors.New("RunSocketServer: could not start the on \"" + address + "\" - error: " + fmt.Sprint(errListen)) + log.Error(errListenSocket) + chanErr <- errListenSocket + return + } + log.Record("RunSocketServer: started to listen on " + address) for { // this blocks until connection or error diff --git a/server/socketserver.go b/server/socketserver.go index 0b2be0a..ec92a86 100644 --- a/server/socketserver.go +++ b/server/socketserver.go @@ -10,39 +10,21 @@ import ( "github.com/foomo/contentserver/log" "github.com/foomo/contentserver/repo" "github.com/foomo/contentserver/responses" + "github.com/foomo/contentserver/status" ) -// simple internal request counter -type stats struct { - requests int64 - chanCount chan int -} - -func newStats() *stats { - s := &stats{ - requests: 0, - chanCount: make(chan int), - } - go func() { - for { - select { - case <-s.chanCount: - s.requests++ - s.chanCount <- 1 - } - } - }() - return s -} - -func (s *stats) countRequest() { - s.chanCount <- 1 - <-s.chanCount -} - type socketServer struct { - stats *stats - repo *repo.Repo + repo *repo.Repo + metrics *status.Metrics +} + +// newSocketServer returns a shiny new socket server +func newSocketServer(repo *repo.Repo) (s *socketServer, err error) { + s = &socketServer{ + repo: repo, + metrics: status.NewMetrics("socketserver"), + } + return } func extractHandlerAndJSONLentgh(header string) (handler Handler, jsonLength int, err error) { @@ -58,12 +40,10 @@ func extractHandlerAndJSONLentgh(header string) (handler Handler, jsonLength int } func (s *socketServer) execute(handler Handler, jsonBytes []byte) (reply []byte) { - s.stats.countRequest() - log.Notice("socketServer.execute: ", s.stats.requests, ", ", handler) if log.SelectedLevel == log.LevelDebug { log.Debug(" incoming json buffer:", string(jsonBytes)) } - reply, handlingError := handleRequest(s.repo, handler, jsonBytes) + reply, handlingError := handleRequest(s.repo, handler, jsonBytes, s.metrics) if handlingError != nil { log.Error("socketServer.execute handlingError :", handlingError) } diff --git a/server/webserver.go b/server/webserver.go index ba113d8..3f59c5b 100644 --- a/server/webserver.go +++ b/server/webserver.go @@ -5,19 +5,23 @@ import ( "net/http" "strings" + "github.com/foomo/contentserver/status" + "github.com/foomo/contentserver/repo" ) type webServer struct { - r *repo.Repo - path string + r *repo.Repo + path string + metrics *status.Metrics } // NewWebServer returns a shiny new web server func NewWebServer(path string, r *repo.Repo) (s http.Handler, err error) { s = &webServer{ - r: r, - path: path, + r: r, + path: path, + metrics: status.NewMetrics("webserver"), } return } @@ -33,7 +37,7 @@ func (s *webServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { http.Error(w, "failed to read incoming request", http.StatusBadRequest) return } - reply, errReply := handleRequest(s.r, Handler(strings.TrimPrefix(r.URL.Path, s.path+"/")), jsonBytes) + reply, errReply := handleRequest(s.r, Handler(strings.TrimPrefix(r.URL.Path, s.path+"/")), jsonBytes, s.metrics) if errReply != nil { http.Error(w, errReply.Error(), http.StatusInternalServerError) return diff --git a/status/metrics.go b/status/metrics.go new file mode 100644 index 0000000..212ae29 --- /dev/null +++ b/status/metrics.go @@ -0,0 +1,43 @@ +package status + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +const MetricLabelHandler = "handler" +const MetricLabelStatus = "status" + +type Metrics struct { + ServiceRequestCounter *prometheus.CounterVec // count the number of requests for each service function + ServiceRequestDuration *prometheus.SummaryVec // count the duration of requests for each service function +} + +func NewMetrics(namespace string) *Metrics { + return &Metrics{ + ServiceRequestCounter: serviceRequestCounter("api", namespace), + ServiceRequestDuration: serviceRequestDuration("api", namespace), + } +} + +func serviceRequestCounter(subsystem, namespace string) *prometheus.CounterVec { + vec := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "count_service_requests", + Help: "count of requests per func", + }, []string{MetricLabelHandler, MetricLabelStatus}) + prometheus.MustRegister(vec) + return vec +} + +func serviceRequestDuration(subsystem, namespace string) *prometheus.SummaryVec { + vec := prometheus.NewSummaryVec(prometheus.SummaryOpts{ + Namespace: namespace, + Subsystem: subsystem, + Name: "time_nanoseconds", + Help: "nanoseconds to unmarshal requests, execute a service function and marshal its reponses", + }, []string{MetricLabelHandler, MetricLabelStatus}) + prometheus.MustRegister(vec) + return vec +} From 79244f8ab3c0b421b0d48120272127e29b967b02 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Frederik=20L=C3=B6ffert?= Date: Tue, 14 May 2019 16:43:19 +0200 Subject: [PATCH 06/79] add "dep" command to makefile hide vendor in gitignore --- .gitignore | 1 + Makefile | 2 ++ 2 files changed, 3 insertions(+) diff --git a/.gitignore b/.gitignore index f47ed53..e0e37da 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,5 @@ *~ /bin/ /pkg/tmp/ +/vendor !.git* diff --git a/Makefile b/Makefile index 063c6f8..ae34959 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,8 @@ IMAGE=docker-registry.bestbytes.net/contentserver all: build test tag: echo $(TAG) +dep: + go mod download && go mod vendor && go install -i ./vendor/... clean: rm -fv bin/contentserve* build: clean From 42adb6a25a05540b2434fcc165b829504abef29c Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Tue, 21 May 2019 09:19:25 +0200 Subject: [PATCH 07/79] simplified loop over channel inputs --- server/socketserver.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/server/socketserver.go b/server/socketserver.go index 0b2be0a..630fa19 100644 --- a/server/socketserver.go +++ b/server/socketserver.go @@ -24,12 +24,9 @@ func newStats() *stats { chanCount: make(chan int), } go func() { - for { - select { - case <-s.chanCount: - s.requests++ - s.chanCount <- 1 - } + for _ = range s.chanCount { + s.requests++ + s.chanCount <- 1 } }() return s From 757c310f8d3548f53798585e7645698f2f57cacf Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Tue, 21 May 2019 09:36:40 +0200 Subject: [PATCH 08/79] handlerequest cleanup --- server/handlerequest.go | 44 ++++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 20 deletions(-) diff --git a/server/handlerequest.go b/server/handlerequest.go index b03e817..8264c21 100644 --- a/server/handlerequest.go +++ b/server/handlerequest.go @@ -13,17 +13,18 @@ import ( func handleRequest(r *repo.Repo, handler Handler, jsonBytes []byte) (replyBytes []byte, err error) { - var reply interface{} - var apiErr error - var jsonErr error - - processIfJSONIsOk := func(err error, processingFunc func()) { - if err != nil { - jsonErr = err - return + var ( + reply interface{} + apiErr error + jsonErr error + processIfJSONIsOk = func(err error, processingFunc func()) { + if err != nil { + jsonErr = err + return + } + processingFunc() } - processingFunc() - } + ) switch handler { case HandlerGetURIs: @@ -56,28 +57,31 @@ func handleRequest(r *repo.Repo, handler Handler, jsonBytes []byte) (replyBytes errorResponse := responses.NewError(1, "unknown handler") reply = errorResponse } + + // check for errors and set reply if jsonErr != nil { - err = jsonErr log.Error(" could not read incoming json:", jsonErr) - errorResponse := responses.NewError(2, "could not read incoming json "+jsonErr.Error()) - reply = errorResponse + err = jsonErr + reply = responses.NewError(2, "could not read incoming json "+jsonErr.Error()) } else if apiErr != nil { log.Error(" an API error occured:", apiErr) err = apiErr reply = responses.NewError(3, "internal error "+apiErr.Error()) } + return encodeReply(reply) } +// encodeReply takes an interface and encodes it as JSON +// it returns the resulting JSON and a marshalling error func encodeReply(reply interface{}) (replyBytes []byte, err error) { - encodedBytes, jsonReplyErr := json.MarshalIndent(map[string]interface{}{ + + // @TODO: why use marshal indent here??? + replyBytes, err = json.MarshalIndent(map[string]interface{}{ "reply": reply, }, "", " ") - if jsonReplyErr != nil { - err = jsonReplyErr - log.Error(" could not encode reply " + fmt.Sprint(jsonReplyErr)) - } else { - replyBytes = encodedBytes + if err != nil { + log.Error(" could not encode reply " + fmt.Sprint(err)) } - return replyBytes, err + return } From 73b9b71dd3ff27a172e490d4a313646bf18dd115 Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Tue, 21 May 2019 09:52:59 +0200 Subject: [PATCH 09/79] removed unused error assignments, code cleanup --- server/handlerequest.go | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/server/handlerequest.go b/server/handlerequest.go index a55bcec..980bcbb 100644 --- a/server/handlerequest.go +++ b/server/handlerequest.go @@ -2,7 +2,6 @@ package server import ( "encoding/json" - "errors" "fmt" "time" @@ -63,20 +62,16 @@ func handleRequest(r *repo.Repo, handler Handler, jsonBytes []byte, metrics *sta }) addMetrics(metrics, HandlerGetRepo, start, jsonErr, apiErr) default: - err = errors.New(log.Error(" can not handle this one " + handler)) - errorResponse := responses.NewError(1, "unknown handler") - reply = errorResponse + reply = responses.NewError(1, "unknown handler: "+string(handler)) addMetrics(metrics, "default", start, jsonErr, apiErr) } // error handling if jsonErr != nil { log.Error(" could not read incoming json:", jsonErr) - err = jsonErr reply = responses.NewError(2, "could not read incoming json "+jsonErr.Error()) } else if apiErr != nil { log.Error(" an API error occured:", apiErr) - err = apiErr reply = responses.NewError(3, "internal error "+apiErr.Error()) } From 284ee996902d386ab66e960c159fe209e1271c3a Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Tue, 21 May 2019 09:55:36 +0200 Subject: [PATCH 10/79] omitted comparison to bool constant --- repo/repo.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/repo/repo.go b/repo/repo.go index 453aa29..8c6dc15 100644 --- a/repo/repo.go +++ b/repo/repo.go @@ -156,7 +156,7 @@ func (repo *Repo) GetContent(r *requests.Content) (c *content.SiteContent, err e if log.SelectedLevel == log.LevelDebug { log.Debug(fmt.Sprintf("resolved: %v, uri: %v, dim: %v, n: %v", resolved, resolvedURI, resolvedDimension, node)) } - if resolved == false { + if !resolved { log.Debug("repo.GetContent", r.URI, "could not be resolved falling back to default dimension", r.Env.Dimensions[0]) // r.Env.Dimensions is validated => we can access it resolvedDimension = r.Env.Dimensions[0] From 7f9b32162e2208756965a0072d53f4fd25adcbe9 Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Tue, 21 May 2019 09:55:55 +0200 Subject: [PATCH 11/79] used a simple channel send/receive instead of with a single case --- repo/loader.go | 19 ++++++++----------- 1 file changed, 8 insertions(+), 11 deletions(-) diff --git a/repo/loader.go b/repo/loader.go index 91f5682..8bfcfe9 100644 --- a/repo/loader.go +++ b/repo/loader.go @@ -14,18 +14,15 @@ import ( func (repo *Repo) updateRoutine() { go func() { - for { - log.Debug("update routine is about to select") - select { - case newDimension := <-repo.updateChannel: - log.Debug("update routine received a new dimension: " + newDimension.Dimension) - err := repo._updateDimension(newDimension.Dimension, newDimension.Node) - log.Debug("update routine received result") - if err != nil { - log.Debug(" update routine error: " + err.Error()) - } - repo.updateDoneChannel <- err + for newDimension := range repo.updateChannel { + log.Debug("update routine received a new dimension: " + newDimension.Dimension) + + err := repo._updateDimension(newDimension.Dimension, newDimension.Node) + log.Debug("update routine received result") + if err != nil { + log.Debug(" update routine error: " + err.Error()) } + repo.updateDoneChannel <- err } }() } From 9d94c09735f4ee141c8d0d3aa329b2674b3dfb19 Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Tue, 21 May 2019 09:56:38 +0200 Subject: [PATCH 12/79] grouped variable declarations --- repo/loader.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/repo/loader.go b/repo/loader.go index 8bfcfe9..d562f44 100644 --- a/repo/loader.go +++ b/repo/loader.go @@ -38,10 +38,12 @@ func (repo *Repo) updateDimension(dimension string, node *content.RepoNode) erro // do not call directly, but only through channel func (repo *Repo) _updateDimension(dimension string, newNode *content.RepoNode) error { newNode.WireParents() - newDirectory := make(map[string]*content.RepoNode) - newURIDirectory := make(map[string]*content.RepoNode) - err := builDirectory(newNode, newDirectory, newURIDirectory) + var ( + newDirectory = make(map[string]*content.RepoNode) + newURIDirectory = make(map[string]*content.RepoNode) + err = builDirectory(newNode, newDirectory, newURIDirectory) + ) if err != nil { return errors.New("update dimension \"" + dimension + "\" failed when building its directory:: " + err.Error()) } From bdb694b0ff1254fa5101f98dc67b5be0a13290ad Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Tue, 21 May 2019 10:03:31 +0200 Subject: [PATCH 13/79] simplified byte equality comparison --- repo/history_test.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/repo/history_test.go b/repo/history_test.go index 1a2e66a..9ef4d65 100644 --- a/repo/history_test.go +++ b/repo/history_test.go @@ -25,7 +25,7 @@ func TestHistoryCurrent(t *testing.T) { if err != nil { t.Fatal(err) } - if bytes.Compare(current, test) != 0 { + if !bytes.Equal(current, test) { t.Fatal(fmt.Sprintf("expected %q, got %q", string(test), string(current))) } } @@ -73,7 +73,6 @@ func TestGetFilesForCleanup(t *testing.T) { assertStringEqual(t, "testdata/order/contentserver-repo-2017-10-21.json", files[1]) } - func assertStringEqual(t *testing.T, expected, actual string) { if expected != actual { t.Errorf("expected string %s differs from the actual %s", expected, actual) From a267dbe1ecab3df35352c4558a006062c1568763 Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Tue, 21 May 2019 10:04:09 +0200 Subject: [PATCH 14/79] history_test cleanup and formatting --- repo/history_test.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/repo/history_test.go b/repo/history_test.go index 9ef4d65..0801663 100644 --- a/repo/history_test.go +++ b/repo/history_test.go @@ -18,8 +18,10 @@ func testHistory() *history { } func TestHistoryCurrent(t *testing.T) { - h := testHistory() - test := []byte("test") + var ( + h = testHistory() + test = []byte("test") + ) h.add(test) current, err := h.getCurrent() if err != nil { @@ -51,7 +53,6 @@ func TestHistoryOrder(t *testing.T) { h.varDir = "testdata/order" files, err := h.getHistory() - if err != nil { t.Fatal("error not expected") } From d2e0af8aca8990607dcafd65a39af9c4feff37e8 Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Tue, 21 May 2019 10:05:01 +0200 Subject: [PATCH 15/79] using fmt.Errorf(...) instead of errors.New(fmt.Sprintf(...)) --- repo/history.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/repo/history.go b/repo/history.go index 9a23859..1e358de 100644 --- a/repo/history.go +++ b/repo/history.go @@ -1,14 +1,14 @@ package repo import ( + "errors" + "fmt" "io/ioutil" "os" "path" "sort" "strings" "time" - "errors" - "fmt" ) const historyRepoJSONPrefix = "contentserver-repo-" @@ -63,7 +63,7 @@ func (h *history) cleanup() error { for _, f := range files { err := os.Remove(f) if err != nil { - return errors.New(fmt.Sprintf("could not remove file %s : %s", f, err.Error())) + return fmt.Errorf("could not remove file %s : %s", f, err.Error()) } } From dd902fe7176c229954eaa4dd4e5517b18fcbc9a0 Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Tue, 21 May 2019 10:05:24 +0200 Subject: [PATCH 16/79] removed redundant return statement --- client/client_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/client/client_test.go b/client/client_test.go index 05cca00..df4a465 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -235,5 +235,4 @@ func benchmarkClientAndServerGetContent(b testing.TB, numGroups, numCalls int, c } // Wait for all HTTP fetches to complete. wg.Wait() - return } From 82733d4b252f1ea3b2488e4ef63e098ea5db2050 Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Tue, 21 May 2019 10:06:21 +0200 Subject: [PATCH 17/79] fix: call of Unmarshal passed a non-pointer as second argument --- client/sockettransport.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/client/sockettransport.go b/client/sockettransport.go index 97b524d..96bb09f 100644 --- a/client/sockettransport.go +++ b/client/sockettransport.go @@ -109,8 +109,10 @@ func (c *socketTransport) call(handler server.Handler, request interface{}, resp responseJSONErr := json.Unmarshal(responseBytes, &serverResponse{Reply: response}) if responseJSONErr != nil { // is it an error ? - remoteErr := responses.Error{} - remoteErrJSONErr := json.Unmarshal(responseBytes, remoteErr) + var ( + remoteErr = responses.Error{} + remoteErrJSONErr = json.Unmarshal(responseBytes, &remoteErr) + ) if remoteErrJSONErr == nil { returnConn(remoteErrJSONErr) return remoteErr From 54ee995295db585d0a906807cd74422052501870 Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Tue, 21 May 2019 10:07:13 +0200 Subject: [PATCH 18/79] commented out unused symbols --- client/connectionpool.go | 4 ++-- repo/repo.go | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/client/connectionpool.go b/client/connectionpool.go index de73455..b5f2dd2 100644 --- a/client/connectionpool.go +++ b/client/connectionpool.go @@ -6,8 +6,8 @@ import ( ) type connectionPool struct { - server string - conn net.Conn + server string + // conn net.Conn chanConnGet chan chan net.Conn chanConnReturn chan connReturn chanDrainPool chan int diff --git a/repo/repo.go b/repo/repo.go index 8c6dc15..7d3b480 100644 --- a/repo/repo.go +++ b/repo/repo.go @@ -327,6 +327,6 @@ func (repo *Repo) hasDimension(d string) bool { return hasDimension } -func uriKeyForState(state string, uri string) string { - return state + "-" + uri -} +// func uriKeyForState(state string, uri string) string { +// return state + "-" + uri +// } From 5ee042bcd42a6a6d9d3f885e2384cb6ae515f028 Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Tue, 21 May 2019 10:12:05 +0200 Subject: [PATCH 19/79] handle unchecked errors --- client/client_test.go | 25 ++++++++++++++++--------- repo/history_test.go | 15 ++++++++++++--- server/webserver.go | 6 +++++- status/healthz.go | 20 +++++++++++++------- 4 files changed, 46 insertions(+), 20 deletions(-) diff --git a/client/client_test.go b/client/client_test.go index df4a465..2e9648e 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -43,21 +43,28 @@ func getAvailableAddr() string { return "127.0.0.1:" + strconv.Itoa(getFreePort()) } -var testServerSocketAddr string -var testServerWebserverAddr string +var ( + testServerSocketAddr string + testServerWebserverAddr string +) func initTestServer(t testing.TB) (socketAddr, webserverAddr string) { socketAddr = getAvailableAddr() webserverAddr = getAvailableAddr() testServer, varDir := mock.GetMockData(t) log.SelectedLevel = log.LevelError - go server.RunServerSocketAndWebServer( - testServer.URL+"/repo-two-dimensions.json", - socketAddr, - webserverAddr, - pathContentserver, - varDir, - ) + go func() { + err := server.RunServerSocketAndWebServer( + testServer.URL+"/repo-two-dimensions.json", + socketAddr, + webserverAddr, + pathContentserver, + varDir, + ) + if err != nil { + t.Fatal("test server crashed: ", err) + } + }() socketClient, errClient := NewClient(socketAddr, 1, time.Duration(time.Millisecond*100)) if errClient != nil { panic(errClient) diff --git a/repo/history_test.go b/repo/history_test.go index 0801663..fc06a6b 100644 --- a/repo/history_test.go +++ b/repo/history_test.go @@ -22,7 +22,10 @@ func TestHistoryCurrent(t *testing.T) { h = testHistory() test = []byte("test") ) - h.add(test) + err := h.add(test) + if err != nil { + t.Fatal("failed to add: ", err) + } current, err := h.getCurrent() if err != nil { t.Fatal(err) @@ -35,10 +38,16 @@ func TestHistoryCurrent(t *testing.T) { func TestHistoryCleanup(t *testing.T) { h := testHistory() for i := 0; i < 50; i++ { - h.add([]byte(fmt.Sprint(i))) + err := h.add([]byte(fmt.Sprint(i))) + if err != nil { + t.Fatal("failed to add: ", err) + } time.Sleep(time.Millisecond * 5) } - h.cleanup() + err := h.cleanup() + if err != nil { + t.Fatal("failed to run cleanup: ", err) + } files, err := h.getHistory() if err != nil { t.Fatal(err) diff --git a/server/webserver.go b/server/webserver.go index 3f59c5b..c638a45 100644 --- a/server/webserver.go +++ b/server/webserver.go @@ -5,6 +5,7 @@ import ( "net/http" "strings" + "github.com/foomo/contentserver/log" "github.com/foomo/contentserver/status" "github.com/foomo/contentserver/repo" @@ -42,5 +43,8 @@ func (s *webServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { http.Error(w, errReply.Error(), http.StatusInternalServerError) return } - w.Write(reply) + _, err := w.Write(reply) + if err != nil { + log.Error("failed to write webServer reply: ", err) + } } diff --git a/status/healthz.go b/status/healthz.go index 21acbbb..d91c712 100644 --- a/status/healthz.go +++ b/status/healthz.go @@ -3,8 +3,9 @@ package status import ( "encoding/json" "fmt" - "github.com/foomo/contentserver/log" "net/http" + + "github.com/foomo/contentserver/log" ) func RunHealthzHandlerListener(address string, serviceName string) { @@ -13,13 +14,18 @@ func RunHealthzHandlerListener(address string, serviceName string) { } func HealthzHandler(serviceName string) http.Handler { - data := map[string]string{ - "service": serviceName, - } - status, _ := json.Marshal(data) - h := http.NewServeMux() + var ( + data = map[string]string{ + "service": serviceName, + } + status, _ = json.Marshal(data) + h = http.NewServeMux() + ) h.Handle("/healthz", http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - w.Write(status) + _, err := w.Write(status) + if err != nil { + log.Error("failed to write healthz status: ", err) + } })) return h From b713a41cf6a791096e6d74d97c2bc9b248b5ee79 Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Tue, 21 May 2019 10:50:47 +0200 Subject: [PATCH 20/79] updated gitignore to ignore local var dir --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index e0e37da..a07e05c 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +var .* *~ /bin/ From e64b07f6c6f0ac1790b11a1ad018215567a4b5d3 Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Tue, 21 May 2019 10:51:21 +0200 Subject: [PATCH 21/79] fixed repo test assertion messages, added benchmark for loading test repo --- repo/repo_test.go | 60 +++++++++++++++++++++++++++++++++++++---------- 1 file changed, 47 insertions(+), 13 deletions(-) diff --git a/repo/repo_test.go b/repo/repo_test.go index 8477032..89a48e3 100644 --- a/repo/repo_test.go +++ b/repo/repo_test.go @@ -15,26 +15,30 @@ func assertRepoIsEmpty(t *testing.T, r *Repo, empty bool) { } } else { if len(r.Directory) == 0 { - t.Fatal("directory should not have been empty, but it is") + t.Fatal("directory is empty, but should have been not") } } } func TestLoad404(t *testing.T) { - mockServer, varDir := mock.GetMockData(t) - server := mockServer.URL + "/repo-no-have" - r := NewRepo(server, varDir) - response := r.Update() + var ( + mockServer, varDir = mock.GetMockData(t) + server = mockServer.URL + "/repo-no-have" + r = NewRepo(server, varDir) + response = r.Update() + ) if response.Success { t.Fatal("can not get a repo, if the server responds with a 404") } } func TestLoadBrokenRepo(t *testing.T) { - mockServer, varDir := mock.GetMockData(t) - server := mockServer.URL + "/repo-broken-json.json" - r := NewRepo(server, varDir) - response := r.Update() + var ( + mockServer, varDir = mock.GetMockData(t) + server = mockServer.URL + "/repo-broken-json.json" + r = NewRepo(server, varDir) + response = r.Update() + ) if response.Success { t.Fatal("how could we load a broken json") } @@ -42,13 +46,17 @@ func TestLoadBrokenRepo(t *testing.T) { func TestLoadRepo(t *testing.T) { - mockServer, varDir := mock.GetMockData(t) - server := mockServer.URL + "/repo-ok.json" - r := NewRepo(server, varDir) + var ( + mockServer, varDir = mock.GetMockData(t) + server = mockServer.URL + "/repo-ok.json" + r = NewRepo(server, varDir) + ) assertRepoIsEmpty(t, r, true) + response := r.Update() assertRepoIsEmpty(t, r, false) - if response.Success == false { + + if !response.Success { t.Fatal("could not load valid repo") } if response.Stats.OwnRuntime > response.Stats.RepoRuntime { @@ -63,6 +71,32 @@ func TestLoadRepo(t *testing.T) { assertRepoIsEmpty(t, nr, false) } +func BenchmarkLoadRepo(b *testing.B) { + + var ( + t = &testing.T{} + mockServer, varDir = mock.GetMockData(t) + server = mockServer.URL + "/repo-ok.json" + r = NewRepo(server, varDir) + ) + if len(r.Directory) > 0 { + b.Fatal("directory should have been empty, but is not") + } + + b.ReportAllocs() + b.ResetTimer() + for n := 0; n < b.N; n++ { + response := r.Update() + if len(r.Directory) == 0 { + b.Fatal("directory is empty, but should have been not") + } + + if !response.Success { + b.Fatal("could not load valid repo") + } + } +} + func TestLoadRepoDuplicateUris(t *testing.T) { mockServer, varDir := mock.GetMockData(t) server := mockServer.URL + "/repo-duplicate-uris.json" From 5e44495adcdf223ae5b4c2837922154f81818522 Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Tue, 21 May 2019 10:59:54 +0200 Subject: [PATCH 22/79] replaced encoding/json with jsoniter high performance pkg --- client/client_test.go | 1 - client/httptransport.go | 1 - client/sockettransport.go | 4 +++- repo/loader.go | 4 +++- server/handlerequest.go | 4 +++- status/healthz.go | 4 +++- 6 files changed, 12 insertions(+), 6 deletions(-) diff --git a/client/client_test.go b/client/client_test.go index 2e9648e..dfcf5e8 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -1,7 +1,6 @@ package client import ( - "encoding/json" "net" "strconv" "sync" diff --git a/client/httptransport.go b/client/httptransport.go index aab070d..f559245 100644 --- a/client/httptransport.go +++ b/client/httptransport.go @@ -2,7 +2,6 @@ package client import ( "bytes" - "encoding/json" "errors" "io/ioutil" "net/http" diff --git a/client/sockettransport.go b/client/sockettransport.go index 96bb09f..0a7959f 100644 --- a/client/sockettransport.go +++ b/client/sockettransport.go @@ -1,7 +1,6 @@ package client import ( - "encoding/json" "errors" "fmt" "io" @@ -11,8 +10,11 @@ import ( "github.com/foomo/contentserver/responses" "github.com/foomo/contentserver/server" + jsoniter "github.com/json-iterator/go" ) +var json = jsoniter.ConfigCompatibleWithStandardLibrary + type serverResponse struct { Reply interface{} } diff --git a/repo/loader.go b/repo/loader.go index d562f44..0aa522a 100644 --- a/repo/loader.go +++ b/repo/loader.go @@ -1,7 +1,6 @@ package repo import ( - "encoding/json" "errors" "fmt" "io/ioutil" @@ -10,8 +9,11 @@ import ( "github.com/foomo/contentserver/content" "github.com/foomo/contentserver/log" + jsoniter "github.com/json-iterator/go" ) +var json = jsoniter.ConfigCompatibleWithStandardLibrary + func (repo *Repo) updateRoutine() { go func() { for newDimension := range repo.updateChannel { diff --git a/server/handlerequest.go b/server/handlerequest.go index 980bcbb..92fdfec 100644 --- a/server/handlerequest.go +++ b/server/handlerequest.go @@ -1,7 +1,6 @@ package server import ( - "encoding/json" "fmt" "time" @@ -10,9 +9,12 @@ import ( "github.com/foomo/contentserver/requests" "github.com/foomo/contentserver/responses" "github.com/foomo/contentserver/status" + jsoniter "github.com/json-iterator/go" "github.com/prometheus/client_golang/prometheus" ) +var json = jsoniter.ConfigCompatibleWithStandardLibrary + func handleRequest(r *repo.Repo, handler Handler, jsonBytes []byte, metrics *status.Metrics) (replyBytes []byte, err error) { var ( diff --git a/status/healthz.go b/status/healthz.go index d91c712..dba2936 100644 --- a/status/healthz.go +++ b/status/healthz.go @@ -1,13 +1,15 @@ package status import ( - "encoding/json" "fmt" "net/http" "github.com/foomo/contentserver/log" + jsoniter "github.com/json-iterator/go" ) +var json = jsoniter.ConfigCompatibleWithStandardLibrary + func RunHealthzHandlerListener(address string, serviceName string) { log.Notice(fmt.Sprintf("starting healthz handler on '%s'" + address)) log.Error(http.ListenAndServe(address, HealthzHandler(serviceName))) From f0df9a632237f24d63cf9ea1e06dec61c7bc2efb Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Tue, 21 May 2019 11:09:51 +0200 Subject: [PATCH 23/79] added profile-test target to makefile --- Makefile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index ae34959..bf3f796 100644 --- a/Makefile +++ b/Makefile @@ -31,4 +31,8 @@ docker-build: docker build -t $(IMAGE):$(TAG) . docker-push: - docker push $(IMAGE):$(TAG) \ No newline at end of file + docker push $(IMAGE):$(TAG) + +profile-test: + go test -run=none -bench=ClientServerParallel4 -cpuprofile=cprof net/http + go tool pprof --text http.test cprof \ No newline at end of file From dc047baf32e1d6a3d19f16dc0e0723b45fdcf8d8 Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Tue, 21 May 2019 11:10:05 +0200 Subject: [PATCH 24/79] repo code cleanup --- repo/history.go | 8 +++++--- repo/repo.go | 15 +++++++-------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/repo/history.go b/repo/history.go index 1e358de..dfdc44f 100644 --- a/repo/history.go +++ b/repo/history.go @@ -11,9 +11,11 @@ import ( "time" ) -const historyRepoJSONPrefix = "contentserver-repo-" -const historyRepoJSONSuffix = ".json" -const maxHistoryVersions = 20 +const ( + historyRepoJSONPrefix = "contentserver-repo-" + historyRepoJSONSuffix = ".json" + maxHistoryVersions = 20 +) type history struct { varDir string diff --git a/repo/repo.go b/repo/repo.go index 7d3b480..564b84c 100644 --- a/repo/repo.go +++ b/repo/repo.go @@ -12,6 +12,8 @@ import ( "github.com/foomo/contentserver/responses" ) +const maxGetURIForNodeRecursionLevel = 1000 + // Dimension dimension in a repo type Dimension struct { Directory map[string]*content.RepoNode @@ -70,8 +72,11 @@ func (repo *Repo) GetNodes(r *requests.Nodes) map[string]*content.Node { } func (repo *Repo) getNodes(nodeRequests map[string]*requests.Node, env *requests.Env) map[string]*content.Node { - nodes := map[string]*content.Node{} - path := []*content.Item{} + + var ( + nodes = map[string]*content.Node{} + path = []*content.Item{} + ) for nodeName, nodeRequest := range nodeRequests { log.Debug(" adding node " + nodeName + " " + nodeRequest.ID) @@ -223,10 +228,6 @@ func (repo *Repo) Update() (updateResponse *responses.Update) { // resolveContent find content in a repository func (repo *Repo) resolveContent(dimensions []string, URI string) (resolved bool, resolvedURI string, resolvedDimension string, repoNode *content.RepoNode) { parts := strings.Split(URI, content.PathSeparator) - resolved = false - resolvedURI = "" - resolvedDimension = "" - repoNode = nil log.Debug("repo.ResolveContent: " + URI) for i := len(parts); i > 0; i-- { testURI := strings.Join(parts[0:i], content.PathSeparator) @@ -253,8 +254,6 @@ func (repo *Repo) resolveContent(dimensions []string, URI string) (resolved bool return } -const maxGetURIForNodeRecursionLevel = 1000 - func (repo *Repo) getURIForNode(dimension string, repoNode *content.RepoNode, recursionLevel int64) (uri string) { if len(repoNode.LinkID) == 0 { uri = repoNode.URI From f20402ef5f4cb2d2593c0b80978a3347bc10a443 Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Tue, 21 May 2019 11:10:33 +0200 Subject: [PATCH 25/79] git ignore profiling files --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index a07e05c..c0dd4c1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,5 @@ +http.test +cprof var .* *~ From 0735b5ad189f41f403bf73d334de58521fa2cc29 Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Tue, 21 May 2019 11:17:03 +0200 Subject: [PATCH 26/79] code cleanup, grouping declarations --- client/client_test.go | 10 +++++----- client/connectionpool.go | 13 +++++++++---- client/sockettransport.go | 14 +++++++++----- content/content.go | 2 +- content/reponode.go | 14 ++++++++++---- contentserver.go | 10 ++++------ server/socketserver.go | 9 ++++++--- status/metrics.go | 6 ++++-- 8 files changed, 48 insertions(+), 30 deletions(-) diff --git a/client/client_test.go b/client/client_test.go index dfcf5e8..bf34007 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -16,6 +16,11 @@ import ( const pathContentserver = "/contentserver" +var ( + testServerSocketAddr string + testServerWebserverAddr string +) + func dump(t *testing.T, v interface{}) { jsonBytes, err := json.MarshalIndent(v, "", " ") if err != nil { @@ -42,11 +47,6 @@ func getAvailableAddr() string { return "127.0.0.1:" + strconv.Itoa(getFreePort()) } -var ( - testServerSocketAddr string - testServerWebserverAddr string -) - func initTestServer(t testing.TB) (socketAddr, webserverAddr string) { socketAddr = getAvailableAddr() webserverAddr = getAvailableAddr() diff --git a/client/connectionpool.go b/client/connectionpool.go index b5f2dd2..efcc348 100644 --- a/client/connectionpool.go +++ b/client/connectionpool.go @@ -34,8 +34,11 @@ func (c *connectionPool) run(connectionPoolSize int, waitTimeout time.Duration) entryTime time.Time chanConn chan net.Conn } - connectionPool := make(map[int]*poolEntry, connectionPoolSize) - waitPool := map[int]*waitPoolEntry{} + + var ( + connectionPool = make(map[int]*poolEntry, connectionPoolSize) + waitPool = map[int]*waitPoolEntry{} + ) for i := 0; i < connectionPoolSize; i++ { connectionPool[i] = &poolEntry{ conn: nil, @@ -110,8 +113,10 @@ RunLoop: } } // waitpool cleanup - waitPoolLoosers := []int{} - now := time.Now() + var ( + waitPoolLoosers = []int{} + now = time.Now() + ) for i, waitPoolEntry := range waitPool { if now.Sub(waitPoolEntry.entryTime) > waitTimeout { waitPoolLoosers = append(waitPoolLoosers, i) diff --git a/client/sockettransport.go b/client/sockettransport.go index 0a7959f..e236ed4 100644 --- a/client/sockettransport.go +++ b/client/sockettransport.go @@ -64,8 +64,10 @@ func (c *socketTransport) call(handler server.Handler, request interface{}, resp jsonBytes = append([]byte(fmt.Sprintf("%s:%d", handler, len(jsonBytes))), jsonBytes...) // send request - written := 0 - l := len(jsonBytes) + var ( + written = 0 + l = len(jsonBytes) + ) for written < l { n, err := conn.Write(jsonBytes[written:]) if err != nil { @@ -76,9 +78,11 @@ func (c *socketTransport) call(handler server.Handler, request interface{}, resp } // read response - responseBytes := []byte{} - buf := make([]byte, 4096) - responseLength := 0 + var ( + responseBytes = []byte{} + buf = make([]byte, 4096) + responseLength = 0 + ) for { n, err := conn.Read(buf) if err != nil && err != io.EOF { diff --git a/content/content.go b/content/content.go index 61cc286..9c67eba 100644 --- a/content/content.go +++ b/content/content.go @@ -4,6 +4,6 @@ package content const ( // Indent for json indentation Indent string = "\t" - // PathSeparator seprator for paths in URIs + // PathSeparator separator for paths in URIs PathSeparator = "/" ) diff --git a/content/reponode.go b/content/reponode.go index 4c806d1..faf9e79 100644 --- a/content/reponode.go +++ b/content/reponode.go @@ -52,15 +52,21 @@ func (node *RepoNode) InPath(path []*Item) bool { // GetPath get a path for a repo node func (node *RepoNode) GetPath() []*Item { - parentNode := node.parent - pathLength := 0 + + var ( + parentNode = node.parent + pathLength = 0 + ) for parentNode != nil { parentNode = parentNode.parent pathLength++ } parentNode = node.parent - i := 0 - path := make([]*Item, pathLength) + + var ( + i = 0 + path = make([]*Item, pathLength) + ) for parentNode != nil { path[i] = parentNode.ToItem([]string{}) parentNode = parentNode.parent diff --git a/contentserver.go b/contentserver.go index 425b1a2..59643f3 100644 --- a/contentserver.go +++ b/contentserver.go @@ -3,11 +3,12 @@ package main import ( "flag" "fmt" - "github.com/foomo/contentserver/metrics" - "github.com/foomo/contentserver/status" "os" "strings" + "github.com/foomo/contentserver/metrics" + "github.com/foomo/contentserver/status" + "github.com/foomo/contentserver/log" "github.com/foomo/contentserver/server" ) @@ -18,11 +19,8 @@ const ( logLevelWarning = "warning" logLevelRecord = "record" logLevelError = "error" -) - -const ( - ServiceName = "Content Server" + ServiceName = "Content Server" DefaultHealthzHandlerAddress = ":8080" DefaultPrometheusListener = ":9200" ) diff --git a/server/socketserver.go b/server/socketserver.go index d030ed7..cb07f57 100644 --- a/server/socketserver.go +++ b/server/socketserver.go @@ -70,9 +70,12 @@ func (s *socketServer) writeResponse(conn net.Conn, reply []byte) { func (s *socketServer) handleConnection(conn net.Conn) { log.Debug("socketServer.handleConnection") - var headerBuffer [1]byte - header := "" - i := 0 + + var ( + headerBuffer [1]byte + header = "" + i = 0 + ) for { i++ // fmt.Println("---->", i) diff --git a/status/metrics.go b/status/metrics.go index 212ae29..8ccc82d 100644 --- a/status/metrics.go +++ b/status/metrics.go @@ -4,8 +4,10 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -const MetricLabelHandler = "handler" -const MetricLabelStatus = "status" +const ( + MetricLabelHandler = "handler" + MetricLabelStatus = "status" +) type Metrics struct { ServiceRequestCounter *prometheus.CounterVec // count the number of requests for each service function From 65b6b2341afaa7fb8b843c1e77699c26d11a20ef Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Tue, 21 May 2019 11:43:06 +0200 Subject: [PATCH 27/79] added free-os-mem flag to contentserver --- contentserver.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/contentserver.go b/contentserver.go index 59643f3..7f1cc79 100644 --- a/contentserver.go +++ b/contentserver.go @@ -4,7 +4,9 @@ import ( "flag" "fmt" "os" + "runtime/debug" "strings" + "time" "github.com/foomo/contentserver/metrics" "github.com/foomo/contentserver/status" @@ -32,6 +34,7 @@ var ( webserverAddress = flag.String("webserver-address", "", "address to bind web server host:port, when empty no webserver will be spawned") webserverPath = flag.String("webserver-path", "/contentserver", "path to export the webserver on - useful when behind a proxy") varDir = flag.String("var-dir", "/var/lib/contentserver", "where to put my data") + flagFreeOSMem = flag.Int("free-os-mem", 0, "free OS mem every X minutes") logLevelOptions = []string{ logLevelError, logLevelRecord, @@ -57,10 +60,22 @@ func exitUsage(code int) { func main() { flag.Parse() + if *showVersionFlag { fmt.Printf("%v\n", uniqushPushVersion) return } + + if *flagFreeOSMem > 0 { + log.Notice("[INFO] freeing OS memory every ", *flagFreeOSMem, " minutes!") + go func() { + for _ = range time.After(time.Duration(*flagFreeOSMem) * time.Minute) { + log.Notice("FreeOSMemory") + debug.FreeOSMemory() + } + }() + } + if len(flag.Args()) == 1 { fmt.Println(*address, flag.Arg(0)) level := log.LevelRecord From 03d5b36706d9189da929805f4695c1ad50a6f285 Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Tue, 21 May 2019 11:43:28 +0200 Subject: [PATCH 28/79] repo history code cleanup --- repo/history.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/repo/history.go b/repo/history.go index dfdc44f..8d7f579 100644 --- a/repo/history.go +++ b/repo/history.go @@ -28,18 +28,21 @@ func newHistory(varDir string) *history { } func (h *history) add(jsonBytes []byte) error { - // historic file name - filename := path.Join(h.varDir, historyRepoJSONPrefix+time.Now().Format(time.RFC3339Nano)+historyRepoJSONSuffix) - err := ioutil.WriteFile(filename, jsonBytes, 0644) + + var ( + // historiy file name + filename = path.Join(h.varDir, historyRepoJSONPrefix+time.Now().Format(time.RFC3339Nano)+historyRepoJSONSuffix) + err = ioutil.WriteFile(filename, jsonBytes, 0644) + ) if err != nil { return err } + // current filename return ioutil.WriteFile(h.getCurrentFilename(), jsonBytes, 0644) } func (h *history) getHistory() (files []string, err error) { - files = []string{} fileInfos, err := ioutil.ReadDir(h.varDir) if err != nil { return From 295cdf66fcae81d17d0fe78cdfa1d27047899967 Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Tue, 21 May 2019 11:45:47 +0200 Subject: [PATCH 29/79] flag naming convention --- contentserver.go | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/contentserver.go b/contentserver.go index 7f1cc79..8bd9ed6 100644 --- a/contentserver.go +++ b/contentserver.go @@ -29,13 +29,15 @@ const ( var ( uniqushPushVersion = "content-server 1.5.0" - showVersionFlag = flag.Bool("version", false, "version info") - address = flag.String("address", "", "address to bind socket server host:port") - webserverAddress = flag.String("webserver-address", "", "address to bind web server host:port, when empty no webserver will be spawned") - webserverPath = flag.String("webserver-path", "/contentserver", "path to export the webserver on - useful when behind a proxy") - varDir = flag.String("var-dir", "/var/lib/contentserver", "where to put my data") - flagFreeOSMem = flag.Int("free-os-mem", 0, "free OS mem every X minutes") - logLevelOptions = []string{ + + flagShowVersionFlag = flag.Bool("version", false, "version info") + flagAddress = flag.String("address", "", "address to bind socket server host:port") + flagWebserverAddress = flag.String("webserver-address", "", "address to bind web server host:port, when empty no webserver will be spawned") + flagWebserverPath = flag.String("webserver-path", "/contentserver", "path to export the webserver on - useful when behind a proxy") + flagVarDir = flag.String("var-dir", "/var/lib/contentserver", "where to put my data") + flagFreeOSMem = flag.Int("free-os-mem", 0, "free OS mem every X minutes") + + logLevelOptions = []string{ logLevelError, logLevelRecord, logLevelWarning, @@ -61,7 +63,7 @@ func exitUsage(code int) { func main() { flag.Parse() - if *showVersionFlag { + if *flagShowVersionFlag { fmt.Printf("%v\n", uniqushPushVersion) return } @@ -77,7 +79,8 @@ func main() { } if len(flag.Args()) == 1 { - fmt.Println(*address, flag.Arg(0)) + fmt.Println(*flagAddress, flag.Arg(0)) + level := log.LevelRecord switch *logLevel { case logLevelError: @@ -92,10 +95,12 @@ func main() { level = log.LevelDebug } log.SelectedLevel = level + + // kickoff metric handlers go metrics.RunPrometheusHandler(DefaultPrometheusListener) go status.RunHealthzHandlerListener(DefaultHealthzHandlerAddress, ServiceName) - err := server.RunServerSocketAndWebServer(flag.Arg(0), *address, *webserverAddress, *webserverPath, *varDir) + err := server.RunServerSocketAndWebServer(flag.Arg(0), *flagAddress, *flagWebserverAddress, *flagWebserverPath, *flagVarDir) if err != nil { fmt.Println("exiting with error", err) os.Exit(1) From 55fd63b82d1439f25a0a239a3125e174e3ef3865 Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Tue, 21 May 2019 12:11:16 +0200 Subject: [PATCH 30/79] added memory profiling flag for dumping heap periodically --- contentserver.go | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/contentserver.go b/contentserver.go index 8bd9ed6..60e80e2 100644 --- a/contentserver.go +++ b/contentserver.go @@ -35,7 +35,10 @@ var ( flagWebserverAddress = flag.String("webserver-address", "", "address to bind web server host:port, when empty no webserver will be spawned") flagWebserverPath = flag.String("webserver-path", "/contentserver", "path to export the webserver on - useful when behind a proxy") flagVarDir = flag.String("var-dir", "/var/lib/contentserver", "where to put my data") - flagFreeOSMem = flag.Int("free-os-mem", 0, "free OS mem every X minutes") + + // debugging / profiling + flagFreeOSMem = flag.Int("free-os-mem", 0, "free OS mem every X minutes") + flagHeapDump = flag.Int("heap-dump", 0, "dump heap every X minutes") logLevelOptions = []string{ logLevelError, @@ -78,6 +81,24 @@ func main() { }() } + if *flagHeapDump > 0 { + log.Notice("[INFO] dumping heap every ", *flagHeapDump, " minutes!") + go func() { + for _ = range time.After(time.Duration(*flagFreeOSMem) * time.Minute) { + log.Notice("HeapDump") + f, err := os.Create("heapdump") + if err != nil { + panic("failed to create heap dump file") + } + debug.WriteHeapDump(f.Fd()) + err = f.Close() + if err != nil { + panic("failed to create heap dump file") + } + } + }() + } + if len(flag.Args()) == 1 { fmt.Println(*flagAddress, flag.Arg(0)) From 3985784579cf8b779b694955efa98ad24a3e16d9 Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Tue, 21 May 2019 12:11:39 +0200 Subject: [PATCH 31/79] added pprof debug server --- server/server.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/server.go b/server/server.go index 60e51de..dba4a19 100644 --- a/server/server.go +++ b/server/server.go @@ -8,6 +8,9 @@ import ( "github.com/foomo/contentserver/log" "github.com/foomo/contentserver/repo" + + // profiling + _ "net/http/pprof" ) // Handler type From 63640b24b27bfb4d579f50cbdb0be0748f5175d3 Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Tue, 21 May 2019 12:11:54 +0200 Subject: [PATCH 32/79] updated makefile targets for profiling --- .gitignore | 4 ++-- Makefile | 29 ++++++++++++++++++++++++----- 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index c0dd4c1..4230dfc 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ -http.test -cprof +*.test +cprof-* var .* *~ diff --git a/Makefile b/Makefile index bf3f796..2ad54f5 100644 --- a/Makefile +++ b/Makefile @@ -24,8 +24,8 @@ build-docker: clean build-arch package: build pkg/build.sh -test: - go test ./... + +# docker docker-build: docker build -t $(IMAGE):$(TAG) . @@ -33,6 +33,25 @@ docker-build: docker-push: docker push $(IMAGE):$(TAG) -profile-test: - go test -run=none -bench=ClientServerParallel4 -cpuprofile=cprof net/http - go tool pprof --text http.test cprof \ No newline at end of file +# testing / benchmarks + +test: + go test ./... + +bench: + go test -run=none -bench=. ./... + +# profiling + +test-cpu-profile: + go test -cpuprofile=cprof-client github.com/foomo/contentserver/client + go tool pprof --text client.test cprof-client + + go test -cpuprofile=cprof-repo github.com/foomo/contentserver/repo + go tool pprof --text repo.test cprof-repo + +test-gctrace: + GODEBUG=gctrace=1 go test ./... + +test-malloctrace: + GODEBUG=allocfreetrace=1 go test ./... \ No newline at end of file From 0827eb9b4a7f6c7caa2e3b6b2676414c21bc8671 Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Tue, 21 May 2019 12:35:56 +0200 Subject: [PATCH 33/79] added makefile section comments --- Makefile | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index 2ad54f5..dd20b00 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,8 @@ SHELL := /bin/bash TAG?=latest IMAGE=docker-registry.bestbytes.net/contentserver +# Utils + all: build test tag: echo $(TAG) @@ -10,6 +12,9 @@ dep: go mod download && go mod vendor && go install -i ./vendor/... clean: rm -fv bin/contentserve* + +# Build + build: clean go build -o bin/contentserver build-arch: clean @@ -25,7 +30,7 @@ build-docker: clean build-arch package: build pkg/build.sh -# docker +# Docker docker-build: docker build -t $(IMAGE):$(TAG) . @@ -33,7 +38,7 @@ docker-build: docker-push: docker push $(IMAGE):$(TAG) -# testing / benchmarks +# Testing / benchmarks test: go test ./... @@ -41,7 +46,7 @@ test: bench: go test -run=none -bench=. ./... -# profiling +# Profiling test-cpu-profile: go test -cpuprofile=cprof-client github.com/foomo/contentserver/client From 18897d2e32ff41b70d0732a8b7d2507c6404fe02 Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Tue, 21 May 2019 12:36:06 +0200 Subject: [PATCH 34/79] updated deps --- go.mod | 7 ++++++- go.sum | 6 ++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 1fe38f3..e57b7a4 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,8 @@ module github.com/foomo/contentserver -require github.com/prometheus/client_golang v0.9.2 +require ( + github.com/json-iterator/go v1.1.6 + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.1 // indirect + github.com/prometheus/client_golang v0.9.2 +) diff --git a/go.sum b/go.sum index 904fd75..ae534c7 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,14 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLM github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= +github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740= github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8= From 1449d6902cc4e8af11f7c77554305390379ae502 Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Tue, 21 May 2019 15:01:13 +0200 Subject: [PATCH 35/79] server cleanup --- server/handlerequest.go | 3 --- server/server.go | 17 +++++------------ server/socketserver.go | 19 +++++++++++-------- server/webserver.go | 9 ++++----- 4 files changed, 20 insertions(+), 28 deletions(-) diff --git a/server/handlerequest.go b/server/handlerequest.go index 92fdfec..ccac8b7 100644 --- a/server/handlerequest.go +++ b/server/handlerequest.go @@ -9,12 +9,9 @@ import ( "github.com/foomo/contentserver/requests" "github.com/foomo/contentserver/responses" "github.com/foomo/contentserver/status" - jsoniter "github.com/json-iterator/go" "github.com/prometheus/client_golang/prometheus" ) -var json = jsoniter.ConfigCompatibleWithStandardLibrary - func handleRequest(r *repo.Repo, handler Handler, jsonBytes []byte, metrics *status.Metrics) (replyBytes []byte, err error) { var ( diff --git a/server/server.go b/server/server.go index dba4a19..bf7e249 100644 --- a/server/server.go +++ b/server/server.go @@ -8,11 +8,14 @@ import ( "github.com/foomo/contentserver/log" "github.com/foomo/contentserver/repo" + jsoniter "github.com/json-iterator/go" // profiling _ "net/http/pprof" ) +var json = jsoniter.ConfigCompatibleWithStandardLibrary + // Handler type type Handler string @@ -64,12 +67,7 @@ func runWebserver( path string, chanErr chan error, ) { - s, errNew := NewWebServer(path, r) - if errNew != nil { - chanErr <- errNew - return - } - chanErr <- http.ListenAndServe(address, s) + chanErr <- http.ListenAndServe(address, NewWebServer(path, r)) } func runSocketServer( @@ -78,12 +76,7 @@ func runSocketServer( chanErr chan error, ) { // create socket server - s, errSocketServer := newSocketServer(repo) - if errSocketServer != nil { - log.Error(errSocketServer) - chanErr <- errSocketServer - return - } + s := newSocketServer(repo) // listen on socket ln, errListen := net.Listen("tcp", address) diff --git a/server/socketserver.go b/server/socketserver.go index cb07f57..eb64d1b 100644 --- a/server/socketserver.go +++ b/server/socketserver.go @@ -18,14 +18,12 @@ type socketServer struct { metrics *status.Metrics } -// @TODO: remove error ? // newSocketServer returns a shiny new socket server -func newSocketServer(repo *repo.Repo) (s *socketServer, err error) { - s = &socketServer{ +func newSocketServer(repo *repo.Repo) *socketServer { + return &socketServer{ repo: repo, metrics: status.NewMetrics("socketserver"), } - return } func extractHandlerAndJSONLentgh(header string) (handler Handler, jsonLength int, err error) { @@ -104,12 +102,17 @@ func (s *socketServer) handleConnection(conn net.Conn) { } log.Debug(fmt.Sprintf(" found json with %d bytes", jsonLength)) if jsonLength > 0 { - // let us try to read some json - jsonBytes := make([]byte, jsonLength) + + var ( + // let us try to read some json + jsonBytes = make([]byte, jsonLength) + jsonLengthCurrent = 1 + readRound = 0 + ) + // that is "{" jsonBytes[0] = 123 - jsonLengthCurrent := 1 - readRound := 0 + for jsonLengthCurrent < jsonLength { readRound++ readLength, jsonReadErr := conn.Read(jsonBytes[jsonLengthCurrent:jsonLength]) diff --git a/server/webserver.go b/server/webserver.go index c638a45..4c5470a 100644 --- a/server/webserver.go +++ b/server/webserver.go @@ -12,19 +12,18 @@ import ( ) type webServer struct { - r *repo.Repo path string + r *repo.Repo metrics *status.Metrics } // NewWebServer returns a shiny new web server -func NewWebServer(path string, r *repo.Repo) (s http.Handler, err error) { - s = &webServer{ - r: r, +func NewWebServer(path string, r *repo.Repo) http.Handler { + return &webServer{ path: path, + r: r, metrics: status.NewMetrics("webserver"), } - return } func (s *webServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { From f7aea048d36a5609818424473bfac45a95575d20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Frederik=20L=C3=B6ffert?= Date: Tue, 21 May 2019 16:06:49 +0200 Subject: [PATCH 36/79] lock update requests --- repo/loader.go | 5 +++++ repo/repo.go | 3 +++ 2 files changed, 8 insertions(+) diff --git a/repo/loader.go b/repo/loader.go index 0aa522a..dbb7d84 100644 --- a/repo/loader.go +++ b/repo/loader.go @@ -130,6 +130,11 @@ func get(URL string) (data []byte, err error) { } func (repo *Repo) update() (repoRuntime int64, jsonBytes []byte, err error) { + + // limit ressources and allow only one update request at once + repo.updateLock.Lock() + defer repo.updateLock.Unlock() + startTimeRepo := time.Now().UnixNano() jsonBytes, err = get(repo.server) repoRuntime = time.Now().UnixNano() - startTimeRepo diff --git a/repo/repo.go b/repo/repo.go index 564b84c..b2cb75b 100644 --- a/repo/repo.go +++ b/repo/repo.go @@ -4,6 +4,7 @@ import ( "errors" "fmt" "strings" + "sync" "time" "github.com/foomo/contentserver/content" @@ -25,6 +26,7 @@ type Dimension struct { type Repo struct { server string Directory map[string]*Dimension + updateLock *sync.Mutex updateChannel chan *repoDimension updateDoneChannel chan error history *history @@ -43,6 +45,7 @@ func NewRepo(server string, varDir string) *Repo { server: server, Directory: map[string]*Dimension{}, history: newHistory(varDir), + updateLock: &sync.Mutex{}, updateChannel: make(chan *repoDimension), updateDoneChannel: make(chan error), } From eca5e3b4f0908ca238dcc9e9aa9ea85c69704396 Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Tue, 21 May 2019 17:29:16 +0200 Subject: [PATCH 37/79] fixed loops for mem profiling --- contentserver.go | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/contentserver.go b/contentserver.go index 60e80e2..e12249e 100644 --- a/contentserver.go +++ b/contentserver.go @@ -74,9 +74,12 @@ func main() { if *flagFreeOSMem > 0 { log.Notice("[INFO] freeing OS memory every ", *flagFreeOSMem, " minutes!") go func() { - for _ = range time.After(time.Duration(*flagFreeOSMem) * time.Minute) { - log.Notice("FreeOSMemory") - debug.FreeOSMemory() + for { + select { + case <-time.After(time.Duration(*flagFreeOSMem) * time.Second): + log.Notice("FreeOSMemory") + debug.FreeOSMemory() + } } }() } @@ -84,16 +87,19 @@ func main() { if *flagHeapDump > 0 { log.Notice("[INFO] dumping heap every ", *flagHeapDump, " minutes!") go func() { - for _ = range time.After(time.Duration(*flagFreeOSMem) * time.Minute) { - log.Notice("HeapDump") - f, err := os.Create("heapdump") - if err != nil { - panic("failed to create heap dump file") - } - debug.WriteHeapDump(f.Fd()) - err = f.Close() - if err != nil { - panic("failed to create heap dump file") + for { + select { + case <-time.After(time.Duration(*flagFreeOSMem) * time.Minute): + log.Notice("HeapDump") + f, err := os.Create("heapdump") + if err != nil { + panic("failed to create heap dump file") + } + debug.WriteHeapDump(f.Fd()) + err = f.Close() + if err != nil { + panic("failed to create heap dump file") + } } } }() From 97633dc9d9651cb11b90971a139652375e7bbc9e Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Tue, 21 May 2019 17:30:20 +0200 Subject: [PATCH 38/79] debug mode: only print number of json bytes and dont dump the entire beast to stdout --- repo/loader.go | 2 +- server/socketserver.go | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/repo/loader.go b/repo/loader.go index 0aa522a..c8544ad 100644 --- a/repo/loader.go +++ b/repo/loader.go @@ -138,7 +138,7 @@ func (repo *Repo) update() (repoRuntime int64, jsonBytes []byte, err error) { log.Debug("we have no json to load - the repo server did not reply", err) return repoRuntime, jsonBytes, err } - log.Debug("loading json from: "+repo.server, string(jsonBytes)) + log.Debug("loading json from: "+repo.server, "length:", len(jsonBytes)) nodes, err := loadNodesFromJSON(jsonBytes) if err != nil { // could not load nodes from json diff --git a/server/socketserver.go b/server/socketserver.go index eb64d1b..f667e3e 100644 --- a/server/socketserver.go +++ b/server/socketserver.go @@ -40,7 +40,8 @@ func extractHandlerAndJSONLentgh(header string) (handler Handler, jsonLength int func (s *socketServer) execute(handler Handler, jsonBytes []byte) (reply []byte) { if log.SelectedLevel == log.LevelDebug { - log.Debug(" incoming json buffer:", string(jsonBytes)) + log.Debug(" incoming json buffer of length: ", len(jsonBytes)) + // log.Debug(" incoming json buffer:", string(jsonBytes)) } reply, handlingError := handleRequest(s.repo, handler, jsonBytes, s.metrics) if handlingError != nil { @@ -127,7 +128,8 @@ func (s *socketServer) handleConnection(conn net.Conn) { } if log.SelectedLevel == log.LevelDebug { - log.Debug(" read json: " + string(jsonBytes)) + log.Debug(" read json, length: ", len(jsonBytes)) + // log.Debug(" read json: " + string(jsonBytes)) } s.writeResponse(conn, s.execute(handler, jsonBytes)) // note: connection remains open From a4097c05f41b310510a23c5d06b30bd16ffcee0e Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Tue, 21 May 2019 17:30:39 +0200 Subject: [PATCH 39/79] added targets for testing --- Makefile | 15 +++++++++++++++ contentserver.graffle | Bin 0 -> 7838 bytes testing/client/client.go | 32 ++++++++++++++++++++++++++++++++ testing/server/server.go | 35 +++++++++++++++++++++++++++++++++++ 4 files changed, 82 insertions(+) create mode 100644 contentserver.graffle create mode 100644 testing/client/client.go create mode 100644 testing/server/server.go diff --git a/Makefile b/Makefile index dd20b00..bc665ea 100644 --- a/Makefile +++ b/Makefile @@ -27,6 +27,12 @@ build-docker: clean build-arch echo "# tagged container `cat .image_id` as $(IMAGE):$(TAG)" rm -vf .image_id .cacert.pem +build-testclient: + go build -o bin/testclient -i github.com/foomo/contentserver/testing/client + +build-testserver: + go build -o bin/testserver -i github.com/foomo/contentserver/testing/server + package: build pkg/build.sh @@ -46,6 +52,15 @@ test: bench: go test -run=none -bench=. ./... +run-testserver: + bin/testserver -json-file var/cse-globus-stage-b-with-main-section.json + +run-contentserver: + contentserver -var-dir var -webserver-address :9191 -address :9999 -log-level debug http://127.0.0.1:1234 + +clean-var: + rm var/contentserver-repo-2019* + # Profiling test-cpu-profile: diff --git a/contentserver.graffle b/contentserver.graffle new file mode 100644 index 0000000000000000000000000000000000000000..75addb694fe252646213483693202f7f577e7fac GIT binary patch literal 7838 zcmV;P9%11hiwFP!000030PTHild8y;_UGS#0=;g_dhygXn3tI@yiV zYjc09(6*5qM^JeY9-?0WfU(T%)@IbD7ebFa%O-)>~+XBD^0|0a|7U5q_ zIFxc0gU_T8o`4@&QE9#}$P&hbcb}=L0l=W2r5v`$d4+-+zvENedHs>}Cb_rEUn)92 zEI9R`7x0%}+C^rioZXZPgjphtqDOgk>>}G0EA$pxX^n?WQWm(kDCN*Ozo=x6n4r?J z;PB0&{DtaI7VtP&9PP}?a};mKYKSmcse|I|C=BiLXAhMdn0+-j%AXDQSU<(oRL_DK z5}a0h?HKHM%JwCAa>RG$s%!?eDe%yeQj<2PfOWER@Ov$Y*o z%JD4n-^wI*TYJvCR+IE}LUBhjc~AnIx+)6kLvnX?exwii@4e2q9~h69u{dG}C_WdF zx?gDR)j}pULsOfE){apeN7CBn!ma6(Yx61&OSyNkNbO{3YiLF4rnE~f`!5l07JJx_ zh<0GiCFDPoXt{>g`m7J!@dHVeHrA(S>uC$HYXe1?_X3jvNsdPV2v+PK<=Bgu zCC8Mi0|oT~YuP67P@kXM1c)E?qu~^N#lm{Aui^MW;W+`fV*OICZWsknE|>yR1e^#Q zK#jo=hB}~mAQa%GL`fn!1WXsnsW6BE+X8w)CV<0$<&sU%FJRiGIKXExl~O7gBp@!R z01N|=x)kyQ7+GKoXcCNdurFv42nrwy+65B@bPAZ%raE{o7!rsUNsG)3NC8>O^$R8h zaz>6_rUhm#QWDuQPJak%6R=+?cUM>6zpUCUWJ39O=vs%x2_-fnJzBmvW?tv3;PV z0KdrA^a-v$nnBilhg1x|zP@%^^AY*8TN`-Quw7@gN#Z$t4s zAG-c4HEQvgyNao4HYnv75QNk$8O(|>pyt6~RYU=0CWAu}htx6~GGY=_D{QEO^(&Py z!w9@?N;xW1hE0$js3IHjZq}x7W@LyPK(*ab2DUu)%#5DJLZ+!?EEYS9?ki~AT)F$0 z9)mF<9uBmK2?G}gw3-MuASSdC2u0C!=%qVhiq?h>g2@838>Jk*&L+A0BGa4JloGok zjWN@?+w0J6eVP@GDg9iClmsG%8H>6^J~M+%Eaf&5J!hth#O&aIN>?%^=sY8l_EJA# zTuUB!1D1(I*&~N5CL7DG-5B|u*^g)94rZ3tc(x{|Ec2`=q$kEK6DWo!kJurm9K2b< zO1vtNs>aS;wUmo?T90*r2IDPagFrJ$W6GwyhI?j*-C=W4vO~5N%u{bEvkgn>`OXtwmEx7|u@&MOrtxb<5aEX~ykDvy_|c@)g%g%*@*w zeZX5oZzuQrnU(Agr*Vx&lS{f6S+U(mzh7<4^%$k{d7OF`65|OC?Vq8;;Qhc z#enPvGg;~mhFL)P3W~GA%nc1nn-452?Cte<05iO|H=2Wu8*`LB7&Mdkb-%QR086HO zZ#NwBNhya@0XLkvYhxeTLmRvTKf#BQ`x@=j>9F8a!Oz#jHkM*?!H(!4GyT1f#udTJ zX$mJtb3K3EH@%UkZ)Q}78l~3e#qS0qJlO_vqd#i23SmZORj=$4atuS6pYj zB%V@y9R50loo!C=FK6bgeDM$pO(^;-qQy~CUmmKY%%7JK2fMRaCX zAP}G#)64mU6k7`Ik0)cdP0m7UGViuaxh);HCo8>U)35u{o8ymaUyo^nce zO&?ZMneXW{!kaqanWx)sDK~uu&n>#^O$nxvtQx*Z31GockvOr*p~Ak3#*Tam_(F^a zRIZGg;$e;AdZ|o-1O!IHl2U~CBW1*M+!4-8EU(fz}QP5SVsu?|yQ!&Ea=x8a;tEiXK1bJ?~XwmF!4 zYsX|ey?N1F>gqF^_vk=%Rf2ler#o*(aK64&eOVh?`j%XY#AL2N=Yhz{%3>0RQ%#c> zjvXOCXokGlc=1$oW{c-AnG(KcOm=ILhzw)NzRKL|(%9PRjM&6RqnD|~ev9TWqL=F& z(Kfxo#voo8bJyNFTtl*`yzpY$wPx#GIPVu$knh*=n6z*l*}3V9uv3%Rp|2#iuHr3d znZ3@5=f17kFG7o%dmCrGYY*dS;5c?iieJ(E%qgT7y}6~96x(&@2XCpcy=2}Zm(k$4 zh`Xz0OKP-rvhNOkl1kXrwForcj`^YNZe?0+O*$SNa9n=2@)U=CZRry)9&&igdhy!N zJjr`&f7}_2TM_buzw8Vbd^+?G)F@20-4!<(?-OjdTKGbv)s9wKXDTGmP5=ziT+>=G zdq#2gI^d-sewABoK?q(9g&)Yna4M_w_Iv=NPm9~uFl=h`*1#J_BY!a%L@3Wk%e`R? zQZ>ScX4)>Sn2)Sp8xLZ8=!k>PCf*TCzuo90T;vi%wx5^=$`gavp_UZJ%6^?jYt{&M z9i_E4qKJDnM(ZsRo3Bgvl^rI#VbFXvqSyZLb&BjPhN*3A*D1Ttup!Q*roR~(9WUJ( zTmJ=stgjWFlyx)P*vl^?Dm(bde`zX)`CyMTFWNA7Il?i`A}9Q2Yve690@s$)nA)s1 zo!rben~u}v%{|KVTS@D!URuL#B0o1?dYx@!4y-Yq721+)bqC8L-_qK6s_fW4bI8=u z&RU^)fs~o_4(s=~UAMO%9R`yuYVTL8VcyM%{fjvktUZ31&4t`*5r@>A3f<=xS}&ku zag#G$%zTljy%C9Fv)L|}2KacZ&hoj0hjWb(Yzu#;^@XrX2a~=acbE)jt$e%EpQmXl*X~b)#zbSPAJz>K>y1aa zGMziUs7X9GoZ&MWw_c|Ks^7DEzo_Uum6Y4)66dr3G1j(6ZC>&3xLvxhrxR-GEhRjsSV zrPM)-TeOj%N;;N9V0iCWtowm68cH%AClMzq1}Fr9Qjc}R$`ZNf1@B0tH!Z2THJIqJ=y79;zIkBdRqZxr%J2PeLky$b8Ko1K?d6YqR^ zdg&2=eo|-|eUek|RX8xnVL$rIxnJ^K@1N@bS+f3Ts6UULrB=x?>`5H%KTjC9lJSil z8(`)B))^UlC*@p?t%wSZ z?3SK-LrwNz7tFp=M!Yq$8}_}^$J3X}?$oySTD}jROZ2&p^{mV@?4-Cryn$30IyH(* z`+vF}CO`djF*7?Ckf%xD{8I{e^SczDIlKI$%9htbn8fbX$UQFHuRRc3-g&NfS+eEC zSGn^|ftNq4xXPSGl0*-NVdZSvhf|1h`j=HB{rlPRAI)~N47Bdc+rYX|a#s}l@`T-$ zdXw=litEC-?zQlbgMvd^V*wZ?H`uh|5v3p;wx|?O?Zu*7zIn}-oqj!nnA~QgLeMPCE z+d-M3+xqga!|-hvyo6kA-oF7| zp(>d97E*f-`-)w-?u>T73!UCX<2SKacbsq9H%ndfs#&kH*WKK7KRI0~xRYRSwe0`6 zly_P7FXg{`=#M=3k1QPAYQ8rkR^!%?@o~qwMx2wuw=`ek-wc@H(I0E=;zXPHW+hf) zl$Sa$cy*9%OnBVFu%G|U7I5f@q?aun+VeEAxNu*AZ-jmpmh1w!4ahM0wJYdKZW;mo74!uy;|joyqqjM z@zSvJB*lLI8PzUp5>MZ(CH6$YD#OwCNlrGYs0qd^#{klJat?Dq5+`&V*vOu0Q?vLehdJAr2$|f%ot(z zbs$fKzf{ie!P$Mg)&939{nn49|0C)DNcumL{$D8R&!{P!@mqnDKb6ycC;iK1{!Z)V z>%TwuH-F^*AG!ZW?*Ebd|3bN6bIiX)(ytcti5I-i^W|!Ot4sWgGCT1jv;WBKKQjBj zFtbZDmcDBE<1>uqTOuDkXi`HwaxhUa7K5<09;?_wlV;^_GqS)D!i)SY6sCH#4)vx8 zyv2~I&&_H`EkTC7NDd2pWfEr1fUc=@I;EM8ef8qFJM*_rbk=mQes6td_j0;vlD|m2lPe! z{okK_lozn-G=@%McjqJDtjtY~{WlByzce)=o~&v!>fbUuxp)Zd_;7-6Up}Py{vo9j z@$ONHTO(=xTuE*tGvfETco(2S*ns=$65*wh^MUy6|xcDYI0Vkw%f^*&I7%#9jH;_P9bmLVQq{ zenEe5kNV(XrOaMJlH8A63DDG@%SL#UBAND;0H z5$e((q(yj)7@>~3E9!$=)Cd*wGlGPY9HBx!LXl7*FNqTFkt9^ekEjxkGzoPiKR}l7 z5699KX~HF8LWO*UHsP%EGvb5_d5b>bnmVD5r3VQV9-~mGkk>>C|6A?(8JR+be1!g> zw(|p0g`?gY`2elLG1J~9Yq(YS8}tg7)Cxb>^Wx757V4P46~R{&3l;Jf(ZV&!!m%Gc zMz!!5*+PZ9rd+s1w@@KJBVD)`OSfqkD&!^k!ad@JTIUD!3rFgO+V2k#Fw{D08itQ3 z7%JpFB8E#6hFa$XR16ieBxLxEjG@+fgOcHzj-f(6O3LsUEklL8CTF-s%updeqh~0o z87kx>1P!&$3z~*|6b%*fBcg^QNkgsk0jh>tXHD4f5m`fpyhqt^N!L)v`~##76|$sm z_>8upLf#^8xF&9>bv{Vn@ECzZg}kP5xJBVmxBMqW4kd|0g?xm{p^ncBLWg@~4i)kv zN{1tzLxp^V)S-^#n%dzbT8FwucgP(si5=<)d4S%bLY5>CpAkIN_eVEq9df#U z(Zgd@4;Av7@ZlEOLp^8sgz}-Jd#H2W1EdcX@{;=D9_>S&=RY8SI1)cp$VccO>KR~7 z0`U<6L_PbxLj!S10Z}0zA%dvyVeQ@IGb)IBJi0*$aZLtM&sZL$gm{b;qC#F%L)@Z; zsPADvA%`f5A?kVZ1N0DeD_xL8+#`sn@3}spi8xY3RLDn&A}ZvOF5)Aqh&s>TA&j^r zi>Q0Y1C$YUk84GX&qyOGI)88x6amBD+IXSKh8hp&- zJBY5fP^wkEd1+03r#}qs)78WeR`VU0&O%|dn1|tlsa83EK5gn?GPY6sgRR+=%eyij zkPP5lumThY#xAJi6VToA&PfC;R&JCOfLBV028Iks3MvG{<9-voH^RWE1qwh@XY0I%oXw2}ylh#R!p6-3JczA~B%#LWo_$qYeG-y&-jR*SNiV z4=cQazyJN`_uCoowpFeY~W_*P}?B)?IcxMZvuL5YzQySaCxAfkRU zk?3_XqDc^~cLgII-)U@KBy0a+nd|;?i}>N=cZnyPJcPBpdIILe0}&c=@YT(oXdR+Z zxPI05Qe2Tnw}_?nsJAsRw_d{%MYKXT5TjRbq0APn%IOm|1CyV()cr5tX~-lz1vfS!j3xM`vtNtw#<{#II$Wu&T~;>}Elcs*Gg1r*g$; z=b2HqSu~~4oL9uzunr-h5s)mOGhnK5s{9nEWlk3D^`HujkA$&2+V4Rvh#W9R&nKYc z9;keX);8g3BUc6&M6-Nl5Y)uxEy_I(ji@vlUa_3YZzml{4(bW|!LYt?)-fuyesQgX z>akw;f)Mv?%huDE?{#$F=&0mMJz&P}y5L#|OIq)BP}R!Jds{&BmX3*Z(t(}*r^Bmm z>0`ZYc%$@2zdWY+@;EAwhv>Kk>b;hIc}^(L(Mfsk8F|p9j z;26%}6eri8bN$ZoS*?**TXL{|unUXF`ZCbZWMpW&aN>8jqoR1z|7IGyGPE{kHid*| zJm#W}VDIR!I_EY*`b#;{qU1dp4p({+oMN0A(Bl=rgu{Rt&1_m)xw$~=xIp2QxYAL3 zjueKFW+E=p0Y087I|)5R1J%06ZP2ov=q>8Mw+NI^{q1t5#*|wKsM9sJjYgwj-pDa^ z4n*DLG}BYcU;2p(AN52s0PaP33_7Qr>GF9O^}o|zlj5Sj=3dx|YC;ufy^LFK$(@f& z9W--39=la}^tqghpkF+dL!;yFx!isUs`}qaPyg&BbUZ3CXPqQUCd}LRtmd=V1b1i# zG>M&Z8u-DSPD+aJVZ(s%hp8|SN6*a9Z;g%>eaF?@4M{1n|GIH5pF#c5Aobof_tuDA zbC;=!Zx5aQbqy{aP`O9xCr3t_qxLG<)+$oFieu;jPCF!A&en|Ba=uem$s8um{q&7H z&(1^`&5!M*xX+?SZPFB4jofl(ak|Y5Q_%>fZ}`4W4QcgdUxuw@H-q|uxjhn$+Ag#;}TqS!>eevlJ&l%b#!dR z1(|(-EGaS_8Cn`TQ<}X59wP?foZUKsy%(Y{AvbTQFYw~V2>pD2DNn;ue)to z3kXS!#u;dLwK#C!OQDhF3~Hi#X1%;KD?(gqsE5C6|I53%YN4ejkGpRI47()Uly`Wr zImz+#y-%+-Uf?;N{VsfeHXePKz+lRK7k(;yAAZKO-%$fN4CmQn!IK$V2V^X~%Po0l wC+kxDG09EloR*31yA7f$QoRo$h|BaA;+TGZYoLFU>i3`jKNBL|?kQ^k07v*_fdBvi literal 0 HcmV?d00001 diff --git a/testing/client/client.go b/testing/client/client.go new file mode 100644 index 0000000..b6884a7 --- /dev/null +++ b/testing/client/client.go @@ -0,0 +1,32 @@ +package main + +import ( + "log" + "time" + + "github.com/davecgh/go-spew/spew" + "github.com/foomo/contentserver/client" +) + +func main() { + serverAdr := "http://127.0.0.1:9191/contentserver" + c, errClient := client.NewHTTPClient(serverAdr) + if errClient != nil { + log.Fatal(errClient) + } + + for i := 1; i <= 50; i++ { + go func() { + log.Println("start update") + resp, errUpdate := c.Update() + if errUpdate != nil { + spew.Dump(resp) + log.Fatal(errUpdate) + } + log.Println(i, "update done", resp) + }() + time.Sleep(5 * time.Second) + } + + log.Println("done") +} diff --git a/testing/server/server.go b/testing/server/server.go new file mode 100644 index 0000000..588c158 --- /dev/null +++ b/testing/server/server.go @@ -0,0 +1,35 @@ +package main + +import ( + "flag" + "log" + "net/http" +) + +type testServer struct { + file string +} + +func main() { + + var ( + flagJSONFile = flag.String("json-file", "", "provide a json source file") + flagAddress = flag.String("addr", ":1234", "set the webserver address") + ) + flag.Parse() + + if *flagJSONFile == "" { + log.Fatal("js source file must be provided") + } + + ts := &testServer{ + file: *flagJSONFile, + } + + log.Println("start test server at", *flagAddress, "serving file:", ts.file) + log.Fatal(http.ListenAndServe(*flagAddress, ts)) +} + +func (ts *testServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { + http.ServeFile(w, r, ts.file) +} From 28292faea93888d18417dcf42eb2a4824bb75505 Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Wed, 22 May 2019 09:52:58 +0200 Subject: [PATCH 40/79] commented out unused RepoNode constructor, updated pprof --- content/reponode.go | 14 +++++++------- contentserver.go | 9 ++++++++- contentserver.graffle | Bin 7838 -> 8683 bytes 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/content/reponode.go b/content/reponode.go index faf9e79..39f45fd 100644 --- a/content/reponode.go +++ b/content/reponode.go @@ -22,13 +22,13 @@ type RepoNode struct { // published from - to is going to be an array of fromTos } -// NewRepoNode constructor -func NewRepoNode() *RepoNode { - return &RepoNode{ - Data: make(map[string]interface{}), - Nodes: make(map[string]*RepoNode), - } -} +// // NewRepoNode constructor +// func NewRepoNode() *RepoNode { +// return &RepoNode{ +// Data: make(map[string]interface{}, 0), // set initial size to zero explicitely? +// Nodes: make(map[string]*RepoNode, 0), +// } +// } // WireParents helper method to reference from child to parent in a tree // recursively diff --git a/contentserver.go b/contentserver.go index e12249e..6aff610 100644 --- a/contentserver.go +++ b/contentserver.go @@ -11,6 +11,9 @@ import ( "github.com/foomo/contentserver/metrics" "github.com/foomo/contentserver/status" + "net/http" + _ "net/http/pprof" + "github.com/foomo/contentserver/log" "github.com/foomo/contentserver/server" ) @@ -66,6 +69,10 @@ func exitUsage(code int) { func main() { flag.Parse() + go func() { + fmt.Println(http.ListenAndServe("localhost:6060", nil)) + }() + if *flagShowVersionFlag { fmt.Printf("%v\n", uniqushPushVersion) return @@ -76,7 +83,7 @@ func main() { go func() { for { select { - case <-time.After(time.Duration(*flagFreeOSMem) * time.Second): + case <-time.After(time.Duration(*flagFreeOSMem) * time.Minute): log.Notice("FreeOSMemory") debug.FreeOSMemory() } diff --git a/contentserver.graffle b/contentserver.graffle index 75addb694fe252646213483693202f7f577e7fac..ccf971fd9bb6820fe0f8b8aaf2472d6e051455fc 100644 GIT binary patch literal 8683 zcmVMj9?(!G24iDVR16ciBSjX1$1 zpbVCvs5|z5znOqaSz6`0&)q%5(T;v)c-G35)5?`Y`TY7{yD)g#I_t!bqksH_zhFN- zIgu6He&qh+CuPF^(*Ehce*5v)|3eR{i9TYUR)L?SPa}ow^VHK%ztn1AwF;bCjh@g? zqdq^LJV70`8YBMn^plsStH0H1S(d#R2=!vcAp$0~(K=o^>vZ3T%6@?YFLrAG1d}PW zzcU9uwS6o7?Z+Q}T{`>UP;$Oyr0~63z<#Zv=kT?$UK{A*hhLL)4ekF8CifDDk?*eK z&FW{e#jrWkl*YE8@vgs$Y1x~9f1*=_*f0s#OW3?%rkCLWNS z#o!Y;fDhnjtf(~K6~{8hgAbpPsR6*`mq-rVL|Z z=o!4ykzHhhk^?%mvW#R|NchOF^&lav|m&5&IF%+Cs$BW79Xm~r@K zhOYAblNr2U&W?5_=o!V^ks9O}4C$aaI}8FFU3T(xmu6Q@4Rl#^kNHzfP4z5@Awiz* z(&p5Kfl{FJbO%--k0tb;M(^DLlAGc0pXnDEsH2hR7d|oTWYY?R2+0{;$?b826;CtG z;ly>voLV-E|{jc(9?n6 z*j?1qIG4loHYLc)fEY}cgy>`Y{(|{;>iy$~h4tefZ|J;zDiSQlyp@;(jSUR`s$kK^ ztLd37h@=-dQnY-hm-IteUui>umkXn+Gq;l(BRwMz{*xx1ruE%$q7(fIzZr~4W~R+NTians zj%V}wTbRUdYfpLCs*|2hC~kjD9+ag`T_lz3%jIt9{7he7e;;+e|G@TGZHs+Y$lK>E zRQEHjxtNVf&Ct}kp*5qt9f$JD=7O#1lPmM09U{38ZIRlIp{;=xsvBgNiux}RY-W4d zkBH{dn96zmnMA`iw8mF`;6^==1hTPwJXwtzfMweo*i}a?Y=5uN8Ia_l4}fSz?op1d zv{`aMsX9>dK48t81Rm(qbD03?vwqYYDDIpi>*^;5lPRAXy|WF%uv!NhH_J zm;j6ua^NxzFlmso#14SUlbXc3K(k2GWpOagNH^p9V8)Z7%XxWv#*$1Vt_duL+)G^< z*h{h@byr}{lg}~sz}vvrseZ;cz>=Ym+&B{y5Mb0S6LuhqDG&69VC7P=+e^S& zr!uME2dP3eKz|Lk3iRNEexBY=r+I0Rf`dy-nK%M?OwU|#59$rtcZV8i@$}jqwm{pW zv25hR%;x>*S&}%|78o8#DM=^H7|1Hg4H!$3pGiJrLU$aKeVy5X$%Gs-uofoINRE^+ zwhNRsIf>b^r0S$*v390$H$bdz;HE?QVhwnC}d4mE8-_6`T#pt}| zdK-%8`M~wxsD6XT+(kr<5&_9EU>Q)8SeRtNlA11sMHVh8GZqe66j1Y|$4GHREwG*n zR&P|y^g{5qL2^{0^y*-Jpt7XTyGfJ6nZ6-y0M&H+3E1+~Gt+;TvN27?11Z~CbXUp8 z&4s&<=m8iI(&0c$m?*e7pw(Ek0WqYFrI=-Phn~AbCTncy<#IR!%^H%USIIDSU&r*O zF`}e)Kx52k>ULUmQy(Q+ZA3q35+#F>VFr>elh2HhNJwrY(^F=o$jlD@r?h2LhLvZ; z@?P%7jBAYrPhgo)8hd1K!6bumV>ig_o!Jj2(hho-#$d7{M=bNKD5NJvEVEP$Z`@~l zm~!wY87uRuNU9n;byXx6?z9f;01dXckX=hHkQ-1Hy^V# zOXqf8-fuZh(S6U~a-yO?drOy7l$o&$WzMu_bvb4@KQV<)c@ZYP;YZn#Sq zTq8CUZ)`<=LT_cYm2ILa1;dK|s& z=T;A3@o4YudObcyayYf*dJ}hL>_fX}gE!#EcrSF{!u@*G%lNhEr>kBQTVvyl?bFM| z^!I)~u83AjQ#je5>gn6Q?)5!=Goe~ke{F4E{r0kt$J^z&*6r6CnK&VnK~Kt%9C2XA z10~+M3$8Vo^ZVXHSPWLl9;XN50Mifi!>}=^F<9>)2cpox2wIus{Z_$n?=YprIYx+w z*3Z0T zUt4q*jX0&fq7RGFnD6Kl!W%i@nWvjBk{i8&=LX&OMg&uf7d2m^L@;BhP#W4~PhsCA zV@JL&`AmugDpmS*>9ES%^;{VN8HkLGjeGK-37fO3%Ci+)U+TUb+T0svX7WMpI@lbH zp(7t-zMT7A(H6Q&uoxSG&}QQLIKq27)M0S3A1=54gjx-(<-RqM1aZ71CKG!%%S;N*kQv6qS@iyc;SDc06bMU1cXurugtJk3(uRMGD3vRZP_$ ztKzC9Xf!^v)VDzE;5MT+g=vFzw^|q1_xRG#v>=~9Y}mL9FO5~O#x+be19clrY1Q(A zgEk%8w!_wiskd@Ww$+(tow=?)=W&k?R97XaH+{78CIsi}bJZVfJxkw`3yBy`_2+ac zabsmR41$rSjc1M>=JlW% z%@wv2PaEVs6rQuFy_h%TT2m*xZqFyFm|eRTk&m|nzBhKaV_IztTOJ&6Tz<0f6o-9l z=tD2+ad^Xe^_tH-$$Kk*&=LlXFs}!H-s;WxbZH~nDb0c3bpd)FMGHuL!9gjMF62m5wT`CLsV5?5jsf-6xjSy`Mf2Q@BxQGeu zkk@!~h=f-64o;l5SM^rS6*p9;p-jBPP?$8*?LvPZt&v=_J6hI; z8q;*^VX%QQbt{So8^7IB%oR3VnXS!qZ4I}tue_Nu4q?6ei#!fVhaK?RK4jX64%*ZH zu2$;}+Pmnms!3R9(8rb0)aitE;<@JZp2?{3Hd^NSd$#5+wYH9)>+?2Q+v~M))Y)vC z{aSjcne$p6bNpI6E!X60ImybfoXTQOxkA)&49CDaIBvH)cB}5xJ5HxLCmaIDTkw_j z>R_hX!JT)jPPOBz*2Ut|+Q}EUXd^wPbWDf9@Xjw-^A`et!W)0Xu)j4MPtO?qRM)Cb zO5*jU{K)eIf3S#FmqM%`IFtSAL#oCmjW4m>+E{sXY;8g(N-x*p{3vx?XZ;)A z`n7hvKFV!;+e!MdwRG&ubj0C@RC2WHC%zecFiU5~mv}xjT!)G`FwONxYd#p$_oL-4 z?H`R97;faG--Az}8Dr0ZLhly|YbQu-`}1-!^Q-!9{;i4st^Rtwz6(XpcUJfy-a4rM zx<+IN&fOUkVDJ0vd6BnBp&PILLmU|awA+vJiu^1gE4ER)zmgCM(u3U< z0$sx~XG#V$z8iyvL`wXa@qjl}uqJM5FlKfscyA9CeCC;5H30BE#WW`qLU@~LA_jPk z0pmA_VRu1Bai5#tcjoE;q=zONW_`0e;GJXl1H1Ew{CNuUApC^;Q zzmmv5TR8E8dKK1b)?2TQI^O#E>9s@r>x)ED>ysU{SK+`$4*StxuKH!)_5P#vKZ#fW z3H9f(6KRzl!yZP#{%ePE%^6?Wu>n@R@sdgX1Ky7n>x# zB7iqSt|eIRyp28BSCbdYh_*&@ zB_kj7@pM($jbbUQH{AzL6@4y@GJNw4JI*SIcaQ=@uZ^r~{}0#O(+@vXLb!7R={jCI z|B(dVT_fR%vr9kAEP4-uIC4iu>T$t-<$=iZ&Ou={Wy^^!ob0;}?jWxFgDVc9Qnd98mvG%DT5+=fGT7C922)D*V&g4AdDHH z4QQBPICBB%t$Iwb(m81E(H58{h91LQq=kO>KCo3#CXa5IxD5Z98BZzfSHUtH=Thw5WrAw%o(G|9~X2++dwR+<`_aFZ_chgF*y4p3* z1M;_ZI>)~mFwV;Vu~se)!~AbnY{dq8)q2H?n`C3c`v!*n_|I?iI7(A9_%_GB%_ARv zTb=;YVQV9P0x3yyv&E_|E?Kly-FoTT)zw;MZ@fgb= zV!7b|{pRde2AodWCl!DHW+g6+JHA;6@@-|T?QiL>*}~wzI8NkWtv321h(`X^#?V*2 zod>)}RI!gq7}(fNN5Fb(t=2Pi54Y{&@7vFO5#GXcCU zqolLR@yV93c$LQzmqA1cnW3COlUb}enM_qo*MwxI7JITT%EjEPN5O!eBj{%CQx8@EPIJr{J!zK8y|S5t;l^k%<`F z&YwFdX%a9fAzt%klX|=H^HZH@y)=KXKm>aPB9B1i5s3VbfykfQ31f4KLDdw;n1zmR*UV3wgnej3M{wdFi%j+r>?t#AFwJ$;9GX>`t?b_Z=^=T0Ac zzpjAezvH?>^DzKC27t!^@E8F8N&~=9oG{|#1#N8qrF?z|M)%D|^AC6Wjfc~JIQ@sy ze>nZWkkg+~BN*{pODBCo;l8u}%X$7*}L14%PkfBr838e}wh) z8x7}^td~bvk0&@QCs{9#uvQONmM2*+kFXxgrE-$>@(AnkG->4|>*W#Fqb-$_td~bv zPbXW?Cs{9#u$GYON!H6FtjC0^KfZ8rg!OnPwQ`d6@(61Qsh(uLJi=N+swY`5kFXwX zshnhe9AW)49AUMJ&8R;lI;s3(^YQ0FeY^UL>K}hm7$H9VSn0KqwEPkLMZ88Lm!rpr zu*kR;$Y3w{weZDQTb{qYh)K@vR*%TUn2OkdrWGLZGzk>0x8LbPcNSRrkJB{u0x57E z-w4%(-4$m7>RW`jLXJ+QWX!CCF#ufIGvc1rrO3goJC}k? zc~)H%1{BB;GunV^?8Xc9o!@T;5-5M|1?Y$@&YZ9SQ(4c_QR-twq%~wk&h)1W1ybe# z?E*!jQ{JHE#8RP0>W8vtbL@zbc8FqeP$2JNDJ@g4aFyQTDLo=@vy~p1N^c-<@cMm< zJTaD*$Q!(+RnF3r&M%ou3nbz%{ffJ^L|$Pqy<{&fQ@@kL^gb5TvejMilwRX8Es$Ta znIb0B0(lRgX@RV=n%-hGEs&q_njSe#%a(ixv+2LxmM$1htL&x)@*a-Uv(B$rP7CBU zuG34N)3Pny$#!}l-)VuoWIg@=Ri0llpBBh_xJpYqKVd*U>MfCXaG)MN?M-IYYk9xI zg<9o7{jrRTzh*-%+x)c_e8GoWAg{5aUNWK{`_X;8sP{3W7RXC})N96|7S|Aa7>Q~IE zrOqq-sh8ZTKbF7HcOQf5eH^L<@{&pQ8jEUy{EABz@u(Kad)QPXu~S|IP?T`k*Zg?;rF^J;meVPGv=a>>K`83${b zqZ>@DRTkE=h1|i#S|AZ4>sM^7<^1RhC+j62Yv~#8WM#dNm$g7%va?=eW-VieFZfvz zH*4wZ?qFywkX4@6TO6&WpZ|oZ^~lm%An)O7EhE5^vGp^y)-w9N!P#2nYb}uXu(p6vZ}Gks$j{hckIb(H@*e)z zvMrS?u%9u&7RXyXuvHG&(u3W>1Y3?W+F|w;7i>90xxxs0$p%{>@8yKOj}^8+UUI`; zF$;AxMEnYoE%pL4L;)W-Rak_i&ecTvZkEr_k#0pW5@n-G2MaD zBoOIs$rI_3EbafLT=ihuMldd=KwW_|8-9mh}3j$M(H|gj5ARzL? z3T5vnd+rsL6JzZ<>9wlHuMrwr{@RKMenh_`YddbXv_I^&oLTA>mxdcN&{!@yfqj=CZ$B zV`=@I&2P9#1DMN;6EG*z*1(9CFK+6Ds~~R*mt=cYaYgP|8`IG(>0xJUU~ajFB}w@T znIMI4-b|UySjS1H{_DK$8=Zw8C=CM4X8E`^CutDNB8|i8t_%jp1vR=@`gcJOxz={UBheWG!MN%+Hj!g_r-GJlvIABP$dS__DKHtX&asDcogL>dtw6QsMWx`Pd z?K5!hNoHXZ*v2IAjiK3lv#w_SWhX^)W*Bs>kPTc9P^iBCdd}W-s{>dM7!jabV^5= zX?O-lXKYriM+!e4M;KHltg7=IyO~j>I?iRmeQ|ObOmO0scuP4vIpd6_HpYQkNC74&hAU~gw-*-S!hxytjd~BqOa2(;_7hf0z zHL+=fa*sn}o*E6`SWZ!ICmp#Q)U)g^d*zF>j((>b@j>s&^;ek3PZ z)Ob$@y@j4FPi>qL=)nSD;z3~g6PuP7ZYt6`E>bupE%dxR`wBzI6A72;B|exaJ2^i@ z1J%06ZBSHB^fvFm(m4WqzrK(9TQyQ+&=x}8>1x|rt(GzG)b&|O=VcwT#5zkhl*>eOmj2sjOeiGBkarQlY zv=Dwb8EsCeJ@M0Pqhmp!dl9?kQcC2%ZJg>E)K3jkZ-u$nMr_qjd^2#=*tdJm{xS!Z z11e_@@iVP>o~6)Or+ImWFj(L=zU?FhA^0V~xXz3}5R4;X6s;MN<$R~Ck~mD1`s;Vq zd9KVHqxq2?XVt*;6qXieG&k!tv^uYrGv)eqZsuHoT~`C|^Fd@B@qOoiTaLGyou8jz z6!-oNa#{WFjj06Kr}y$EaMm{gu!KzGtJj)uS7OX_^*0(HGgONF_rx&oDuA-R$c|f$ z3M)UJ^-2)$NY8VI>PEZuQvKVUS0XQ}PN#K2sE#+A?>W<(SBmjmtN+bM$;v^;#NOFo znVuXEdBx#N97aB3PhY(Z;Z@16{^B1saQUm)T)?|`NuFmrdFqJBTQ!1BrHfS)tWE$w z>)EFWyxDyB6NlLQ$p6DRv8p-eo_Ok<`LreuS8FFphG-(RtaR&(-eJY7J)D#*9h(o4 z%lHHdlTV#7v^{V}G+PB8TVRfJcB>eA1Bk9du1?P@cxhti!^UekLA_=-@xg8{4THSQCi|6hO$7TO;zw4nu^rN|mZ%X% z$3Hi?S(c@M7)SZI1??^t8*bN9U?eG%7tt-VUO$)x9Wb29<{sli`B!I$#lnqa*uyB; z-!=g@yExdO86zxCaxnT>r;{{T57 JXNe}C007y~HiiHI literal 7838 zcmV;P9%11hiwFP!000030PTHild8y;_UGS#0=;g_dhygXn3tI@yiV zYjc09(6*5qM^JeY9-?0WfU(T%)@IbD7ebFa%O-)>~+XBD^0|0a|7U5q_ zIFxc0gU_T8o`4@&QE9#}$P&hbcb}=L0l=W2r5v`$d4+-+zvENedHs>}Cb_rEUn)92 zEI9R`7x0%}+C^rioZXZPgjphtqDOgk>>}G0EA$pxX^n?WQWm(kDCN*Ozo=x6n4r?J z;PB0&{DtaI7VtP&9PP}?a};mKYKSmcse|I|C=BiLXAhMdn0+-j%AXDQSU<(oRL_DK z5}a0h?HKHM%JwCAa>RG$s%!?eDe%yeQj<2PfOWER@Ov$Y*o z%JD4n-^wI*TYJvCR+IE}LUBhjc~AnIx+)6kLvnX?exwii@4e2q9~h69u{dG}C_WdF zx?gDR)j}pULsOfE){apeN7CBn!ma6(Yx61&OSyNkNbO{3YiLF4rnE~f`!5l07JJx_ zh<0GiCFDPoXt{>g`m7J!@dHVeHrA(S>uC$HYXe1?_X3jvNsdPV2v+PK<=Bgu zCC8Mi0|oT~YuP67P@kXM1c)E?qu~^N#lm{Aui^MW;W+`fV*OICZWsknE|>yR1e^#Q zK#jo=hB}~mAQa%GL`fn!1WXsnsW6BE+X8w)CV<0$<&sU%FJRiGIKXExl~O7gBp@!R z01N|=x)kyQ7+GKoXcCNdurFv42nrwy+65B@bPAZ%raE{o7!rsUNsG)3NC8>O^$R8h zaz>6_rUhm#QWDuQPJak%6R=+?cUM>6zpUCUWJ39O=vs%x2_-fnJzBmvW?tv3;PV z0KdrA^a-v$nnBilhg1x|zP@%^^AY*8TN`-Quw7@gN#Z$t4s zAG-c4HEQvgyNao4HYnv75QNk$8O(|>pyt6~RYU=0CWAu}htx6~GGY=_D{QEO^(&Py z!w9@?N;xW1hE0$js3IHjZq}x7W@LyPK(*ab2DUu)%#5DJLZ+!?EEYS9?ki~AT)F$0 z9)mF<9uBmK2?G}gw3-MuASSdC2u0C!=%qVhiq?h>g2@838>Jk*&L+A0BGa4JloGok zjWN@?+w0J6eVP@GDg9iClmsG%8H>6^J~M+%Eaf&5J!hth#O&aIN>?%^=sY8l_EJA# zTuUB!1D1(I*&~N5CL7DG-5B|u*^g)94rZ3tc(x{|Ec2`=q$kEK6DWo!kJurm9K2b< zO1vtNs>aS;wUmo?T90*r2IDPagFrJ$W6GwyhI?j*-C=W4vO~5N%u{bEvkgn>`OXtwmEx7|u@&MOrtxb<5aEX~ykDvy_|c@)g%g%*@*w zeZX5oZzuQrnU(Agr*Vx&lS{f6S+U(mzh7<4^%$k{d7OF`65|OC?Vq8;;Qhc z#enPvGg;~mhFL)P3W~GA%nc1nn-452?Cte<05iO|H=2Wu8*`LB7&Mdkb-%QR086HO zZ#NwBNhya@0XLkvYhxeTLmRvTKf#BQ`x@=j>9F8a!Oz#jHkM*?!H(!4GyT1f#udTJ zX$mJtb3K3EH@%UkZ)Q}78l~3e#qS0qJlO_vqd#i23SmZORj=$4atuS6pYj zB%V@y9R50loo!C=FK6bgeDM$pO(^;-qQy~CUmmKY%%7JK2fMRaCX zAP}G#)64mU6k7`Ik0)cdP0m7UGViuaxh);HCo8>U)35u{o8ymaUyo^nce zO&?ZMneXW{!kaqanWx)sDK~uu&n>#^O$nxvtQx*Z31GockvOr*p~Ak3#*Tam_(F^a zRIZGg;$e;AdZ|o-1O!IHl2U~CBW1*M+!4-8EU(fz}QP5SVsu?|yQ!&Ea=x8a;tEiXK1bJ?~XwmF!4 zYsX|ey?N1F>gqF^_vk=%Rf2ler#o*(aK64&eOVh?`j%XY#AL2N=Yhz{%3>0RQ%#c> zjvXOCXokGlc=1$oW{c-AnG(KcOm=ILhzw)NzRKL|(%9PRjM&6RqnD|~ev9TWqL=F& z(Kfxo#voo8bJyNFTtl*`yzpY$wPx#GIPVu$knh*=n6z*l*}3V9uv3%Rp|2#iuHr3d znZ3@5=f17kFG7o%dmCrGYY*dS;5c?iieJ(E%qgT7y}6~96x(&@2XCpcy=2}Zm(k$4 zh`Xz0OKP-rvhNOkl1kXrwForcj`^YNZe?0+O*$SNa9n=2@)U=CZRry)9&&igdhy!N zJjr`&f7}_2TM_buzw8Vbd^+?G)F@20-4!<(?-OjdTKGbv)s9wKXDTGmP5=ziT+>=G zdq#2gI^d-sewABoK?q(9g&)Yna4M_w_Iv=NPm9~uFl=h`*1#J_BY!a%L@3Wk%e`R? zQZ>ScX4)>Sn2)Sp8xLZ8=!k>PCf*TCzuo90T;vi%wx5^=$`gavp_UZJ%6^?jYt{&M z9i_E4qKJDnM(ZsRo3Bgvl^rI#VbFXvqSyZLb&BjPhN*3A*D1Ttup!Q*roR~(9WUJ( zTmJ=stgjWFlyx)P*vl^?Dm(bde`zX)`CyMTFWNA7Il?i`A}9Q2Yve690@s$)nA)s1 zo!rben~u}v%{|KVTS@D!URuL#B0o1?dYx@!4y-Yq721+)bqC8L-_qK6s_fW4bI8=u z&RU^)fs~o_4(s=~UAMO%9R`yuYVTL8VcyM%{fjvktUZ31&4t`*5r@>A3f<=xS}&ku zag#G$%zTljy%C9Fv)L|}2KacZ&hoj0hjWb(Yzu#;^@XrX2a~=acbE)jt$e%EpQmXl*X~b)#zbSPAJz>K>y1aa zGMziUs7X9GoZ&MWw_c|Ks^7DEzo_Uum6Y4)66dr3G1j(6ZC>&3xLvxhrxR-GEhRjsSV zrPM)-TeOj%N;;N9V0iCWtowm68cH%AClMzq1}Fr9Qjc}R$`ZNf1@B0tH!Z2THJIqJ=y79;zIkBdRqZxr%J2PeLky$b8Ko1K?d6YqR^ zdg&2=eo|-|eUek|RX8xnVL$rIxnJ^K@1N@bS+f3Ts6UULrB=x?>`5H%KTjC9lJSil z8(`)B))^UlC*@p?t%wSZ z?3SK-LrwNz7tFp=M!Yq$8}_}^$J3X}?$oySTD}jROZ2&p^{mV@?4-Cryn$30IyH(* z`+vF}CO`djF*7?Ckf%xD{8I{e^SczDIlKI$%9htbn8fbX$UQFHuRRc3-g&NfS+eEC zSGn^|ftNq4xXPSGl0*-NVdZSvhf|1h`j=HB{rlPRAI)~N47Bdc+rYX|a#s}l@`T-$ zdXw=litEC-?zQlbgMvd^V*wZ?H`uh|5v3p;wx|?O?Zu*7zIn}-oqj!nnA~QgLeMPCE z+d-M3+xqga!|-hvyo6kA-oF7| zp(>d97E*f-`-)w-?u>T73!UCX<2SKacbsq9H%ndfs#&kH*WKK7KRI0~xRYRSwe0`6 zly_P7FXg{`=#M=3k1QPAYQ8rkR^!%?@o~qwMx2wuw=`ek-wc@H(I0E=;zXPHW+hf) zl$Sa$cy*9%OnBVFu%G|U7I5f@q?aun+VeEAxNu*AZ-jmpmh1w!4ahM0wJYdKZW;mo74!uy;|joyqqjM z@zSvJB*lLI8PzUp5>MZ(CH6$YD#OwCNlrGYs0qd^#{klJat?Dq5+`&V*vOu0Q?vLehdJAr2$|f%ot(z zbs$fKzf{ie!P$Mg)&939{nn49|0C)DNcumL{$D8R&!{P!@mqnDKb6ycC;iK1{!Z)V z>%TwuH-F^*AG!ZW?*Ebd|3bN6bIiX)(ytcti5I-i^W|!Ot4sWgGCT1jv;WBKKQjBj zFtbZDmcDBE<1>uqTOuDkXi`HwaxhUa7K5<09;?_wlV;^_GqS)D!i)SY6sCH#4)vx8 zyv2~I&&_H`EkTC7NDd2pWfEr1fUc=@I;EM8ef8qFJM*_rbk=mQes6td_j0;vlD|m2lPe! z{okK_lozn-G=@%McjqJDtjtY~{WlByzce)=o~&v!>fbUuxp)Zd_;7-6Up}Py{vo9j z@$ONHTO(=xTuE*tGvfETco(2S*ns=$65*wh^MUy6|xcDYI0Vkw%f^*&I7%#9jH;_P9bmLVQq{ zenEe5kNV(XrOaMJlH8A63DDG@%SL#UBAND;0H z5$e((q(yj)7@>~3E9!$=)Cd*wGlGPY9HBx!LXl7*FNqTFkt9^ekEjxkGzoPiKR}l7 z5699KX~HF8LWO*UHsP%EGvb5_d5b>bnmVD5r3VQV9-~mGkk>>C|6A?(8JR+be1!g> zw(|p0g`?gY`2elLG1J~9Yq(YS8}tg7)Cxb>^Wx757V4P46~R{&3l;Jf(ZV&!!m%Gc zMz!!5*+PZ9rd+s1w@@KJBVD)`OSfqkD&!^k!ad@JTIUD!3rFgO+V2k#Fw{D08itQ3 z7%JpFB8E#6hFa$XR16ieBxLxEjG@+fgOcHzj-f(6O3LsUEklL8CTF-s%updeqh~0o z87kx>1P!&$3z~*|6b%*fBcg^QNkgsk0jh>tXHD4f5m`fpyhqt^N!L)v`~##76|$sm z_>8upLf#^8xF&9>bv{Vn@ECzZg}kP5xJBVmxBMqW4kd|0g?xm{p^ncBLWg@~4i)kv zN{1tzLxp^V)S-^#n%dzbT8FwucgP(si5=<)d4S%bLY5>CpAkIN_eVEq9df#U z(Zgd@4;Av7@ZlEOLp^8sgz}-Jd#H2W1EdcX@{;=D9_>S&=RY8SI1)cp$VccO>KR~7 z0`U<6L_PbxLj!S10Z}0zA%dvyVeQ@IGb)IBJi0*$aZLtM&sZL$gm{b;qC#F%L)@Z; zsPADvA%`f5A?kVZ1N0DeD_xL8+#`sn@3}spi8xY3RLDn&A}ZvOF5)Aqh&s>TA&j^r zi>Q0Y1C$YUk84GX&qyOGI)88x6amBD+IXSKh8hp&- zJBY5fP^wkEd1+03r#}qs)78WeR`VU0&O%|dn1|tlsa83EK5gn?GPY6sgRR+=%eyij zkPP5lumThY#xAJi6VToA&PfC;R&JCOfLBV028Iks3MvG{<9-voH^RWE1qwh@XY0I%oXw2}ylh#R!p6-3JczA~B%#LWo_$qYeG-y&-jR*SNiV z4=cQazyJN`_uCoowpFeY~W_*P}?B)?IcxMZvuL5YzQySaCxAfkRU zk?3_XqDc^~cLgII-)U@KBy0a+nd|;?i}>N=cZnyPJcPBpdIILe0}&c=@YT(oXdR+Z zxPI05Qe2Tnw}_?nsJAsRw_d{%MYKXT5TjRbq0APn%IOm|1CyV()cr5tX~-lz1vfS!j3xM`vtNtw#<{#II$Wu&T~;>}Elcs*Gg1r*g$; z=b2HqSu~~4oL9uzunr-h5s)mOGhnK5s{9nEWlk3D^`HujkA$&2+V4Rvh#W9R&nKYc z9;keX);8g3BUc6&M6-Nl5Y)uxEy_I(ji@vlUa_3YZzml{4(bW|!LYt?)-fuyesQgX z>akw;f)Mv?%huDE?{#$F=&0mMJz&P}y5L#|OIq)BP}R!Jds{&BmX3*Z(t(}*r^Bmm z>0`ZYc%$@2zdWY+@;EAwhv>Kk>b;hIc}^(L(Mfsk8F|p9j z;26%}6eri8bN$ZoS*?**TXL{|unUXF`ZCbZWMpW&aN>8jqoR1z|7IGyGPE{kHid*| zJm#W}VDIR!I_EY*`b#;{qU1dp4p({+oMN0A(Bl=rgu{Rt&1_m)xw$~=xIp2QxYAL3 zjueKFW+E=p0Y087I|)5R1J%06ZP2ov=q>8Mw+NI^{q1t5#*|wKsM9sJjYgwj-pDa^ z4n*DLG}BYcU;2p(AN52s0PaP33_7Qr>GF9O^}o|zlj5Sj=3dx|YC;ufy^LFK$(@f& z9W--39=la}^tqghpkF+dL!;yFx!isUs`}qaPyg&BbUZ3CXPqQUCd}LRtmd=V1b1i# zG>M&Z8u-DSPD+aJVZ(s%hp8|SN6*a9Z;g%>eaF?@4M{1n|GIH5pF#c5Aobof_tuDA zbC;=!Zx5aQbqy{aP`O9xCr3t_qxLG<)+$oFieu;jPCF!A&en|Ba=uem$s8um{q&7H z&(1^`&5!M*xX+?SZPFB4jofl(ak|Y5Q_%>fZ}`4W4QcgdUxuw@H-q|uxjhn$+Ag#;}TqS!>eevlJ&l%b#!dR z1(|(-EGaS_8Cn`TQ<}X59wP?foZUKsy%(Y{AvbTQFYw~V2>pD2DNn;ue)to z3kXS!#u;dLwK#C!OQDhF3~Hi#X1%;KD?(gqsE5C6|I53%YN4ejkGpRI47()Uly`Wr zImz+#y-%+-Uf?;N{VsfeHXePKz+lRK7k(;yAAZKO-%$fN4CmQn!IK$V2V^X~%Po0l wC+kxDG09EloR*31yA7f$QoRo$h|BaA;+TGZYoLFU>i3`jKNBL|?kQ^k07v*_fdBvi From e2a51bb5a5996cd375db6aac513463e305990b3f Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Thu, 23 May 2019 09:16:04 +0200 Subject: [PATCH 41/79] updated testclient logging --- testing/client/client.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/client/client.go b/testing/client/client.go index b6884a7..eb31011 100644 --- a/testing/client/client.go +++ b/testing/client/client.go @@ -16,15 +16,15 @@ func main() { } for i := 1; i <= 50; i++ { - go func() { + go func(num int) { log.Println("start update") resp, errUpdate := c.Update() if errUpdate != nil { spew.Dump(resp) log.Fatal(errUpdate) } - log.Println(i, "update done", resp) - }() + log.Println(num, "update done", resp) + }(i) time.Sleep(5 * time.Second) } From 2faa088178658911021623963b006dc244fbbdd6 Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Thu, 23 May 2019 09:16:47 +0200 Subject: [PATCH 42/79] added more profiling targets to Makefile --- Makefile | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index bc665ea..6089af2 100644 --- a/Makefile +++ b/Makefile @@ -74,4 +74,14 @@ test-gctrace: GODEBUG=gctrace=1 go test ./... test-malloctrace: - GODEBUG=allocfreetrace=1 go test ./... \ No newline at end of file + GODEBUG=allocfreetrace=1 go test ./... + +trace: + curl http://localhost:6060/debug/pprof/trace?seconds=60 > cs-trace + go tool trace cs-trace + +pprof-heap-web: + go tool pprof -http=":8081" http://localhost:6060/debug/pprof/heap + +pprof-cpu-web: + go tool pprof -http=":8081" http://localhost:6060/debug/pprof/profile \ No newline at end of file From 2decb53ec16587d887730f23a6e1f78fe78552db Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Thu, 23 May 2019 09:17:09 +0200 Subject: [PATCH 43/79] added main datastructures to graffle --- contentserver.graffle | Bin 8683 -> 11228 bytes 1 file changed, 0 insertions(+), 0 deletions(-) diff --git a/contentserver.graffle b/contentserver.graffle index ccf971fd9bb6820fe0f8b8aaf2472d6e051455fc..6e61615cac53f55274301c97d9ddbc2e6885c3b1 100644 GIT binary patch literal 11228 zcmaLdQ*b2!!X@CSW7}3I>Dac_v2EMz*v^e@+ct05v2C0G?CjRo)b7lj_w#=0t3wn8 z4f4MN4sz*h-C7{ovU6Oczq|O!>|D(a_9yP@EC-`YNDLJ8_DI*+6J|^XRIFO{+ZSxe znYXs8s)v@P(mb~P4jSbhqn@bV80w7S`#pImyAXh7nELf8@c{Z-zh5kW6k7iMOw>&A zet$Ch{TWB(_e%7TrIi6!(ik;Za6!lFM5CBZFJpxY$PBO4Atu5A8b>zm>U7Ey*c~y`*u*@ zFQV&Sz{)M(IHgX%_s{2i<+@>O%CVa7EkHk#IqdEmyuo`ck@eyJqRu;Qke0rcJ#z)` zYu;MuDsPVIHsC3r&etY*Z-rotp&Rd19D*jQ5kJ-dZA9-D@>y-8S0`V$vdK=+m8#3O z1_Xp9h3oRdM3m`fgdKV)LQD@)YGIZo7k?MzMG&G7f(7S`tw<*3nDIm@!Vui2JVPF? ztB58XM6VmO1r%gKXUSs>2ZI$>=>xo04G3TuzDP1}?U~PfpUhZbF#AzErA6ORcDvD| z#KxEjDbA8~=XK4|U@ebd6{D4rha#n2*xxZ?y*6+b=^PYM!@7j;rLn9&{W1=_C2^ja zCAKLJV^=YX0I#%=$H@_4+}eZB;X{_WEK}KY@!EQ8*^&>17)&BG$l=80YyB7hB=mYJ z0zx_NhhfSfBiE9~Xt)L+msmMqg13@Kfu1K9!`XS`#Sz@gS^V6%@wKPC>H< zpZSYS7B=DDW%7f?%m33T-)~?YVRNOFa?sMIYA6TC#sMrFPdJP8YYVr_CO3O1GG==+*uq zy|zJ~8;%u*9Kg)fabc~Fe%MHuD-C~u_&FhAC;kcLC=WOGjrAt&lyMeSWTne+F1=o#<}OB0<{Q6ooZ5p%nv=L z$d<;V5e?4mFBV}3KtIDUwk=xiB6*r*HaFBy#mVC3uc+a$N1QhlanU7d%LjAgHhY|x z>(@RWFXDQwkrvN(?VSKq38<3_ILx(eWJvJDEIH8lti|5ldUtjiAoV7M);>5ev~PAr zhi@7tUf@_FN$fsVy7RI9hlH5i4*8|C8#tQ9Spa5$N}wS^ z@z?T!X!tm<__R^#y1s7oUum%9;#VzCuaP$f0Is5a2kcEj?gdjn!zx-BD<+6Z_202J9 zf)oD{DeuHGl8{3GQ$=>K5{HW|9ZiFNoD00U?qbQKD7ub!*2yWozM?9#Xt&PJ)Zt*w z#JwjlF?OWjj)szhQsS*dC=^kBUkW3uh;xt5pO3sTQpWUL5>qj$p_?KOS3wP!!He}F z?1n`?azlTE%bOVfg73lSM0XS1)&Elw+UGF$=LkBy0H@*~;vv7b(vr4-m-S>VbfcKj z0P2#|^tP(Ka4oUD7HQH6;E+tp$YMzvqf@Gqwn;(IX3j*K6xdT3ZX8XM6wv`z1x?8x zePr|r*X|@m%!M9S6t+7+_)`_`+1^@2T*;*JL(b?v--PH@C-&QvA^20o6;rd z@r+Zl>=+G`;UPNv1n6GFbgUs<{r+o`^%!Nmw8}u(T&AAF$iOk?h$_eflSy*n{(cdo!iJ}`NR;@%7K5H90bfi;6(=HzbU?Vep2hKW6w|ay zbTneroCU)$7zwDR&-?OIqP<{cX8zN2Ult_s7qH)w8KnUishHHR;lX{0%aMwHC5Tak zDxy~b_-dpC(~h3n2@`k_u2rwhO>=RB&wd_xkjxsAxR9cGHS=kJn-)gFmRHj5j2&8N z5otCmd3n1)s3#@UkS2FfF)~OW&Arf&)O!fo7~_y*Gkj^Ip7q;{avJ%*p?&IBGfs#S ziy~DNzu&@k^`$yAuisQm3J)TQ{=Bhiw7!2nf*k%q*{d-liMeKO*M|Qd#tBFH{%?5M zZLEv*RO?Y+@$>PkA!`=>q!0gCD0qi7Ru=o(9NRFC!n-^C}T7W^-~W0$*I1qLftCYN5wxmCdbt4pR^mwUGEsG#zC&%T-tN zdd1HqRC>?Jd#ENk3g|rsS;iP&DOtwqsO}`D7u8yU5k*FMUip)o>n~!E^a43|sEied zbHTmA&!|LP!^vXr3=Xg5@IhvGtN@%S8DXsl*LW*lS|Y{J1s5eLMgF;#st&`_F1N%~ zT9+RCyZ9}wBxBcnnPPrJR~2{BAo1ncp12sMGiGj*8iyqTTwbf3xiRMGwp9WAmFO_Q z5s=`c$dJ(I~k&xKzIJqxUsN!+MV1jTcI6UU32u?2@1;?@mSfw@h)K2BDJz6T7 znNAmRZtAr$&mGf~MRKRz<|#H;^svPCPSo?enq5SSEX^NK_HLwQd>Uy)L1$4S$x4MRE#@e$c*DM_#qER?g%pS-@s1I3avJ=N z)s*t#?-2>AOG}#!i3HYSrGcT>Rph%*OD^t1O5VM<&llu8as^*o(oY{Iy%q;ml;(#YiYll&?Z>`!nrwFwnB5D2_)k?i3^n7U7Hh4r~P}2u*s0V$y=f!X7$xSln88Sto)fnM`ZTOv91l zx%aV6!Eue@U!D644e$sJ}afQRQs#-^CwKFSB_*)&4v z&AXIO0M{e&v{{$~>+A&q^Y&`cAdUOc=&oX&%*bGtVVmt;yc0j<;9M~?rVFo@oI>}O z2|IT&KvzM0r159PFSRYsQ{&AXj?=`SJ^G^4FC9R$yyv^7_3h}ft<{V2(-#NwlJ2|t zMviXEJxw9n>p1!08AtIUTO z^2n~;a(YP4c`-x63gc7i@g{A}QvQ_QfKz)j+{qLdxt>-})nJI_>8suslFNxd9(oz3B z44%|5|9)qgcAQUCq7UB_fJ>hLC_aqUH;}0$$+1dyQhn!@qD;})FqN2|A_7C${Zy{_ zF6o74uI722701VUw_%c&*UQ3pa&rHGwFI1`puOdn_haH%ocafi*Mg&a zmvzJ9pndiuUl#`Dat|^+tzR=%S1a6}HNqm{ay=CKb3M}L6ZxoqGBj>36Mr|kEvaD4 zI2~L1e0>JosxkWQ4?%L8dOVaYtxkq4KFYhpecApHy247r!_}b*?Bqi~#Dk{3MTBIP zq27(seB&jXcPe<3q;<_S<#f4<1K5#S3R#!!m4A<)_OjjpF;D!1?Oy`N_l8t0De0}svq5)84#)52$K|Ly+p_gv5Wx&w6L zUquL|nf-@ys*G5tpN2<%ZvQm}io{*$?L?o29etrkoD68LG5!`~-^4i}FusD^oelTT^3IN4d{s*ek#8rTh@U^{ds~Xse;?3CKfshRC1V@7=JkcG zfZr;Y#vu@f1NwI_Pi`d3BjD`A0y*~jHiKjJMgO#l^yB%p!R)`Z<81|P5IhQidx5(C zXv__2st_TJ=h@^FdiRIs_D93@MSB|ldRT?PYPauATgF1U*nD~L3-A)!$tHeZn)<4lz_KPq;7YTSDbTbcNN!V7k_Dh%rLe^2(N7e~(h z*tnm68;C_B^yzhjUwi6r_H?ivu5ZE7W4+{Da@kl5TQ);idd)X`y3XxdK(qtldrr9A z=I0W#{gsQ~grq;Sox?o}Zo=vH^|Tn#AJ6!8@Bmc^I)o38Vx)8belS1~W{Sjg5T?LG z>z*mlTJly>rL4rTsUe?!IEEy^il$}|1f((#>KAb9Rffyz7<;U)idRSglq#}}Asu=i zyDru}P;;(3h@A_j5M(vnpDrs1xa`P;{CvwVt0iXpkiJ}d68-#o)L$o+`F=B=3;1}^ zS0~LNZQ4|<%*X0{#J4lzkd3-i-_l{6P>H8p4$F<~ee&`)?t^bYp z`?ze?x9;i+pRD9ve%ki)7J(Rh82NU@`Aq%T#9PQjXIW9iit0I~BmcFKi7b3dp&sDx zuVk_L(!yJAdVDh2kpx2difFe{efEZD}q3@voUXP zEnTDY`$g;iIagnfNrCXsC zAeH^`Toy(w?33G37>fFCnx%A3;^-lP%hyEyTlPq$weY0b*M%bJ>r%T^Ij(z*3aa)T zaC2tAO+KpBrK<+D1As{t+-C6ICS~zxsp%B>QeOw#jX0O|>IRY^JVVu@bueadjFM;P z@))kdL;<;Y4zf3B-EdeXc%ldIK9o!0t5CmgfdsVAS_hr!o(W$*ae@4YLNicWo{k&! z+^+7Wa|@f`FZt?rxJR!e<=@88{2tT_hm#3^!;BQHK93|X0!}<5U z+-#CFks_5R$(p0mEM*m4N>UH6pubqhC>YVc#z^_N-}c)YD;zk;@|GJ2V6?HQm{L z++GBej$H*f2>8-0iMianCHoYecN^tM*g&_V91^QS?WGl0)CiQRxU2ZYvsjURrxL_ z`wRF9vYGtS`0+9AmJ){q=%%aWXXyPBP|qZfe`66SynWVWyNOfXH~#yg!N%Vbbg6ao z5PsbWH=r(1{9c&+SqG2kc8?I3Nhi~w=N+x>@j5qY?_SH*LeTs9bmPkTwSF0Y%XY3k zW8cen#oGHf^8I`Fg8j$y`uTS&%nM-5|7Nbt6EhuKM*z{+yPv)-_e;E8@tX|mZ7jr( z0qe~y9pQRdq86S&<_>WS%Z&ZHn7p_7RoIgh68a;mK;xbr`>#EAD2F{+k>B|>ohGY4*#5cJk%zL;D zk>e>Dz3)&lCH?OHt^xsif3Lreu)yBDgw9*Ij6M%N8(zD>GbUmX2@GCrj{*$dE3^oa z;RzYM?yOhS?$?t()Bvy#Y_T`uMf}cKYK){bGhcDPq)>0L@1|Gr1QCE%D2wv z?At!-x3}rtNb0W|UA=F6)h3#rd^!5UuiqH_9gi&BxITdJhui+@xbO#*gdQ)^4qyM$ z3<9UobL<>l0ar9`|GOR55yG3>1#8wCxCXf_;T(HT#Roh)PJaHRJCsI#qow6IO=rt- z$;q$|(RT)nk`cx*Qir|H(egM2A)lf0IL^nv`-_QMeO(g~T@B8@$*=nK8VEPLgU?9n zL`XFL!Ig0ikADX-Hr_vFhRUrCVetxVEh=c0U!dQ``%m^y58@0SP@;Rh7`>iQ>g@63 zuKaXwpG5TXy}!>(YyCfKLp>fFjqG3L--DXh>1u!Ly}efoT6AM{#6vw)2bdnpBNOV2 z=C8-md>=BHNn&~YN(OmR3Ee1yFusDzOTqSt#ERPQ4XA#3$CDvc#>c)3>i~3uF6<6d8J!hUO}}O4sKC({hG~mPLDrx$WC4G~ zXhRCghMY2}11Ia@#Q9-B0#-4UOyzlaaDKzvG$4RE8YKWk=2;)6s3bT+_$@5K#t0|C zWJNkS;X71}JizrUN*2Hv%3(-VR3(axD#)Ce9{*gJ&iZMLzHt= z1K3LSr(pHco3l)IpHeD9!7_O=GSZWm)p~Jaj)wZzZgy37IG)wV*w$J?RTr$b@(Mx$ zEl0B=aGKj3EFx-kY*trMZ#k$NDsjsSPNZzVBAt^s-OeW+$%IVqy)6F%_`c=k*Jr_@ zgcI@-i2v#HIw#_P)H3C(%A63ymKDEjV)%PG48Jk%dNAe(GPweNx#f~)k<0(mvvXzL zHRB013xAz|R+YbA&x&nBrxD>9n@+)oiFWA{UZZ8MQ|6?|LgiVx_$?&6rV*yEJ%ORL z#vI(lHmMn&t7oC?Iy(YppIS9tx5_+Fgv_PTIwfFEaxK)L7`1yD+GD|-wKY<3;l_`^ z-V)65QRj4%z zFvR}$AFFlMan1exDt)Cne&oWy=nD=d*I0+d-oxJLgX8FaZoQ^f1P#OD|3H1y7L&4+ znQav~wjoNO?L${{{lc6sOwWQSV|8;!xwPKd*xR37mX2TKMPJ@7_6chD+xhQHEBYF@ z$r$@=Is!|O^}3Zsjv-C(-S#-5THb-#)I5NwI5)|w9sgUi&g$u2)BZ)N5T{Z30-~Ie znq?HwD5P_i2Z`wLsuZwgwvAT!uG{=IZ#%z-5~S`*In>HSTb|U;ABY-mE&+DE2Km!^ zBw>tG@dq}r0S84ffpu{$h(D*OH*GAfkqSY?Qjo&{nC0RVAA(C9j}Q!|0& zgy7^~h4T3!!Z!?>AJCsUAD9-{RG<|Ho&Pn2AriAfR8>4?Qc#yKJ>atatgA0ImP*5@F(ZvMNjLb7+S z#Nb!F`q`d{Uq z)&zvox$eGvDi@uYqg*;iUa0&xXWG$DS^r;Sc3aDw|77<&BYe-t;EYZdMOrToU?(+n z&a+0?pkowLSkbWPpRiK@9`Bnj5}60|NIdfn515TrH94dP4L2ut#+t{ke84FPV#E$r zH|96e@J}-#ak#To`p2WlfOpFx%$02_EwLt`Yh8#5m%lt0JsO;r4%&CH8aj!YLfr#L z6GVrO`_7wLN5K@v@Up-{`qaPDjb!_8g1{-Dg)hRqts>!m$@yn0MG2b-qAO3z>Zm!2 zR+BnvQ=pid?-fgHbCt_686MCTw~UEGb@HV!B|%&uspQMO5%(&p&H=l2upmO1m1X!r zB5z86aKfqp(%lZFi_2{&)l1pGi#%;x9%p{?fwRc==vd$+h!Xv1%8{b)J8Hs^QXi{0 z^L`k~Wx?>vE41i=6#ZAeFH1=hPaB>fLb4dpL07dGa;HsK5*%P;SydUMC&RS;@>D#* zXq-hqg7~&18&=Ens+Q%V^%;4IrGYf6l%q`e8<1s_Zd9^YiZh*MF7pbpkIAonHTGlD zrFm(4Tu@$>mPovF1Z8_*)$g%Lpn%V{UZ|&5_bjs zDY(jHI7X9KtN97XMy2IAZR2MESC3h!gGj$r#wr47?m|sk+$>B{QLa}Bd$Eal%L3CkR7<7+Og9#7u#clRl&#((5Sl+RL_I)uZPmj zB5YnA1~c|rZNZvIhFF468FL0?SzPF~$R9AG(r{s{r|JCD^5^huLD7ksU<>4vaX>c( zms1{NFD1-)a{ssG@5m=T$0KYso&AXnj-+WPtsU0>8>wv@n^sj@a&n;3iBBV!bb?J^ zzBv^E3@WCRwhC_MS@?(MN8vhZ99@Ai8aql!M)e>n#h4$0PdH@Aw`?Wk`4re{x(>>F zUXuS(Kk=!3T*|<&gwv&YgsE(0spTmUDzui^9v#7U#mPoi=R7AuAY_}dFu@V;vNMLx zyuBG&8|Jbi?jSfH^$}iawKkVF2zO} z>q*|q%&wol$(+cgsy(ysVTgUU=A0B_*xZ@(NFs?9+N$HeZxa|2$|i7WLRVjf(5c@3 zIl|rH$s@Bu((JV_JPMT1MwCt(gyum;!elqOKy!R()8mmrR8cAiXM6b zfRa#R3{ZNZB!5I};<5B5b(AF>sf#;l>*=p?uM`K;RUK~3sEiURFIV_x2&hl_pJ+@i zp5himjU>L#P;FShE!ELk`(z zdEMeN?}@V=sITC`3uxno>)_0TQs3bfmUhh;HQ`>#yA&5Vd16~OeB6NX3vL9`d|8q; z@#{u)@eMUtpfWA)O(?%QC-l&hsW&S*Nu=gxI}Tk7xFfJCdPvnso3zO&vZOXR)Y`D8 z5-r;>hl_Mg6aS0p>yU^PaE!GLjZ)I(uBuUAQHiT*YPaFhLuzFPUH(VaUr|p| zyRLEFIJ{^Vq@+$--!-&C`14^xFi5g7axcy{w#x2>RpY7_rd-h$4B!RqEAbXi&88*u zhk3T>rf9763h(_}*nj%XG_p>@-!4qZ(!7or3pKh0%gNGF7L5=QHzcP@XldLsD^EAm z)S_~-kkojXJW!$lJwqh%^}^d7eosmQT)!;B$GGg-=WQ}ibQWirOi|(HbQ`}6VLjV} z&YlQM#36aAdyF0SkR}~-@}$8)$tHL-ZC9H0Ff~X#&(mGgky)?OHI1J1I=EuI@M!Wa zH+mPoVn#K42#Nxv6zclVTANF8D;n!)Lju~{h9>uyF6f>9=!>C0ZH(EN!{;Ta@5*lVb)Z5Kux{RZ5Gmv(j+dz6T zD0F^KhhJn*=VY|34)03JdLI^>+k5d5(W&xdzy8S)vvs{Pz_rL^z)5as=6c#Qcrz_V znLL!1kQJd@($dx6Zvmrdq6HA5$b=Z9DF5SKFt0{N(Z4^VNN+-8m$*IS(Ok-|6?ob<9HYRjDT)3%zuA2@b z3F*MU0tO4pLe%sGbMm{}IT17pq)ojdAnz|RWEvxJWf`_CzQg1P$sRfsMqYUbwnwUE zh#GQ4ZG5o}Yujk2;NVwW(2)1URo}a?qsn3q@6?Rd;$P&+@BRlJwOZ z&l9aGpLewCxa3q***2IZE*iY1TvZqwnYSk33I`58@fH!I@z3gRM>r5#33qxLP5oA&`VbiIp z-VLgMEms|lJ2&&C z7?$U@CN1_Bz$JJ5?|2Yf7?g6i}X~AsH*H>@x3*ai)z?83nhA-oM11CIIG^JZEDBNbVH59K{r7Xi?%Vz zYE$;l>Ziup&H1K5%r2#W1Rr&kruU13HsWU4U5)Nk43@OOWL7Jo+0)|tpw)m0N!vsy zf7x2vw9VCx-qv_|7tQSmaib8)t;JH|E==rH0$9tK@S^+nyEew1;HRdoaSP2^YdqKp z>P!^KZW1A7xN9vHmnWvIH3h$a;W{5CK;9Jzpgp}c$4-0gRcqn-_$t6W`YLTJWO(B@ zng2TOkgHffsj@p~$9*Wzw26VlA%qwx=knVn9355MF$3<*$?Nsxj69pga9`BxTR#%vRwX&{Mn<9sA0+<6~(mJ{RZ1vkiDL60A*4~4yr8-2bv@zH)X z?~!+Ibj1I#6#l}~xA&>_T{fmX4jH%foWQ$VZCcl027Ped&+7!l@1mk3KdnLDIr2<{0p8phYkqIQG{33f{bJ=p-^_uFv|9H|&hv9a zEZ$3H@yu{S!}I*cmv)~a z%36J?c|OlAkU_%XIbg!r=_dJ8H!NAhb9G7`Ea}G;a)%yh*4>q3(mIHocsJfUnp1YQ zX8Hc7kWTeYS`+-e`I(QB8Mtw9d8X=f#Hq2Cmv6;Y{ITMh_^&u)|6|%uJP7t)g|2J< zqPPNTJ^$7pflUg8q&KmB3CA;pE#fyiMV+kg_4-^xdSCP)Sybz9-F9Ry*UZzUO$ej< zNR?u*0y2|d1(p`o; zU@`C1^P}gML*j$piY6Z;to)}>-_Vxz);-K$=^yTy$?JuBns@OOmM6!nhR=*)>N<9| z?DfycThtD6>Sp%4n=Gjc-Z`P~+>@xAkx1^bZZkK2crRSvJ+G_a($TryImLIigqWRx zqj3TvzNXc6lku7_sW;DVyvQq`_mC*g)hl>+%7kNBDFP0DaF|s<4y;&Q_mAl<@Gbbd zz|JoSgn-#+V*9JP>z%_OjIhI3!b5JG1Is8UsG3l~aRa~H*n0hKN$5M{9dDxTz>0Ar z{*}^pBAz#eyCS6{i}_>RV0Xx$E@2pDD*iS?ORnFxL8q5L_uY>h#Nr#8G`sRPmLmt> zwIZtvOfr6YDOBy!)*GOQuCG!WbY*iZ(`@Z-w-B7t!1|0H<{c^KURfW1SBgDgKayfE+otSW?p#k9-Tv?#yVjrlT7 u3mfpR;_I$uhi0>e-)$sTQtKfgx7$4iY|)3(z6tof^gf9YIW4zMj9?(!G24iDVR16ciBSjX1$1 zpbVCvs5|z5znOqaSz6`0&)q%5(T;v)c-G35)5?`Y`TY7{yD)g#I_t!bqksH_zhFN- zIgu6He&qh+CuPF^(*Ehce*5v)|3eR{i9TYUR)L?SPa}ow^VHK%ztn1AwF;bCjh@g? zqdq^LJV70`8YBMn^plsStH0H1S(d#R2=!vcAp$0~(K=o^>vZ3T%6@?YFLrAG1d}PW zzcU9uwS6o7?Z+Q}T{`>UP;$Oyr0~63z<#Zv=kT?$UK{A*hhLL)4ekF8CifDDk?*eK z&FW{e#jrWkl*YE8@vgs$Y1x~9f1*=_*f0s#OW3?%rkCLWNS z#o!Y;fDhnjtf(~K6~{8hgAbpPsR6*`mq-rVL|Z z=o!4ykzHhhk^?%mvW#R|NchOF^&lav|m&5&IF%+Cs$BW79Xm~r@K zhOYAblNr2U&W?5_=o!V^ks9O}4C$aaI}8FFU3T(xmu6Q@4Rl#^kNHzfP4z5@Awiz* z(&p5Kfl{FJbO%--k0tb;M(^DLlAGc0pXnDEsH2hR7d|oTWYY?R2+0{;$?b826;CtG z;ly>voLV-E|{jc(9?n6 z*j?1qIG4loHYLc)fEY}cgy>`Y{(|{;>iy$~h4tefZ|J;zDiSQlyp@;(jSUR`s$kK^ ztLd37h@=-dQnY-hm-IteUui>umkXn+Gq;l(BRwMz{*xx1ruE%$q7(fIzZr~4W~R+NTians zj%V}wTbRUdYfpLCs*|2hC~kjD9+ag`T_lz3%jIt9{7he7e;;+e|G@TGZHs+Y$lK>E zRQEHjxtNVf&Ct}kp*5qt9f$JD=7O#1lPmM09U{38ZIRlIp{;=xsvBgNiux}RY-W4d zkBH{dn96zmnMA`iw8mF`;6^==1hTPwJXwtzfMweo*i}a?Y=5uN8Ia_l4}fSz?op1d zv{`aMsX9>dK48t81Rm(qbD03?vwqYYDDIpi>*^;5lPRAXy|WF%uv!NhH_J zm;j6ua^NxzFlmso#14SUlbXc3K(k2GWpOagNH^p9V8)Z7%XxWv#*$1Vt_duL+)G^< z*h{h@byr}{lg}~sz}vvrseZ;cz>=Ym+&B{y5Mb0S6LuhqDG&69VC7P=+e^S& zr!uME2dP3eKz|Lk3iRNEexBY=r+I0Rf`dy-nK%M?OwU|#59$rtcZV8i@$}jqwm{pW zv25hR%;x>*S&}%|78o8#DM=^H7|1Hg4H!$3pGiJrLU$aKeVy5X$%Gs-uofoINRE^+ zwhNRsIf>b^r0S$*v390$H$bdz;HE?QVhwnC}d4mE8-_6`T#pt}| zdK-%8`M~wxsD6XT+(kr<5&_9EU>Q)8SeRtNlA11sMHVh8GZqe66j1Y|$4GHREwG*n zR&P|y^g{5qL2^{0^y*-Jpt7XTyGfJ6nZ6-y0M&H+3E1+~Gt+;TvN27?11Z~CbXUp8 z&4s&<=m8iI(&0c$m?*e7pw(Ek0WqYFrI=-Phn~AbCTncy<#IR!%^H%USIIDSU&r*O zF`}e)Kx52k>ULUmQy(Q+ZA3q35+#F>VFr>elh2HhNJwrY(^F=o$jlD@r?h2LhLvZ; z@?P%7jBAYrPhgo)8hd1K!6bumV>ig_o!Jj2(hho-#$d7{M=bNKD5NJvEVEP$Z`@~l zm~!wY87uRuNU9n;byXx6?z9f;01dXckX=hHkQ-1Hy^V# zOXqf8-fuZh(S6U~a-yO?drOy7l$o&$WzMu_bvb4@KQV<)c@ZYP;YZn#Sq zTq8CUZ)`<=LT_cYm2ILa1;dK|s& z=T;A3@o4YudObcyayYf*dJ}hL>_fX}gE!#EcrSF{!u@*G%lNhEr>kBQTVvyl?bFM| z^!I)~u83AjQ#je5>gn6Q?)5!=Goe~ke{F4E{r0kt$J^z&*6r6CnK&VnK~Kt%9C2XA z10~+M3$8Vo^ZVXHSPWLl9;XN50Mifi!>}=^F<9>)2cpox2wIus{Z_$n?=YprIYx+w z*3Z0T zUt4q*jX0&fq7RGFnD6Kl!W%i@nWvjBk{i8&=LX&OMg&uf7d2m^L@;BhP#W4~PhsCA zV@JL&`AmugDpmS*>9ES%^;{VN8HkLGjeGK-37fO3%Ci+)U+TUb+T0svX7WMpI@lbH zp(7t-zMT7A(H6Q&uoxSG&}QQLIKq27)M0S3A1=54gjx-(<-RqM1aZ71CKG!%%S;N*kQv6qS@iyc;SDc06bMU1cXurugtJk3(uRMGD3vRZP_$ ztKzC9Xf!^v)VDzE;5MT+g=vFzw^|q1_xRG#v>=~9Y}mL9FO5~O#x+be19clrY1Q(A zgEk%8w!_wiskd@Ww$+(tow=?)=W&k?R97XaH+{78CIsi}bJZVfJxkw`3yBy`_2+ac zabsmR41$rSjc1M>=JlW% z%@wv2PaEVs6rQuFy_h%TT2m*xZqFyFm|eRTk&m|nzBhKaV_IztTOJ&6Tz<0f6o-9l z=tD2+ad^Xe^_tH-$$Kk*&=LlXFs}!H-s;WxbZH~nDb0c3bpd)FMGHuL!9gjMF62m5wT`CLsV5?5jsf-6xjSy`Mf2Q@BxQGeu zkk@!~h=f-64o;l5SM^rS6*p9;p-jBPP?$8*?LvPZt&v=_J6hI; z8q;*^VX%QQbt{So8^7IB%oR3VnXS!qZ4I}tue_Nu4q?6ei#!fVhaK?RK4jX64%*ZH zu2$;}+Pmnms!3R9(8rb0)aitE;<@JZp2?{3Hd^NSd$#5+wYH9)>+?2Q+v~M))Y)vC z{aSjcne$p6bNpI6E!X60ImybfoXTQOxkA)&49CDaIBvH)cB}5xJ5HxLCmaIDTkw_j z>R_hX!JT)jPPOBz*2Ut|+Q}EUXd^wPbWDf9@Xjw-^A`et!W)0Xu)j4MPtO?qRM)Cb zO5*jU{K)eIf3S#FmqM%`IFtSAL#oCmjW4m>+E{sXY;8g(N-x*p{3vx?XZ;)A z`n7hvKFV!;+e!MdwRG&ubj0C@RC2WHC%zecFiU5~mv}xjT!)G`FwONxYd#p$_oL-4 z?H`R97;faG--Az}8Dr0ZLhly|YbQu-`}1-!^Q-!9{;i4st^Rtwz6(XpcUJfy-a4rM zx<+IN&fOUkVDJ0vd6BnBp&PILLmU|awA+vJiu^1gE4ER)zmgCM(u3U< z0$sx~XG#V$z8iyvL`wXa@qjl}uqJM5FlKfscyA9CeCC;5H30BE#WW`qLU@~LA_jPk z0pmA_VRu1Bai5#tcjoE;q=zONW_`0e;GJXl1H1Ew{CNuUApC^;Q zzmmv5TR8E8dKK1b)?2TQI^O#E>9s@r>x)ED>ysU{SK+`$4*StxuKH!)_5P#vKZ#fW z3H9f(6KRzl!yZP#{%ePE%^6?Wu>n@R@sdgX1Ky7n>x# zB7iqSt|eIRyp28BSCbdYh_*&@ zB_kj7@pM($jbbUQH{AzL6@4y@GJNw4JI*SIcaQ=@uZ^r~{}0#O(+@vXLb!7R={jCI z|B(dVT_fR%vr9kAEP4-uIC4iu>T$t-<$=iZ&Ou={Wy^^!ob0;}?jWxFgDVc9Qnd98mvG%DT5+=fGT7C922)D*V&g4AdDHH z4QQBPICBB%t$Iwb(m81E(H58{h91LQq=kO>KCo3#CXa5IxD5Z98BZzfSHUtH=Thw5WrAw%o(G|9~X2++dwR+<`_aFZ_chgF*y4p3* z1M;_ZI>)~mFwV;Vu~se)!~AbnY{dq8)q2H?n`C3c`v!*n_|I?iI7(A9_%_GB%_ARv zTb=;YVQV9P0x3yyv&E_|E?Kly-FoTT)zw;MZ@fgb= zV!7b|{pRde2AodWCl!DHW+g6+JHA;6@@-|T?QiL>*}~wzI8NkWtv321h(`X^#?V*2 zod>)}RI!gq7}(fNN5Fb(t=2Pi54Y{&@7vFO5#GXcCU zqolLR@yV93c$LQzmqA1cnW3COlUb}enM_qo*MwxI7JITT%EjEPN5O!eBj{%CQx8@EPIJr{J!zK8y|S5t;l^k%<`F z&YwFdX%a9fAzt%klX|=H^HZH@y)=KXKm>aPB9B1i5s3VbfykfQ31f4KLDdw;n1zmR*UV3wgnej3M{wdFi%j+r>?t#AFwJ$;9GX>`t?b_Z=^=T0Ac zzpjAezvH?>^DzKC27t!^@E8F8N&~=9oG{|#1#N8qrF?z|M)%D|^AC6Wjfc~JIQ@sy ze>nZWkkg+~BN*{pODBCo;l8u}%X$7*}L14%PkfBr838e}wh) z8x7}^td~bvk0&@QCs{9#uvQONmM2*+kFXxgrE-$>@(AnkG->4|>*W#Fqb-$_td~bv zPbXW?Cs{9#u$GYON!H6FtjC0^KfZ8rg!OnPwQ`d6@(61Qsh(uLJi=N+swY`5kFXwX zshnhe9AW)49AUMJ&8R;lI;s3(^YQ0FeY^UL>K}hm7$H9VSn0KqwEPkLMZ88Lm!rpr zu*kR;$Y3w{weZDQTb{qYh)K@vR*%TUn2OkdrWGLZGzk>0x8LbPcNSRrkJB{u0x57E z-w4%(-4$m7>RW`jLXJ+QWX!CCF#ufIGvc1rrO3goJC}k? zc~)H%1{BB;GunV^?8Xc9o!@T;5-5M|1?Y$@&YZ9SQ(4c_QR-twq%~wk&h)1W1ybe# z?E*!jQ{JHE#8RP0>W8vtbL@zbc8FqeP$2JNDJ@g4aFyQTDLo=@vy~p1N^c-<@cMm< zJTaD*$Q!(+RnF3r&M%ou3nbz%{ffJ^L|$Pqy<{&fQ@@kL^gb5TvejMilwRX8Es$Ta znIb0B0(lRgX@RV=n%-hGEs&q_njSe#%a(ixv+2LxmM$1htL&x)@*a-Uv(B$rP7CBU zuG34N)3Pny$#!}l-)VuoWIg@=Ri0llpBBh_xJpYqKVd*U>MfCXaG)MN?M-IYYk9xI zg<9o7{jrRTzh*-%+x)c_e8GoWAg{5aUNWK{`_X;8sP{3W7RXC})N96|7S|Aa7>Q~IE zrOqq-sh8ZTKbF7HcOQf5eH^L<@{&pQ8jEUy{EABz@u(Kad)QPXu~S|IP?T`k*Zg?;rF^J;meVPGv=a>>K`83${b zqZ>@DRTkE=h1|i#S|AZ4>sM^7<^1RhC+j62Yv~#8WM#dNm$g7%va?=eW-VieFZfvz zH*4wZ?qFywkX4@6TO6&WpZ|oZ^~lm%An)O7EhE5^vGp^y)-w9N!P#2nYb}uXu(p6vZ}Gks$j{hckIb(H@*e)z zvMrS?u%9u&7RXyXuvHG&(u3W>1Y3?W+F|w;7i>90xxxs0$p%{>@8yKOj}^8+UUI`; zF$;AxMEnYoE%pL4L;)W-Rak_i&ecTvZkEr_k#0pW5@n-G2MaD zBoOIs$rI_3EbafLT=ihuMldd=KwW_|8-9mh}3j$M(H|gj5ARzL? z3T5vnd+rsL6JzZ<>9wlHuMrwr{@RKMenh_`YddbXv_I^&oLTA>mxdcN&{!@yfqj=CZ$B zV`=@I&2P9#1DMN;6EG*z*1(9CFK+6Ds~~R*mt=cYaYgP|8`IG(>0xJUU~ajFB}w@T znIMI4-b|UySjS1H{_DK$8=Zw8C=CM4X8E`^CutDNB8|i8t_%jp1vR=@`gcJOxz={UBheWG!MN%+Hj!g_r-GJlvIABP$dS__DKHtX&asDcogL>dtw6QsMWx`Pd z?K5!hNoHXZ*v2IAjiK3lv#w_SWhX^)W*Bs>kPTc9P^iBCdd}W-s{>dM7!jabV^5= zX?O-lXKYriM+!e4M;KHltg7=IyO~j>I?iRmeQ|ObOmO0scuP4vIpd6_HpYQkNC74&hAU~gw-*-S!hxytjd~BqOa2(;_7hf0z zHL+=fa*sn}o*E6`SWZ!ICmp#Q)U)g^d*zF>j((>b@j>s&^;ek3PZ z)Ob$@y@j4FPi>qL=)nSD;z3~g6PuP7ZYt6`E>bupE%dxR`wBzI6A72;B|exaJ2^i@ z1J%06ZBSHB^fvFm(m4WqzrK(9TQyQ+&=x}8>1x|rt(GzG)b&|O=VcwT#5zkhl*>eOmj2sjOeiGBkarQlY zv=Dwb8EsCeJ@M0Pqhmp!dl9?kQcC2%ZJg>E)K3jkZ-u$nMr_qjd^2#=*tdJm{xS!Z z11e_@@iVP>o~6)Or+ImWFj(L=zU?FhA^0V~xXz3}5R4;X6s;MN<$R~Ck~mD1`s;Vq zd9KVHqxq2?XVt*;6qXieG&k!tv^uYrGv)eqZsuHoT~`C|^Fd@B@qOoiTaLGyou8jz z6!-oNa#{WFjj06Kr}y$EaMm{gu!KzGtJj)uS7OX_^*0(HGgONF_rx&oDuA-R$c|f$ z3M)UJ^-2)$NY8VI>PEZuQvKVUS0XQ}PN#K2sE#+A?>W<(SBmjmtN+bM$;v^;#NOFo znVuXEdBx#N97aB3PhY(Z;Z@16{^B1saQUm)T)?|`NuFmrdFqJBTQ!1BrHfS)tWE$w z>)EFWyxDyB6NlLQ$p6DRv8p-eo_Ok<`LreuS8FFphG-(RtaR&(-eJY7J)D#*9h(o4 z%lHHdlTV#7v^{V}G+PB8TVRfJcB>eA1Bk9du1?P@cxhti!^UekLA_=-@xg8{4THSQCi|6hO$7TO;zw4nu^rN|mZ%X% z$3Hi?S(c@M7)SZI1??^t8*bN9U?eG%7tt-VUO$)x9Wb29<{sli`B!I$#lnqa*uyB; z-!=g@yExdO86zxCaxnT>r;{{T57 JXNe}C007y~HiiHI From e874cc2b168712c26a39bdc85bb01e19fce53a32 Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Thu, 23 May 2019 10:35:30 +0200 Subject: [PATCH 44/79] implemented queuing update requests and canceling new ones when the queue is full --- Makefile | 2 +- repo/loader.go | 38 ++++++++++++++++++++++++---------- repo/repo.go | 44 ++++++++++++++++++++++++++-------------- server/server.go | 23 ++++++++++++++++----- testing/client/client.go | 2 +- 5 files changed, 76 insertions(+), 33 deletions(-) diff --git a/Makefile b/Makefile index 6089af2..1e079d9 100644 --- a/Makefile +++ b/Makefile @@ -56,7 +56,7 @@ run-testserver: bin/testserver -json-file var/cse-globus-stage-b-with-main-section.json run-contentserver: - contentserver -var-dir var -webserver-address :9191 -address :9999 -log-level debug http://127.0.0.1:1234 + contentserver -var-dir var -webserver-address :9191 -address :9999 -log-level notice http://127.0.0.1:1234 clean-var: rm var/contentserver-repo-2019* diff --git a/repo/loader.go b/repo/loader.go index 783bd95..a151efc 100644 --- a/repo/loader.go +++ b/repo/loader.go @@ -17,14 +17,15 @@ var json = jsoniter.ConfigCompatibleWithStandardLibrary func (repo *Repo) updateRoutine() { go func() { for newDimension := range repo.updateChannel { - log.Debug("update routine received a new dimension: " + newDimension.Dimension) + log.Notice("update routine received a new dimension: " + newDimension.Dimension) err := repo._updateDimension(newDimension.Dimension, newDimension.Node) - log.Debug("update routine received result") + log.Notice("update routine received result") if err != nil { log.Debug(" update routine error: " + err.Error()) } repo.updateDoneChannel <- err + repo.updateCompleteChannel <- true } }() } @@ -110,11 +111,19 @@ func loadNodesFromJSON(jsonBytes []byte) (nodes map[string]*content.RepoNode, er } func (repo *Repo) tryToRestoreCurrent() error { - currentJSONBytes, err := repo.history.getCurrent() - if err != nil { - return err + + select { + case repo.updateInProgressChannel <- time.Now(): + log.Notice("update request added to queue") + currentJSONBytes, err := repo.history.getCurrent() + if err != nil { + return err + } + return repo.loadJSONBytes(currentJSONBytes) + default: + log.Notice("invalidation request ignored, queue seems to be full") + return errors.New("queue full") } - return repo.loadJSONBytes(currentJSONBytes) } func get(URL string) (data []byte, err error) { @@ -130,11 +139,6 @@ func get(URL string) (data []byte, err error) { } func (repo *Repo) update() (repoRuntime int64, jsonBytes []byte, err error) { - - // limit ressources and allow only one update request at once - repo.updateLock.Lock() - defer repo.updateLock.Unlock() - startTimeRepo := time.Now().UnixNano() jsonBytes, err = get(repo.server) repoRuntime = time.Now().UnixNano() - startTimeRepo @@ -157,6 +161,18 @@ func (repo *Repo) update() (repoRuntime int64, jsonBytes []byte, err error) { return repoRuntime, jsonBytes, nil } +// limit ressources and allow only one update request at once +func (repo *Repo) tryUpdate() (repoRuntime int64, jsonBytes []byte, err error) { + select { + case repo.updateInProgressChannel <- time.Now(): + log.Notice("update request added to queue") + return repo.update() + default: + log.Notice("invalidation request ignored, queue seems to be full") + return 0, nil, errors.New("queue full") + } +} + func (repo *Repo) loadJSONBytes(jsonBytes []byte) error { nodes, err := loadNodesFromJSON(jsonBytes) if err != nil { diff --git a/repo/repo.go b/repo/repo.go index b2cb75b..a654162 100644 --- a/repo/repo.go +++ b/repo/repo.go @@ -4,7 +4,6 @@ import ( "errors" "fmt" "strings" - "sync" "time" "github.com/foomo/contentserver/content" @@ -24,12 +23,14 @@ type Dimension struct { // Repo content repositiory type Repo struct { - server string - Directory map[string]*Dimension - updateLock *sync.Mutex - updateChannel chan *repoDimension - updateDoneChannel chan error - history *history + server string + Directory map[string]*Dimension + // updateLock sync.Mutex + updateChannel chan *repoDimension + updateDoneChannel chan error + history *history + updateInProgressChannel chan time.Time + updateCompleteChannel chan bool } type repoDimension struct { @@ -42,15 +43,26 @@ func NewRepo(server string, varDir string) *Repo { log.Notice("creating new repo for " + server) log.Notice(" using var dir:" + varDir) repo := &Repo{ - server: server, - Directory: map[string]*Dimension{}, - history: newHistory(varDir), - updateLock: &sync.Mutex{}, - updateChannel: make(chan *repoDimension), - updateDoneChannel: make(chan error), + server: server, + Directory: map[string]*Dimension{}, + history: newHistory(varDir), + updateChannel: make(chan *repoDimension), + updateDoneChannel: make(chan error), + updateInProgressChannel: make(chan time.Time, 1), + updateCompleteChannel: make(chan bool), } + go func() { + for { + select { + case t := <-repo.updateInProgressChannel: + log.Notice("got timestamp: ", t, "waiting for update to complete") + <-repo.updateCompleteChannel + log.Notice("update completed!") + } + } + }() go repo.updateRoutine() - log.Record("trying to restore pervious state") + log.Record("trying to restore previous state") restoreErr := repo.tryToRestoreCurrent() if restoreErr != nil { log.Record(" could not restore previous repo content:" + restoreErr.Error()) @@ -194,10 +206,12 @@ func (repo *Repo) Update() (updateResponse *responses.Update) { return float64(float64(nanoSeconds) / float64(1000000000.0)) } startTime := time.Now().UnixNano() - updateRepotime, jsonBytes, updateErr := repo.update() + updateRepotime, jsonBytes, updateErr := repo.tryUpdate() updateResponse = &responses.Update{} updateResponse.Stats.RepoRuntime = floatSeconds(updateRepotime) + log.Notice("Update triggered") + if updateErr != nil { updateResponse.Success = false updateResponse.Stats.NumberOfNodes = -1 diff --git a/server/server.go b/server/server.go index bf7e249..59b1b73 100644 --- a/server/server.go +++ b/server/server.go @@ -5,6 +5,7 @@ import ( "fmt" "net" "net/http" + "os" "github.com/foomo/contentserver/log" "github.com/foomo/contentserver/repo" @@ -40,23 +41,35 @@ func Run(server string, address string, varDir string) error { func RunServerSocketAndWebServer( server string, address string, - webserverAdresss string, + webserverAddress string, webserverPath string, varDir string, ) error { - if address == "" && webserverAdresss == "" { + if address == "" && webserverAddress == "" { return errors.New("one of the addresses needs to be set") } log.Record("building repo with content from " + server) r := repo.NewRepo(server, varDir) - go r.Update() + + // start initial update and handle error + go func() { + resp := r.Update() + if !resp.Success { + log.Error("failed to update: ", resp) + os.Exit(1) + } + }() + // update can run in bg chanErr := make(chan error) + if address != "" { + log.Notice("starting socketserver on: ", address) go runSocketServer(r, address, chanErr) } - if webserverAdresss != "" { - go runWebserver(r, webserverAdresss, webserverPath, chanErr) + if webserverAddress != "" { + log.Notice("starting webserver on: ", webserverAddress) + go runWebserver(r, webserverAddress, webserverPath, chanErr) } return <-chanErr } diff --git a/testing/client/client.go b/testing/client/client.go index eb31011..1023dce 100644 --- a/testing/client/client.go +++ b/testing/client/client.go @@ -25,7 +25,7 @@ func main() { } log.Println(num, "update done", resp) }(i) - time.Sleep(5 * time.Second) + time.Sleep(1 * time.Second) } log.Println("done") From d9f6cc60c45616892f0ca2284fd44ad4e1423bf1 Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Thu, 23 May 2019 10:39:44 +0200 Subject: [PATCH 45/79] cleanup --- repo/loader.go | 4 ++-- server/server.go | 3 --- 2 files changed, 2 insertions(+), 5 deletions(-) diff --git a/repo/loader.go b/repo/loader.go index a151efc..3839d8d 100644 --- a/repo/loader.go +++ b/repo/loader.go @@ -121,8 +121,8 @@ func (repo *Repo) tryToRestoreCurrent() error { } return repo.loadJSONBytes(currentJSONBytes) default: - log.Notice("invalidation request ignored, queue seems to be full") - return errors.New("queue full") + log.Notice("update request ignored, queue seems to be full") + return errors.New("Update request queue is full. Please try again later.") } } diff --git a/server/server.go b/server/server.go index 59b1b73..6de2063 100644 --- a/server/server.go +++ b/server/server.go @@ -10,9 +10,6 @@ import ( "github.com/foomo/contentserver/log" "github.com/foomo/contentserver/repo" jsoniter "github.com/json-iterator/go" - - // profiling - _ "net/http/pprof" ) var json = jsoniter.ConfigCompatibleWithStandardLibrary From e9245a200c9de25607df180b095234ec92852633 Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Thu, 23 May 2019 14:20:38 +0200 Subject: [PATCH 46/79] replaced logger, fixed update queue mechanism --- .gitignore | 1 + client/client_test.go | 8 +++- contentserver.go | 40 ++++++---------- log/log.go | 79 ------------------------------ logger/log.go | 39 +++++++++++++++ metrics/prometheus.go | 12 +++-- repo/loader.go | 84 ++++++++++++++++---------------- repo/mock/mock.go | 3 +- repo/repo.go | 103 ++++++++++++++++++++++++---------------- repo/repo_test.go | 6 +++ server/handlerequest.go | 11 +++-- server/server.go | 38 ++++++++++----- server/socketserver.go | 49 ++++++++++--------- server/webserver.go | 5 +- status/healthz.go | 13 +++-- 15 files changed, 251 insertions(+), 240 deletions(-) delete mode 100644 log/log.go create mode 100644 logger/log.go diff --git a/.gitignore b/.gitignore index 4230dfc..8a71575 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +*.log *.test cprof-* var diff --git a/client/client_test.go b/client/client_test.go index bf34007..d7e46ee 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -8,7 +8,7 @@ import ( "time" "github.com/foomo/contentserver/content" - "github.com/foomo/contentserver/log" + . "github.com/foomo/contentserver/logger" "github.com/foomo/contentserver/repo/mock" "github.com/foomo/contentserver/requests" "github.com/foomo/contentserver/server" @@ -21,6 +21,10 @@ var ( testServerWebserverAddr string ) +func init() { + SetupLogging(true, "contentserver_client_test.log") +} + func dump(t *testing.T, v interface{}) { jsonBytes, err := json.MarshalIndent(v, "", " ") if err != nil { @@ -51,7 +55,7 @@ func initTestServer(t testing.TB) (socketAddr, webserverAddr string) { socketAddr = getAvailableAddr() webserverAddr = getAvailableAddr() testServer, varDir := mock.GetMockData(t) - log.SelectedLevel = log.LevelError + go func() { err := server.RunServerSocketAndWebServer( testServer.URL+"/repo-two-dimensions.json", diff --git a/contentserver.go b/contentserver.go index 6aff610..abd3d6c 100644 --- a/contentserver.go +++ b/contentserver.go @@ -3,19 +3,19 @@ package main import ( "flag" "fmt" + "net/http" + _ "net/http/pprof" "os" "runtime/debug" "strings" "time" + "github.com/apex/log" + . "github.com/foomo/contentserver/logger" "github.com/foomo/contentserver/metrics" - "github.com/foomo/contentserver/status" - - "net/http" - _ "net/http/pprof" - - "github.com/foomo/contentserver/log" "github.com/foomo/contentserver/server" + "github.com/foomo/contentserver/status" + "go.uber.org/zap" ) const ( @@ -38,6 +38,7 @@ var ( flagWebserverAddress = flag.String("webserver-address", "", "address to bind web server host:port, when empty no webserver will be spawned") flagWebserverPath = flag.String("webserver-path", "/contentserver", "path to export the webserver on - useful when behind a proxy") flagVarDir = flag.String("var-dir", "/var/lib/contentserver", "where to put my data") + flagDebug = flag.Bool("debug", true, "toggle debug mode") // debugging / profiling flagFreeOSMem = flag.Int("free-os-mem", 0, "free OS mem every X minutes") @@ -69,6 +70,8 @@ func exitUsage(code int) { func main() { flag.Parse() + SetupLogging(*flagDebug, "contentserver.log") + go func() { fmt.Println(http.ListenAndServe("localhost:6060", nil)) }() @@ -79,12 +82,14 @@ func main() { } if *flagFreeOSMem > 0 { - log.Notice("[INFO] freeing OS memory every ", *flagFreeOSMem, " minutes!") + Log.Info("dumping heap every $interval minutes", zap.Int("interval", *flagHeapDump)) + Log.Info("freeing OS memory every $interval minutes", zap.Int("interval", *flagFreeOSMem)) go func() { for { select { case <-time.After(time.Duration(*flagFreeOSMem) * time.Minute): - log.Notice("FreeOSMemory") + Log.Info("dumping heap every $interval minutes", zap.Int("interval", *flagHeapDump)) + log.Info("FreeOSMemory") debug.FreeOSMemory() } } @@ -92,12 +97,12 @@ func main() { } if *flagHeapDump > 0 { - log.Notice("[INFO] dumping heap every ", *flagHeapDump, " minutes!") + Log.Info("dumping heap every $interval minutes", zap.Int("interval", *flagHeapDump)) go func() { for { select { case <-time.After(time.Duration(*flagFreeOSMem) * time.Minute): - log.Notice("HeapDump") + log.Info("HeapDump") f, err := os.Create("heapdump") if err != nil { panic("failed to create heap dump file") @@ -115,21 +120,6 @@ func main() { if len(flag.Args()) == 1 { fmt.Println(*flagAddress, flag.Arg(0)) - level := log.LevelRecord - switch *logLevel { - case logLevelError: - level = log.LevelError - case logLevelRecord: - level = log.LevelRecord - case logLevelWarning: - level = log.LevelWarning - case logLevelNotice: - level = log.LevelNotice - case logLevelDebug: - level = log.LevelDebug - } - log.SelectedLevel = level - // kickoff metric handlers go metrics.RunPrometheusHandler(DefaultPrometheusListener) go status.RunHealthzHandlerListener(DefaultHealthzHandlerAddress, ServiceName) diff --git a/log/log.go b/log/log.go deleted file mode 100644 index 892d737..0000000 --- a/log/log.go +++ /dev/null @@ -1,79 +0,0 @@ -package log - -import ( - "fmt" - "strings" - "time" -) - -// Level logging level enum -type Level int - -const ( - // LevelError an error - as bad as it gets - LevelError Level = 0 - // LevelRecord put this to the logs in any case - LevelRecord Level = 1 - // LevelWarning not that bad - LevelWarning Level = 2 - // LevelNotice almost on debug level - LevelNotice Level = 3 - // LevelDebug we are debugging - LevelDebug Level = 4 -) - -// SelectedLevel selected log level -var SelectedLevel = LevelDebug - -var prefices = map[Level]string{ - LevelRecord: "record : ", - LevelError: "error : ", - LevelWarning: "warning : ", - LevelNotice: "notice : ", - LevelDebug: "debug : ", -} - -func log(msg string, level Level) string { - if level <= SelectedLevel { - prefix := time.Now().Format(time.RFC3339Nano) + " " + prefices[level] - lines := strings.Split(msg, "\n") - for i := 0; i < len(lines); i++ { - fmt.Println(level, prefix+lines[i]) - } - } - return msg -} - -func logThings(msgs []interface{}, level Level) string { - r := "" - for _, msg := range msgs { - r += "\n" + fmt.Sprint(msg) - } - r = strings.Trim(r, "\n") - return log(r, level) -} - -// Debug write debug messages to the log -func Debug(msgs ...interface{}) string { - return logThings(msgs, LevelDebug) -} - -// Notice write notice messages to the log -func Notice(msgs ...interface{}) string { - return logThings(msgs, LevelNotice) -} - -// Warning write warning messages to the log -func Warning(msgs ...interface{}) string { - return logThings(msgs, LevelWarning) -} - -// Record write record messages to the log -func Record(msgs ...interface{}) string { - return logThings(msgs, LevelRecord) -} - -// Error write error messages to the log -func Error(msgs ...interface{}) string { - return logThings(msgs, LevelError) -} diff --git a/logger/log.go b/logger/log.go new file mode 100644 index 0000000..df0d920 --- /dev/null +++ b/logger/log.go @@ -0,0 +1,39 @@ +package logger + +import ( + "log" + "os" + + "go.uber.org/zap" +) + +var ( + // Log is the logger instance exposed by this package + // call Setup() prior to using it + // want JSON output? Set LOG_JSON env var to 1! + Log *zap.Logger +) + +// SetupLogging configures the logger +func SetupLogging(debug bool, outputPath string) { + + var err error + if debug { + zc := zap.NewDevelopmentConfig() + if os.Getenv("LOG_JSON") == "1" { + zc.Encoding = "json" + } + zc.OutputPaths = append(zc.OutputPaths, outputPath) + Log, err = zc.Build() + } else { + zc := zap.NewProductionConfig() + if os.Getenv("LOG_JSON") == "1" { + zc.Encoding = "json" + } + zc.OutputPaths = append(zc.OutputPaths, outputPath) + Log, err = zc.Build() + } + if err != nil { + log.Fatalf("can't initialize zap logger: %v", err) + } +} diff --git a/metrics/prometheus.go b/metrics/prometheus.go index 4ae5eaa..919b92a 100644 --- a/metrics/prometheus.go +++ b/metrics/prometheus.go @@ -1,11 +1,11 @@ package metrics import ( - "fmt" "net/http" - "github.com/foomo/contentserver/log" + . "github.com/foomo/contentserver/logger" "github.com/prometheus/client_golang/prometheus/promhttp" + "go.uber.org/zap" ) func PrometheusHandler() http.Handler { @@ -15,6 +15,10 @@ func PrometheusHandler() http.Handler { } func RunPrometheusHandler(listener string) { - log.Notice(fmt.Sprintf("starting prometheus handler on address '%s'", listener)) - log.Error(http.ListenAndServe(listener, PrometheusHandler())) + Log.Info("starting prometheus handler on", + zap.String("address", listener), + ) + Log.Error("server failed: ", + zap.Error(http.ListenAndServe(listener, PrometheusHandler())), + ) } diff --git a/repo/loader.go b/repo/loader.go index 3839d8d..4d7f1c8 100644 --- a/repo/loader.go +++ b/repo/loader.go @@ -8,33 +8,41 @@ import ( "time" "github.com/foomo/contentserver/content" - "github.com/foomo/contentserver/log" + . "github.com/foomo/contentserver/logger" jsoniter "github.com/json-iterator/go" + "go.uber.org/zap" ) -var json = jsoniter.ConfigCompatibleWithStandardLibrary +var ( + json = jsoniter.ConfigCompatibleWithStandardLibrary +) + +type updateResponse struct { + repoRuntime int64 + jsonBytes []byte + err error +} func (repo *Repo) updateRoutine() { - go func() { - for newDimension := range repo.updateChannel { - log.Notice("update routine received a new dimension: " + newDimension.Dimension) + for newDimension := range repo.updateChannel { + Log.Info("update routine received a new dimension", zap.String("dimension", newDimension.Dimension)) - err := repo._updateDimension(newDimension.Dimension, newDimension.Node) - log.Notice("update routine received result") - if err != nil { - log.Debug(" update routine error: " + err.Error()) - } - repo.updateDoneChannel <- err - repo.updateCompleteChannel <- true + err := repo._updateDimension(newDimension.Dimension, newDimension.Node) + Log.Info("update routine received result") + if err != nil { + Log.Debug("update dimension failed", zap.Error(err)) } - }() + repo.updateDoneChannel <- err + } } func (repo *Repo) updateDimension(dimension string, node *content.RepoNode) error { + Log.Debug("trying to push dimension into update channel", zap.String("dimension", dimension), zap.String("nodeName", node.Name)) repo.updateChannel <- &repoDimension{ Dimension: dimension, Node: node, } + Log.Debug("waiting for done signal") return <-repo.updateDoneChannel } @@ -71,7 +79,9 @@ func (repo *Repo) _updateDimension(dimension string, newNode *content.RepoNode) } func builDirectory(dirNode *content.RepoNode, directory map[string]*content.RepoNode, uRIDirectory map[string]*content.RepoNode) error { - log.Debug("repo.buildDirectory: " + dirNode.ID) + + // Log.Debug("buildDirectory", zap.String("ID", dirNode.ID)) + existingNode, ok := directory[dirNode.ID] if ok { return errors.New("duplicate node with id:" + existingNode.ID) @@ -111,19 +121,11 @@ func loadNodesFromJSON(jsonBytes []byte) (nodes map[string]*content.RepoNode, er } func (repo *Repo) tryToRestoreCurrent() error { - - select { - case repo.updateInProgressChannel <- time.Now(): - log.Notice("update request added to queue") - currentJSONBytes, err := repo.history.getCurrent() - if err != nil { - return err - } - return repo.loadJSONBytes(currentJSONBytes) - default: - log.Notice("update request ignored, queue seems to be full") - return errors.New("Update request queue is full. Please try again later.") + currentJSONBytes, err := repo.history.getCurrent() + if err != nil { + return err } + return repo.loadJSONBytes(currentJSONBytes) } func get(URL string) (data []byte, err error) { @@ -144,10 +146,10 @@ func (repo *Repo) update() (repoRuntime int64, jsonBytes []byte, err error) { repoRuntime = time.Now().UnixNano() - startTimeRepo if err != nil { // we have no json to load - the repo server did not reply - log.Debug("we have no json to load - the repo server did not reply", err) + Log.Debug("failed to load json", zap.Error(err)) return repoRuntime, jsonBytes, err } - log.Debug("loading json from: "+repo.server, "length:", len(jsonBytes)) + Log.Debug("loading json", zap.String("server", repo.server), zap.Int("length", len(jsonBytes))) nodes, err := loadNodesFromJSON(jsonBytes) if err != nil { // could not load nodes from json @@ -163,12 +165,14 @@ func (repo *Repo) update() (repoRuntime int64, jsonBytes []byte, err error) { // limit ressources and allow only one update request at once func (repo *Repo) tryUpdate() (repoRuntime int64, jsonBytes []byte, err error) { + c := make(chan updateResponse) select { - case repo.updateInProgressChannel <- time.Now(): - log.Notice("update request added to queue") - return repo.update() + case repo.updateInProgressChannel <- c: + Log.Info("update request added to queue") + ur := <-c + return ur.repoRuntime, ur.jsonBytes, ur.err default: - log.Notice("invalidation request ignored, queue seems to be full") + Log.Info("update request ignored, queue is full") return 0, nil, errors.New("queue full") } } @@ -176,22 +180,22 @@ func (repo *Repo) tryUpdate() (repoRuntime int64, jsonBytes []byte, err error) { func (repo *Repo) loadJSONBytes(jsonBytes []byte) error { nodes, err := loadNodesFromJSON(jsonBytes) if err != nil { - log.Debug("could not parse json", string(jsonBytes)) + Log.Debug("could not parse json", zap.String("json", string(jsonBytes))) return err } err = repo.loadNodes(nodes) if err == nil { historyErr := repo.history.add(jsonBytes) if historyErr != nil { - log.Warning("could not add valid json to history:" + historyErr.Error()) + Log.Error("could not add valid json to history", zap.Error(historyErr)) } else { - log.Record("added valid json to history") + Log.Info("added valid json to history") } cleanUpErr := repo.history.cleanup() if cleanUpErr != nil { - log.Warning("an error occured while cleaning up my history:", cleanUpErr) + Log.Error("an error occured while cleaning up my history", zap.Error(cleanUpErr)) } else { - log.Record("cleaned up history") + Log.Info("cleaned up history") } } return err @@ -201,10 +205,10 @@ func (repo *Repo) loadNodes(newNodes map[string]*content.RepoNode) error { newDimensions := []string{} for dimension, newNode := range newNodes { newDimensions = append(newDimensions, dimension) - log.Debug("loading nodes for dimension " + dimension) + Log.Debug("loading nodes for dimension", zap.String("dimension", dimension)) loadErr := repo.updateDimension(dimension, newNode) if loadErr != nil { - log.Debug(" failed to load " + dimension + ": " + loadErr.Error()) + Log.Debug("failed to load", zap.String("dimension", dimension), zap.Error(loadErr)) return loadErr } } @@ -219,7 +223,7 @@ func (repo *Repo) loadNodes(newNodes map[string]*content.RepoNode) error { // we need to throw away orphaned dimensions for dimension := range repo.Directory { if !dimensionIsValid(dimension) { - log.Notice("removing orphaned dimension:" + dimension) + Log.Info("removing orphaned dimension", zap.String("dimension", dimension)) delete(repo.Directory, dimension) } } diff --git a/repo/mock/mock.go b/repo/mock/mock.go index 54b7736..2b1ae02 100644 --- a/repo/mock/mock.go +++ b/repo/mock/mock.go @@ -9,13 +9,12 @@ import ( "testing" "time" - "github.com/foomo/contentserver/log" "github.com/foomo/contentserver/requests" ) // GetMockData mock data to run a repo func GetMockData(t testing.TB) (server *httptest.Server, varDir string) { - log.SelectedLevel = log.LevelError + _, filename, _, _ := runtime.Caller(0) mockDir := path.Dir(filename) diff --git a/repo/repo.go b/repo/repo.go index a654162..85a0ed3 100644 --- a/repo/repo.go +++ b/repo/repo.go @@ -7,9 +7,10 @@ import ( "time" "github.com/foomo/contentserver/content" - "github.com/foomo/contentserver/log" + . "github.com/foomo/contentserver/logger" "github.com/foomo/contentserver/requests" "github.com/foomo/contentserver/responses" + "go.uber.org/zap" ) const maxGetURIForNodeRecursionLevel = 1000 @@ -29,8 +30,7 @@ type Repo struct { updateChannel chan *repoDimension updateDoneChannel chan error history *history - updateInProgressChannel chan time.Time - updateCompleteChannel chan bool + updateInProgressChannel chan chan updateResponse } type repoDimension struct { @@ -40,34 +40,45 @@ type repoDimension struct { // NewRepo constructor func NewRepo(server string, varDir string) *Repo { - log.Notice("creating new repo for " + server) - log.Notice(" using var dir:" + varDir) + + Log.Info("creating new repo", + zap.String("server", server), + zap.String("varDir", varDir), + ) repo := &Repo{ server: server, Directory: map[string]*Dimension{}, history: newHistory(varDir), updateChannel: make(chan *repoDimension), updateDoneChannel: make(chan error), - updateInProgressChannel: make(chan time.Time, 1), - updateCompleteChannel: make(chan bool), + updateInProgressChannel: make(chan chan updateResponse, 1), } go func() { for { select { - case t := <-repo.updateInProgressChannel: - log.Notice("got timestamp: ", t, "waiting for update to complete") - <-repo.updateCompleteChannel - log.Notice("update completed!") + case resChan := <-repo.updateInProgressChannel: + Log.Info("waiting for update to complete") + start := time.Now() + + repoRuntime, jsonBytes, errUpdate := repo.update() + + resChan <- updateResponse{ + repoRuntime: repoRuntime, + jsonBytes: jsonBytes, + err: errUpdate, + } + + Log.Info("update completed", zap.Duration("duration", time.Since(start))) } } }() go repo.updateRoutine() - log.Record("trying to restore previous state") + Log.Info("trying to restore previous state") restoreErr := repo.tryToRestoreCurrent() if restoreErr != nil { - log.Record(" could not restore previous repo content:" + restoreErr.Error()) + Log.Error(" could not restore previous repo content", zap.Error(restoreErr)) } else { - log.Record(" restored previous repo content") + Log.Info("restored previous repo content") } return repo } @@ -93,7 +104,7 @@ func (repo *Repo) getNodes(nodeRequests map[string]*requests.Node, env *requests path = []*content.Item{} ) for nodeName, nodeRequest := range nodeRequests { - log.Debug(" adding node " + nodeName + " " + nodeRequest.ID) + Log.Debug("adding node", zap.String("name", nodeName), zap.String("requestID", nodeRequest.ID)) groups := env.Groups if len(nodeRequest.Groups) > 0 { @@ -104,26 +115,29 @@ func (repo *Repo) getNodes(nodeRequests map[string]*requests.Node, env *requests nodes[nodeName] = nil if !ok && nodeRequest.Dimension == "" { - log.Debug(" could not get dimension root node for dimension " + nodeRequest.Dimension) + Log.Debug("could not get dimension root node", zap.String("dimension", nodeRequest.Dimension)) for _, dimension := range env.Dimensions { dimensionNode, ok = repo.Directory[dimension] if ok { - log.Debug(" searched for root node in env.dimension " + dimension + " with success") + Log.Debug("found root node in env.Dimensions", zap.String("dimension", dimension)) break } - log.Debug(" searched for root node in env.dimension " + dimension + " without success") + Log.Debug("could NOT find root node in env.Dimensions", zap.String("dimension", dimension)) } } if !ok { - log.Warning("could not get dimension root node for nodeRequest.Dimension: " + nodeRequest.Dimension) + Log.Error("could not get dimension root node", zap.String("nodeRequest.Dimension", nodeRequest.Dimension)) continue } treeNode, ok := dimensionNode.Directory[nodeRequest.ID] if ok { nodes[nodeName] = repo.getNode(treeNode, nodeRequest.Expand, nodeRequest.MimeTypes, path, 0, groups, nodeRequest.DataFields) } else { - log.Warning("you are requesting an invalid tree node for " + nodeName + " : " + nodeRequest.ID) + Log.Error("an invalid tree node was requested", + zap.String("nodeName", nodeName), + zap.String("ID", nodeRequest.ID), + ) } } return nodes @@ -142,18 +156,18 @@ func (repo *Repo) GetContent(r *requests.Content) (c *content.SiteContent, err e // add more input validation err = repo.validateContentRequest(r) if err != nil { - log.Debug("repo.GetContent invalid request", err) + Log.Error("repo.GetContent invalid request", zap.Error(err)) return } - log.Debug("repo.GetContent: ", r.URI) + Log.Debug("repo.GetContent", zap.String("URI", r.URI)) c = content.NewSiteContent() resolved, resolvedURI, resolvedDimension, node := repo.resolveContent(r.Env.Dimensions, r.URI) if resolved { if !node.CanBeAccessedByGroups(r.Env.Groups) { - log.Notice("401 for " + r.URI) + Log.Warn("resolvecontent got status 401", zap.String("URI", r.URI)) c.Status = content.StatusForbidden } else { - log.Notice("200 for " + r.URI) + Log.Info("resolvecontent got status 200", zap.String("URI", r.URI)) c.Status = content.StatusOk c.Data = node.Data } @@ -169,15 +183,22 @@ func (repo *Repo) GetContent(r *requests.Content) (c *content.SiteContent, err e } c.URIs = uris } else { - log.Notice("404 for " + r.URI) + Log.Info("resolvecontent got status 404", zap.String("URI", r.URI)) c.Status = content.StatusNotFound c.Dimension = r.Env.Dimensions[0] } - if log.SelectedLevel == log.LevelDebug { - log.Debug(fmt.Sprintf("resolved: %v, uri: %v, dim: %v, n: %v", resolved, resolvedURI, resolvedDimension, node)) - } + + Log.Debug("got content", + zap.Bool("resolved", resolved), + zap.String("resolvedURI", resolvedURI), + zap.String("resolvedDimension", resolvedDimension), + zap.String("nodeName", node.Name), + ) if !resolved { - log.Debug("repo.GetContent", r.URI, "could not be resolved falling back to default dimension", r.Env.Dimensions[0]) + Log.Debug("failed to resolve, falling back to default dimension", + zap.String("URI", r.URI), + zap.String("defaultDimension", r.Env.Dimensions[0]), + ) // r.Env.Dimensions is validated => we can access it resolvedDimension = r.Env.Dimensions[0] } @@ -210,27 +231,27 @@ func (repo *Repo) Update() (updateResponse *responses.Update) { updateResponse = &responses.Update{} updateResponse.Stats.RepoRuntime = floatSeconds(updateRepotime) - log.Notice("Update triggered") + Log.Info("Update triggered") if updateErr != nil { updateResponse.Success = false updateResponse.Stats.NumberOfNodes = -1 updateResponse.Stats.NumberOfURIs = -1 // let us try to restore the world from a file - log.Error("could not update repository:" + updateErr.Error()) + Log.Error("could not update repository:" + updateErr.Error()) updateResponse.ErrorMessage = updateErr.Error() restoreErr := repo.tryToRestoreCurrent() if restoreErr != nil { - log.Error("failed to restore preceding repo version: " + restoreErr.Error()) + Log.Error("failed to restore preceding repo version", zap.Error(restoreErr)) } else { - log.Record("restored current repo from local history") + Log.Info("restored current repo from local history") } } else { updateResponse.Success = true // persist the currently loaded one historyErr := repo.history.add(jsonBytes) if historyErr != nil { - log.Warning("could not persist current repo in history: " + historyErr.Error()) + Log.Warn("could not persist current repo in history", zap.Error(historyErr)) } // add some stats for dimension := range repo.Directory { @@ -245,7 +266,7 @@ func (repo *Repo) Update() (updateResponse *responses.Update) { // resolveContent find content in a repository func (repo *Repo) resolveContent(dimensions []string, URI string) (resolved bool, resolvedURI string, resolvedDimension string, repoNode *content.RepoNode) { parts := strings.Split(URI, content.PathSeparator) - log.Debug("repo.ResolveContent: " + URI) + Log.Debug("repo.ResolveContent", zap.String("URI", URI)) for i := len(parts); i > 0; i-- { testURI := strings.Join(parts[0:i], content.PathSeparator) if testURI == "" { @@ -253,11 +274,13 @@ func (repo *Repo) resolveContent(dimensions []string, URI string) (resolved bool } for _, dimension := range dimensions { if d, ok := repo.Directory[dimension]; ok { - log.Debug(" testing[" + dimension + "]: " + testURI) + Log.Debug("checking", + zap.String("dimension", dimension), + zap.String("URI", testURI), + ) if repoNode, ok := d.URIDirectory[testURI]; ok { resolved = true - log.Debug(" found => " + testURI) - log.Debug(" destination " + fmt.Sprint(repoNode.DestinationID)) + Log.Debug("found node", zap.String("URI", testURI), zap.String("destination", repoNode.DestinationID)) if len(repoNode.DestinationID) > 0 { if destionationNode, destinationNodeOk := d.Directory[repoNode.DestinationID]; destinationNodeOk { repoNode = destionationNode @@ -279,7 +302,7 @@ func (repo *Repo) getURIForNode(dimension string, repoNode *content.RepoNode, re linkedNode, ok := repo.Directory[dimension].Directory[repoNode.LinkID] if ok { if recursionLevel > maxGetURIForNodeRecursionLevel { - log.Error("maxGetURIForNodeRecursionLevel reached for", repoNode.ID, "link id", repoNode.LinkID, "in dimension", dimension) + Log.Error("maxGetURIForNodeRecursionLevel reached", zap.String("repoNode.ID", repoNode.ID), zap.String("linkID", repoNode.LinkID), zap.String("dimension", dimension)) return "" } return repo.getURIForNode(dimension, linkedNode, recursionLevel+1) @@ -298,7 +321,7 @@ func (repo *Repo) getURI(dimension string, id string) string { func (repo *Repo) getNode(repoNode *content.RepoNode, expanded bool, mimeTypes []string, path []*content.Item, level int, groups []string, dataFields []string) *content.Node { node := content.NewNode() node.Item = repoNode.ToItem(dataFields) - log.Debug("repo.GetNode: " + repoNode.ID) + Log.Debug("getNode", zap.String("ID", repoNode.ID)) for _, childID := range repoNode.Index { childNode := repoNode.Nodes[childID] if (level == 0 || expanded || !expanded && childNode.InPath(path)) && !childNode.Hidden && childNode.CanBeAccessedByGroups(groups) && childNode.IsOneOfTheseMimeTypes(mimeTypes) { diff --git a/repo/repo_test.go b/repo/repo_test.go index 89a48e3..4c0d96e 100644 --- a/repo/repo_test.go +++ b/repo/repo_test.go @@ -4,10 +4,16 @@ import ( "strings" "testing" + . "github.com/foomo/contentserver/logger" + _ "github.com/foomo/contentserver/logger" "github.com/foomo/contentserver/repo/mock" "github.com/foomo/contentserver/requests" ) +func init() { + SetupLogging(true, "contentserver_repo_test.log") +} + func assertRepoIsEmpty(t *testing.T, r *Repo, empty bool) { if empty { if len(r.Directory) > 0 { diff --git a/server/handlerequest.go b/server/handlerequest.go index ccac8b7..bf24297 100644 --- a/server/handlerequest.go +++ b/server/handlerequest.go @@ -1,10 +1,11 @@ package server import ( - "fmt" "time" - "github.com/foomo/contentserver/log" + "go.uber.org/zap" + + . "github.com/foomo/contentserver/logger" "github.com/foomo/contentserver/repo" "github.com/foomo/contentserver/requests" "github.com/foomo/contentserver/responses" @@ -67,10 +68,10 @@ func handleRequest(r *repo.Repo, handler Handler, jsonBytes []byte, metrics *sta // error handling if jsonErr != nil { - log.Error(" could not read incoming json:", jsonErr) + Log.Error("could not read incoming json", zap.Error(jsonErr)) reply = responses.NewError(2, "could not read incoming json "+jsonErr.Error()) } else if apiErr != nil { - log.Error(" an API error occured:", apiErr) + Log.Error("an API error occured", zap.Error(apiErr)) reply = responses.NewError(3, "internal error "+apiErr.Error()) } @@ -107,7 +108,7 @@ func encodeReply(reply interface{}) (replyBytes []byte, err error) { "reply": reply, }, "", " ") if err != nil { - log.Error(" could not encode reply " + fmt.Sprint(err)) + Log.Error("could not encode reply", zap.Error(err)) } return } diff --git a/server/server.go b/server/server.go index 6de2063..82108e1 100644 --- a/server/server.go +++ b/server/server.go @@ -7,12 +7,15 @@ import ( "net/http" "os" - "github.com/foomo/contentserver/log" + . "github.com/foomo/contentserver/logger" "github.com/foomo/contentserver/repo" jsoniter "github.com/json-iterator/go" + "go.uber.org/zap" ) -var json = jsoniter.ConfigCompatibleWithStandardLibrary +var ( + json = jsoniter.ConfigCompatibleWithStandardLibrary +) // Handler type type Handler string @@ -45,14 +48,21 @@ func RunServerSocketAndWebServer( if address == "" && webserverAddress == "" { return errors.New("one of the addresses needs to be set") } - log.Record("building repo with content from " + server) + Log.Info("building repo with content", zap.String("server", server)) + r := repo.NewRepo(server, varDir) // start initial update and handle error go func() { resp := r.Update() if !resp.Success { - log.Error("failed to update: ", resp) + Log.Error("failed to update", + zap.String("error", resp.ErrorMessage), + zap.Int("NumberOfNodes", resp.Stats.NumberOfNodes), + zap.Int("NumberOfURIs", resp.Stats.NumberOfURIs), + zap.Float64("OwnRuntime", resp.Stats.OwnRuntime), + zap.Float64("RepoRuntime", resp.Stats.RepoRuntime), + ) os.Exit(1) } }() @@ -61,11 +71,11 @@ func RunServerSocketAndWebServer( chanErr := make(chan error) if address != "" { - log.Notice("starting socketserver on: ", address) + Log.Info("starting socketserver", zap.String("address", address)) go runSocketServer(r, address, chanErr) } if webserverAddress != "" { - log.Notice("starting webserver on: ", webserverAddress) + Log.Info("starting webserver", zap.String("webserverAddress", webserverAddress)) go runWebserver(r, webserverAddress, webserverPath, chanErr) } return <-chanErr @@ -91,24 +101,26 @@ func runSocketServer( // listen on socket ln, errListen := net.Listen("tcp", address) if errListen != nil { - errListenSocket := errors.New("RunSocketServer: could not start the on \"" + address + "\" - error: " + fmt.Sprint(errListen)) - log.Error(errListenSocket) - chanErr <- errListenSocket + Log.Error("runSocketServer: could not start", + zap.String("address", address), + zap.Error(errListen), + ) + chanErr <- errors.New("runSocketServer: could not start the on \"" + address + "\" - error: " + fmt.Sprint(errListen)) return } - log.Record("RunSocketServer: started to listen on " + address) + Log.Info("runSocketServer: started listening", zap.String("address", address)) for { // this blocks until connection or error conn, err := ln.Accept() if err != nil { - log.Error("RunSocketServer: could not accept connection" + fmt.Sprint(err)) + Log.Error("runSocketServer: could not accept connection", zap.Error(err)) continue } - log.Debug("new connection") + // a goroutine handles conn so that the loop can accept other connections go func() { - log.Debug("accepted connection") + Log.Debug("accepted connection", zap.String("source", conn.RemoteAddr().String())) s.handleConnection(conn) conn.Close() // log.Debug("connection closed") diff --git a/server/socketserver.go b/server/socketserver.go index f667e3e..3adacf1 100644 --- a/server/socketserver.go +++ b/server/socketserver.go @@ -7,7 +7,9 @@ import ( "strconv" "strings" - "github.com/foomo/contentserver/log" + "go.uber.org/zap" + + . "github.com/foomo/contentserver/logger" "github.com/foomo/contentserver/repo" "github.com/foomo/contentserver/responses" "github.com/foomo/contentserver/status" @@ -39,13 +41,10 @@ func extractHandlerAndJSONLentgh(header string) (handler Handler, jsonLength int } func (s *socketServer) execute(handler Handler, jsonBytes []byte) (reply []byte) { - if log.SelectedLevel == log.LevelDebug { - log.Debug(" incoming json buffer of length: ", len(jsonBytes)) - // log.Debug(" incoming json buffer:", string(jsonBytes)) - } + Log.Debug("incoming json buffer", zap.Int("length", len(jsonBytes))) reply, handlingError := handleRequest(s.repo, handler, jsonBytes, s.metrics) if handlingError != nil { - log.Error("socketServer.execute handlingError :", handlingError) + Log.Error("socketServer.execute failed", zap.Error(handlingError)) } return reply } @@ -53,22 +52,24 @@ func (s *socketServer) execute(handler Handler, jsonBytes []byte) (reply []byte) func (s *socketServer) writeResponse(conn net.Conn, reply []byte) { headerBytes := []byte(strconv.Itoa(len(reply))) reply = append(headerBytes, reply...) - log.Debug(" replying: " + string(reply)) + Log.Debug("replying", zap.String("reply", string(reply))) n, writeError := conn.Write(reply) if writeError != nil { - log.Error("socketServer.writeResponse: could not write my reply: " + fmt.Sprint(writeError)) + Log.Error("socketServer.writeResponse: could not write reply", zap.Error(writeError)) return } if n < len(reply) { - log.Error(fmt.Sprintf("socketServer.writeResponse: write too short %q instead of %q", n, len(reply))) + Log.Error("socketServer.writeResponse: write too short", + zap.Int("got", n), + zap.Int("expected", len(reply)), + ) return } - log.Debug(" replied. waiting for next request on open connection") - + Log.Debug("replied. waiting for next request on open connection") } func (s *socketServer) handleConnection(conn net.Conn) { - log.Debug("socketServer.handleConnection") + Log.Debug("socketServer.handleConnection") var ( headerBuffer [1]byte @@ -81,7 +82,7 @@ func (s *socketServer) handleConnection(conn net.Conn) { // let us read with 1 byte steps on conn until we find "{" _, readErr := conn.Read(headerBuffer[0:]) if readErr != nil { - log.Debug(" looks like the client closed the connection: ", readErr) + Log.Debug("looks like the client closed the connection", zap.Error(readErr)) return } // read next byte @@ -92,16 +93,16 @@ func (s *socketServer) handleConnection(conn net.Conn) { // reset header header = "" if headerErr != nil { - log.Error("invalid request could not read header", headerErr) + Log.Error("invalid request could not read header", zap.Error(headerErr)) encodedErr, encodingErr := encodeReply(responses.NewError(4, "invalid header "+headerErr.Error())) if encodingErr == nil { s.writeResponse(conn, encodedErr) } else { - log.Error("could not respond to invalid request", encodingErr) + Log.Error("could not respond to invalid request", zap.Error(encodingErr)) } return } - log.Debug(fmt.Sprintf(" found json with %d bytes", jsonLength)) + Log.Debug("found json", zap.Int("length", jsonLength)) if jsonLength > 0 { var ( @@ -120,22 +121,24 @@ func (s *socketServer) handleConnection(conn net.Conn) { if jsonReadErr != nil { //@fixme we need to force a read timeout (SetReadDeadline?), if expected jsonLength is lower than really sent bytes (e.g. if client implements protocol wrong) //@todo should we check for io.EOF here - log.Error(" could not read json - giving up with this client connection" + fmt.Sprint(jsonReadErr)) + Log.Error("could not read json - giving up with this client connection", zap.Error(jsonReadErr)) return } jsonLengthCurrent += readLength - log.Debug(fmt.Sprintf(" read so far %d of %d bytes in read cycle %d", jsonLengthCurrent, jsonLength, readRound)) + Log.Debug("read cycle status", + zap.Int("jsonLengthCurrent", jsonLengthCurrent), + zap.Int("jsonLength", jsonLength), + zap.Int("readRound", readRound), + ) } - if log.SelectedLevel == log.LevelDebug { - log.Debug(" read json, length: ", len(jsonBytes)) - // log.Debug(" read json: " + string(jsonBytes)) - } + Log.Debug("read json", zap.Int("length", len(jsonBytes))) + s.writeResponse(conn, s.execute(handler, jsonBytes)) // note: connection remains open continue } - log.Error("can not read empty json") + Log.Error("can not read empty json") return } // adding to header byte by byte diff --git a/server/webserver.go b/server/webserver.go index 4c5470a..61481b4 100644 --- a/server/webserver.go +++ b/server/webserver.go @@ -5,9 +5,10 @@ import ( "net/http" "strings" - "github.com/foomo/contentserver/log" "github.com/foomo/contentserver/status" + "go.uber.org/zap" + . "github.com/foomo/contentserver/logger" "github.com/foomo/contentserver/repo" ) @@ -44,6 +45,6 @@ func (s *webServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { } _, err := w.Write(reply) if err != nil { - log.Error("failed to write webServer reply: ", err) + Log.Error("failed to write webServer reply", zap.Error(err)) } } diff --git a/status/healthz.go b/status/healthz.go index dba2936..077aa11 100644 --- a/status/healthz.go +++ b/status/healthz.go @@ -4,15 +4,18 @@ import ( "fmt" "net/http" - "github.com/foomo/contentserver/log" + . "github.com/foomo/contentserver/logger" jsoniter "github.com/json-iterator/go" + "go.uber.org/zap" ) -var json = jsoniter.ConfigCompatibleWithStandardLibrary +var ( + json = jsoniter.ConfigCompatibleWithStandardLibrary +) func RunHealthzHandlerListener(address string, serviceName string) { - log.Notice(fmt.Sprintf("starting healthz handler on '%s'" + address)) - log.Error(http.ListenAndServe(address, HealthzHandler(serviceName))) + Log.Info(fmt.Sprintf("starting healthz handler on '%s'" + address)) + Log.Error("healthz server failed", zap.Error(http.ListenAndServe(address, HealthzHandler(serviceName)))) } func HealthzHandler(serviceName string) http.Handler { @@ -26,7 +29,7 @@ func HealthzHandler(serviceName string) http.Handler { h.Handle("/healthz", http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { _, err := w.Write(status) if err != nil { - log.Error("failed to write healthz status: ", err) + Log.Error("failed to write healthz status", zap.Error(err)) } })) From 71403194e2808ddcbd9ba9e1b26558f3bcd80a3c Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Thu, 23 May 2019 14:23:26 +0200 Subject: [PATCH 47/79] removed log level flag --- contentserver.go | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/contentserver.go b/contentserver.go index abd3d6c..55706fe 100644 --- a/contentserver.go +++ b/contentserver.go @@ -7,7 +7,6 @@ import ( _ "net/http/pprof" "os" "runtime/debug" - "strings" "time" "github.com/apex/log" @@ -43,22 +42,6 @@ var ( // debugging / profiling flagFreeOSMem = flag.Int("free-os-mem", 0, "free OS mem every X minutes") flagHeapDump = flag.Int("heap-dump", 0, "dump heap every X minutes") - - logLevelOptions = []string{ - logLevelError, - logLevelRecord, - logLevelWarning, - logLevelNotice, - logLevelDebug, - } - logLevel = flag.String( - "log-level", - logLevelRecord, - fmt.Sprintf( - "one of %s", - strings.Join(logLevelOptions, ", "), - ), - ) ) func exitUsage(code int) { From 7c29ec73e4ff57b187cc02a4907f12ebeed38ee0 Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Thu, 23 May 2019 14:23:42 +0200 Subject: [PATCH 48/79] updated Dockerfile to log json in production --- Dockerfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Dockerfile b/Dockerfile index 5d7aa1f..ca8d1c6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -21,6 +21,7 @@ FROM alpine ENV CONTENT_SERVER_LOG_LEVEL=error ENV CONTENT_SERVER_ADDR=0.0.0.0:80 ENV CONTENT_SERVER_VAR_DIR=/var/lib/contentserver +ENV LOG_JSON=1 RUN apk add --update --no-cache ca-certificates curl bash && rm -rf /var/cache/apk/* From 79d828bb2312ad9a7a71fd4e173953d3d30c95d3 Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Thu, 23 May 2019 15:29:07 +0200 Subject: [PATCH 49/79] implemented stefans feedback on prometheus metrics --- server/handlerequest.go | 21 ++++++++------------- server/server.go | 4 +++- server/socketserver.go | 5 ++--- server/webserver.go | 13 +++++-------- status/metrics.go | 24 ++++++++++++------------ 5 files changed, 30 insertions(+), 37 deletions(-) diff --git a/server/handlerequest.go b/server/handlerequest.go index bf24297..49cfbb2 100644 --- a/server/handlerequest.go +++ b/server/handlerequest.go @@ -13,7 +13,7 @@ import ( "github.com/prometheus/client_golang/prometheus" ) -func handleRequest(r *repo.Repo, handler Handler, jsonBytes []byte, metrics *status.Metrics) (replyBytes []byte, err error) { +func handleRequest(r *repo.Repo, handler Handler, jsonBytes []byte, source string) (replyBytes []byte, err error) { var ( reply interface{} @@ -36,35 +36,30 @@ func handleRequest(r *repo.Repo, handler Handler, jsonBytes []byte, metrics *sta processIfJSONIsOk(json.Unmarshal(jsonBytes, &getURIRequest), func() { reply = r.GetURIs(getURIRequest.Dimension, getURIRequest.IDs) }) - addMetrics(metrics, HandlerGetURIs, start, jsonErr, apiErr) case HandlerGetContent: contentRequest := &requests.Content{} processIfJSONIsOk(json.Unmarshal(jsonBytes, &contentRequest), func() { reply, apiErr = r.GetContent(contentRequest) }) - addMetrics(metrics, HandlerGetContent, start, jsonErr, apiErr) case HandlerGetNodes: nodesRequest := &requests.Nodes{} processIfJSONIsOk(json.Unmarshal(jsonBytes, &nodesRequest), func() { reply = r.GetNodes(nodesRequest) }) - addMetrics(metrics, HandlerGetNodes, start, jsonErr, apiErr) case HandlerUpdate: updateRequest := &requests.Update{} processIfJSONIsOk(json.Unmarshal(jsonBytes, &updateRequest), func() { reply = r.Update() }) - addMetrics(metrics, HandlerUpdate, start, jsonErr, apiErr) case HandlerGetRepo: repoRequest := &requests.Repo{} processIfJSONIsOk(json.Unmarshal(jsonBytes, &repoRequest), func() { reply = r.GetRepo() }) - addMetrics(metrics, HandlerGetRepo, start, jsonErr, apiErr) default: reply = responses.NewError(1, "unknown handler: "+string(handler)) - addMetrics(metrics, "default", start, jsonErr, apiErr) } + addMetrics(metrics, handler, start, jsonErr, apiErr, source) // error handling if jsonErr != nil { @@ -78,7 +73,7 @@ func handleRequest(r *repo.Repo, handler Handler, jsonBytes []byte, metrics *sta return encodeReply(reply) } -func addMetrics(metrics *status.Metrics, handlerName Handler, start time.Time, errJSON error, errAPI error) { +func addMetrics(metrics *status.Metrics, handlerName Handler, start time.Time, errJSON error, errAPI error, source string) { var ( duration = time.Since(start) @@ -91,22 +86,22 @@ func addMetrics(metrics *status.Metrics, handlerName Handler, start time.Time, e metrics.ServiceRequestCounter.With(prometheus.Labels{ status.MetricLabelHandler: string(handlerName), status.MetricLabelStatus: s, + status.MetricLabelSource: source, }).Inc() metrics.ServiceRequestDuration.With(prometheus.Labels{ status.MetricLabelHandler: string(handlerName), status.MetricLabelStatus: s, - }).Observe(float64(duration.Nanoseconds())) + status.MetricLabelSource: source, + }).Observe(float64(duration.Seconds())) } // encodeReply takes an interface and encodes it as JSON // it returns the resulting JSON and a marshalling error func encodeReply(reply interface{}) (replyBytes []byte, err error) { - - // @TODO: why use marshal indent here??? - replyBytes, err = json.MarshalIndent(map[string]interface{}{ + replyBytes, err = json.Marshal(map[string]interface{}{ "reply": reply, - }, "", " ") + }) if err != nil { Log.Error("could not encode reply", zap.Error(err)) } diff --git a/server/server.go b/server/server.go index 82108e1..6dd97b1 100644 --- a/server/server.go +++ b/server/server.go @@ -9,12 +9,14 @@ import ( . "github.com/foomo/contentserver/logger" "github.com/foomo/contentserver/repo" + "github.com/foomo/contentserver/status" jsoniter "github.com/json-iterator/go" "go.uber.org/zap" ) var ( - json = jsoniter.ConfigCompatibleWithStandardLibrary + json = jsoniter.ConfigCompatibleWithStandardLibrary + metrics = status.NewMetrics() ) // Handler type diff --git a/server/socketserver.go b/server/socketserver.go index 3adacf1..e639395 100644 --- a/server/socketserver.go +++ b/server/socketserver.go @@ -23,8 +23,7 @@ type socketServer struct { // newSocketServer returns a shiny new socket server func newSocketServer(repo *repo.Repo) *socketServer { return &socketServer{ - repo: repo, - metrics: status.NewMetrics("socketserver"), + repo: repo, } } @@ -42,7 +41,7 @@ func extractHandlerAndJSONLentgh(header string) (handler Handler, jsonLength int func (s *socketServer) execute(handler Handler, jsonBytes []byte) (reply []byte) { Log.Debug("incoming json buffer", zap.Int("length", len(jsonBytes))) - reply, handlingError := handleRequest(s.repo, handler, jsonBytes, s.metrics) + reply, handlingError := handleRequest(s.repo, handler, jsonBytes, "socketserver") if handlingError != nil { Log.Error("socketServer.execute failed", zap.Error(handlingError)) } diff --git a/server/webserver.go b/server/webserver.go index 61481b4..e12b7f1 100644 --- a/server/webserver.go +++ b/server/webserver.go @@ -5,7 +5,6 @@ import ( "net/http" "strings" - "github.com/foomo/contentserver/status" "go.uber.org/zap" . "github.com/foomo/contentserver/logger" @@ -13,17 +12,15 @@ import ( ) type webServer struct { - path string - r *repo.Repo - metrics *status.Metrics + path string + r *repo.Repo } // NewWebServer returns a shiny new web server func NewWebServer(path string, r *repo.Repo) http.Handler { return &webServer{ - path: path, - r: r, - metrics: status.NewMetrics("webserver"), + path: path, + r: r, } } @@ -38,7 +35,7 @@ func (s *webServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { http.Error(w, "failed to read incoming request", http.StatusBadRequest) return } - reply, errReply := handleRequest(s.r, Handler(strings.TrimPrefix(r.URL.Path, s.path+"/")), jsonBytes, s.metrics) + reply, errReply := handleRequest(s.r, Handler(strings.TrimPrefix(r.URL.Path, s.path+"/")), jsonBytes, "webserver") if errReply != nil { http.Error(w, errReply.Error(), http.StatusInternalServerError) return diff --git a/status/metrics.go b/status/metrics.go index 8ccc82d..c28e5c2 100644 --- a/status/metrics.go +++ b/status/metrics.go @@ -7,6 +7,8 @@ import ( const ( MetricLabelHandler = "handler" MetricLabelStatus = "status" + MetricLabelSource = "source" + namespace = "contentserver" ) type Metrics struct { @@ -14,32 +16,30 @@ type Metrics struct { ServiceRequestDuration *prometheus.SummaryVec // count the duration of requests for each service function } -func NewMetrics(namespace string) *Metrics { +func NewMetrics() *Metrics { return &Metrics{ - ServiceRequestCounter: serviceRequestCounter("api", namespace), - ServiceRequestDuration: serviceRequestDuration("api", namespace), + ServiceRequestCounter: serviceRequestCounter(), + ServiceRequestDuration: serviceRequestDuration(), } } -func serviceRequestCounter(subsystem, namespace string) *prometheus.CounterVec { +func serviceRequestCounter() *prometheus.CounterVec { vec := prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: namespace, - Subsystem: subsystem, - Name: "count_service_requests", + Name: "service_request_count", Help: "count of requests per func", - }, []string{MetricLabelHandler, MetricLabelStatus}) + }, []string{MetricLabelHandler, MetricLabelStatus, MetricLabelSource}) prometheus.MustRegister(vec) return vec } -func serviceRequestDuration(subsystem, namespace string) *prometheus.SummaryVec { +func serviceRequestDuration() *prometheus.SummaryVec { vec := prometheus.NewSummaryVec(prometheus.SummaryOpts{ Namespace: namespace, - Subsystem: subsystem, - Name: "time_nanoseconds", - Help: "nanoseconds to unmarshal requests, execute a service function and marshal its reponses", - }, []string{MetricLabelHandler, MetricLabelStatus}) + Name: "service_request_duration_seconds", + Help: "seconds to unmarshal requests, execute a service function and marshal its reponses", + }, []string{MetricLabelHandler, MetricLabelStatus, MetricLabelSource}) prometheus.MustRegister(vec) return vec } From 647853292bd6fb1990b4c3e160bea9a6a8337995 Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Thu, 23 May 2019 16:23:24 +0200 Subject: [PATCH 50/79] added new metrics --- Makefile | 2 +- repo/loader.go | 6 ++- repo/repo.go | 11 ++++- server/handlerequest.go | 19 ++------ server/server.go | 4 +- server/socketserver.go | 4 ++ status/metrics.go | 102 +++++++++++++++++++++++++++++++++++----- 7 files changed, 115 insertions(+), 33 deletions(-) diff --git a/Makefile b/Makefile index 1e079d9..935a186 100644 --- a/Makefile +++ b/Makefile @@ -56,7 +56,7 @@ run-testserver: bin/testserver -json-file var/cse-globus-stage-b-with-main-section.json run-contentserver: - contentserver -var-dir var -webserver-address :9191 -address :9999 -log-level notice http://127.0.0.1:1234 + contentserver -var-dir var -webserver-address :9191 -address :9999 http://127.0.0.1:1234 clean-var: rm var/contentserver-repo-2019* diff --git a/repo/loader.go b/repo/loader.go index 4d7f1c8..c8aebe4 100644 --- a/repo/loader.go +++ b/repo/loader.go @@ -9,6 +9,7 @@ import ( "github.com/foomo/contentserver/content" . "github.com/foomo/contentserver/logger" + "github.com/foomo/contentserver/status" jsoniter "github.com/json-iterator/go" "go.uber.org/zap" ) @@ -172,8 +173,9 @@ func (repo *Repo) tryUpdate() (repoRuntime int64, jsonBytes []byte, err error) { ur := <-c return ur.repoRuntime, ur.jsonBytes, ur.err default: - Log.Info("update request ignored, queue is full") - return 0, nil, errors.New("queue full") + Log.Info("update request rejected, queue is full") + status.M.UpdatesRejectedCounter.WithLabelValues().Inc() + return 0, nil, errors.New("update rejected: queue full") } } diff --git a/repo/repo.go b/repo/repo.go index 85a0ed3..c676d41 100644 --- a/repo/repo.go +++ b/repo/repo.go @@ -6,6 +6,8 @@ import ( "strings" "time" + "github.com/foomo/contentserver/status" + "github.com/foomo/contentserver/content" . "github.com/foomo/contentserver/logger" "github.com/foomo/contentserver/requests" @@ -62,13 +64,20 @@ func NewRepo(server string, varDir string) *Repo { repoRuntime, jsonBytes, errUpdate := repo.update() + if errUpdate != nil { + status.M.UpdatesFailedCounter.WithLabelValues(errUpdate.Error()).Inc() + } + resChan <- updateResponse{ repoRuntime: repoRuntime, jsonBytes: jsonBytes, err: errUpdate, } - Log.Info("update completed", zap.Duration("duration", time.Since(start))) + duration := time.Since(start) + Log.Info("update completed", zap.Duration("duration", duration)) + status.M.UpdatesCompletedCounter.WithLabelValues().Inc() + status.M.UpdateDuration.WithLabelValues().Observe(duration.Seconds()) } } }() diff --git a/server/handlerequest.go b/server/handlerequest.go index 49cfbb2..e0f4c86 100644 --- a/server/handlerequest.go +++ b/server/handlerequest.go @@ -10,7 +10,6 @@ import ( "github.com/foomo/contentserver/requests" "github.com/foomo/contentserver/responses" "github.com/foomo/contentserver/status" - "github.com/prometheus/client_golang/prometheus" ) func handleRequest(r *repo.Repo, handler Handler, jsonBytes []byte, source string) (replyBytes []byte, err error) { @@ -28,6 +27,7 @@ func handleRequest(r *repo.Repo, handler Handler, jsonBytes []byte, source strin processingFunc() } ) + status.M.ContentRequestCounter.WithLabelValues(source).Inc() // handle and process switch handler { @@ -59,7 +59,7 @@ func handleRequest(r *repo.Repo, handler Handler, jsonBytes []byte, source strin default: reply = responses.NewError(1, "unknown handler: "+string(handler)) } - addMetrics(metrics, handler, start, jsonErr, apiErr, source) + addMetrics(handler, start, jsonErr, apiErr, source) // error handling if jsonErr != nil { @@ -73,7 +73,7 @@ func handleRequest(r *repo.Repo, handler Handler, jsonBytes []byte, source strin return encodeReply(reply) } -func addMetrics(metrics *status.Metrics, handlerName Handler, start time.Time, errJSON error, errAPI error, source string) { +func addMetrics(handlerName Handler, start time.Time, errJSON error, errAPI error, source string) { var ( duration = time.Since(start) @@ -83,17 +83,8 @@ func addMetrics(metrics *status.Metrics, handlerName Handler, start time.Time, e s = "failed" } - metrics.ServiceRequestCounter.With(prometheus.Labels{ - status.MetricLabelHandler: string(handlerName), - status.MetricLabelStatus: s, - status.MetricLabelSource: source, - }).Inc() - - metrics.ServiceRequestDuration.With(prometheus.Labels{ - status.MetricLabelHandler: string(handlerName), - status.MetricLabelStatus: s, - status.MetricLabelSource: source, - }).Observe(float64(duration.Seconds())) + status.M.ServiceRequestCounter.WithLabelValues(string(handlerName), s, source).Inc() + status.M.ServiceRequestDuration.WithLabelValues(string(handlerName), s, source).Observe(float64(duration.Seconds())) } // encodeReply takes an interface and encodes it as JSON diff --git a/server/server.go b/server/server.go index 6dd97b1..82108e1 100644 --- a/server/server.go +++ b/server/server.go @@ -9,14 +9,12 @@ import ( . "github.com/foomo/contentserver/logger" "github.com/foomo/contentserver/repo" - "github.com/foomo/contentserver/status" jsoniter "github.com/json-iterator/go" "go.uber.org/zap" ) var ( - json = jsoniter.ConfigCompatibleWithStandardLibrary - metrics = status.NewMetrics() + json = jsoniter.ConfigCompatibleWithStandardLibrary ) // Handler type diff --git a/server/socketserver.go b/server/socketserver.go index e639395..5d2f34f 100644 --- a/server/socketserver.go +++ b/server/socketserver.go @@ -69,6 +69,7 @@ func (s *socketServer) writeResponse(conn net.Conn, reply []byte) { func (s *socketServer) handleConnection(conn net.Conn) { Log.Debug("socketServer.handleConnection") + status.M.NumSocketsGauge.WithLabelValues(conn.RemoteAddr().String()).Inc() var ( headerBuffer [1]byte @@ -82,6 +83,7 @@ func (s *socketServer) handleConnection(conn net.Conn) { _, readErr := conn.Read(headerBuffer[0:]) if readErr != nil { Log.Debug("looks like the client closed the connection", zap.Error(readErr)) + status.M.NumSocketsGauge.WithLabelValues(conn.RemoteAddr().String()).Dec() return } // read next byte @@ -121,6 +123,7 @@ func (s *socketServer) handleConnection(conn net.Conn) { //@fixme we need to force a read timeout (SetReadDeadline?), if expected jsonLength is lower than really sent bytes (e.g. if client implements protocol wrong) //@todo should we check for io.EOF here Log.Error("could not read json - giving up with this client connection", zap.Error(jsonReadErr)) + status.M.NumSocketsGauge.WithLabelValues(conn.RemoteAddr().String()).Dec() return } jsonLengthCurrent += readLength @@ -138,6 +141,7 @@ func (s *socketServer) handleConnection(conn net.Conn) { continue } Log.Error("can not read empty json") + status.M.NumSocketsGauge.WithLabelValues(conn.RemoteAddr().String()).Dec() return } // adding to header byte by byte diff --git a/status/metrics.go b/status/metrics.go index c28e5c2..0bda46e 100644 --- a/status/metrics.go +++ b/status/metrics.go @@ -4,22 +4,40 @@ import ( "github.com/prometheus/client_golang/prometheus" ) +// M is the Metrics instance +var M = NewMetrics() + const ( - MetricLabelHandler = "handler" - MetricLabelStatus = "status" - MetricLabelSource = "source" - namespace = "contentserver" + namespace = "contentserver" + + metricLabelHandler = "handler" + metricLabelStatus = "status" + metricLabelSource = "source" + metricLabelRemote = "remote" + metricLabelError = "error" ) type Metrics struct { - ServiceRequestCounter *prometheus.CounterVec // count the number of requests for each service function - ServiceRequestDuration *prometheus.SummaryVec // count the duration of requests for each service function + ServiceRequestCounter *prometheus.CounterVec // count the number of requests for each service function + ServiceRequestDuration *prometheus.SummaryVec // observe the duration of requests for each service function + UpdatesRejectedCounter *prometheus.CounterVec // count the number of completed updates + UpdatesCompletedCounter *prometheus.CounterVec // count the number of rejected updates + UpdatesFailedCounter *prometheus.CounterVec // count the number of updates that had an error + UpdateDuration *prometheus.SummaryVec // observe the duration of each repo.update() call + ContentRequestCounter *prometheus.CounterVec // count the total number of content requests + NumSocketsGauge *prometheus.GaugeVec // keep track of the total number of open sockets } func NewMetrics() *Metrics { return &Metrics{ - ServiceRequestCounter: serviceRequestCounter(), - ServiceRequestDuration: serviceRequestDuration(), + ServiceRequestCounter: serviceRequestCounter(), + ServiceRequestDuration: serviceRequestDuration(), + UpdatesRejectedCounter: updatesRejectedCounter(), + UpdatesCompletedCounter: updatesCompletedCounter(), + UpdatesFailedCounter: updatesFailedCounter(), + UpdateDuration: updateDuration(), + ContentRequestCounter: contentRequestCounter(), + NumSocketsGauge: numSocketsGauge(), } } @@ -28,8 +46,8 @@ func serviceRequestCounter() *prometheus.CounterVec { prometheus.CounterOpts{ Namespace: namespace, Name: "service_request_count", - Help: "count of requests per func", - }, []string{MetricLabelHandler, MetricLabelStatus, MetricLabelSource}) + Help: "Count of requests for each handler", + }, []string{metricLabelHandler, metricLabelStatus, metricLabelSource}) prometheus.MustRegister(vec) return vec } @@ -38,8 +56,68 @@ func serviceRequestDuration() *prometheus.SummaryVec { vec := prometheus.NewSummaryVec(prometheus.SummaryOpts{ Namespace: namespace, Name: "service_request_duration_seconds", - Help: "seconds to unmarshal requests, execute a service function and marshal its reponses", - }, []string{MetricLabelHandler, MetricLabelStatus, MetricLabelSource}) + Help: "Seconds to unmarshal requests, execute a service function and marshal its reponses", + }, []string{metricLabelHandler, metricLabelStatus, metricLabelSource}) + prometheus.MustRegister(vec) + return vec +} + +func updatesRejectedCounter() *prometheus.CounterVec { + vec := prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Name: "updates_rejected_count", + Help: "Number of updates that were rejected because the queue was full", + }, []string{}) + prometheus.MustRegister(vec) + return vec +} + +func updatesCompletedCounter() *prometheus.CounterVec { + vec := prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Name: "updates_completed_count", + Help: "Number of updates that were successfully completed", + }, []string{}) + prometheus.MustRegister(vec) + return vec +} + +func updatesFailedCounter() *prometheus.CounterVec { + vec := prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Name: "updates_failed_count", + Help: "Number of updates that failed due to an error", + }, []string{metricLabelError}) + prometheus.MustRegister(vec) + return vec +} + +func updateDuration() *prometheus.SummaryVec { + vec := prometheus.NewSummaryVec(prometheus.SummaryOpts{ + Namespace: namespace, + Name: "update_duration_seconds", + Help: "Duration in seconds for each successful repo.update() call", + }, []string{}) + prometheus.MustRegister(vec) + return vec +} + +func numSocketsGauge() *prometheus.GaugeVec { + vec := prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: namespace, + Name: "num_sockets_total", + Help: "Total number of currently open socket connections", + }, []string{metricLabelRemote}) + prometheus.MustRegister(vec) + return vec +} + +func contentRequestCounter() *prometheus.CounterVec { + vec := prometheus.NewCounterVec(prometheus.CounterOpts{ + Namespace: namespace, + Name: "content_request_count", + Help: "Number of requests for content", + }, []string{metricLabelSource}) prometheus.MustRegister(vec) return vec } From 33364e3af8637d09e62463d24a9287bcb0cafea2 Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Thu, 23 May 2019 16:33:01 +0200 Subject: [PATCH 51/79] added metric constructors --- status/metrics.go | 150 ++++++++++++++++++++++------------------------ 1 file changed, 70 insertions(+), 80 deletions(-) diff --git a/status/metrics.go b/status/metrics.go index 0bda46e..ed97324 100644 --- a/status/metrics.go +++ b/status/metrics.go @@ -5,7 +5,7 @@ import ( ) // M is the Metrics instance -var M = NewMetrics() +var M = newMetrics() const ( namespace = "contentserver" @@ -17,6 +17,7 @@ const ( metricLabelError = "error" ) +// Metrics is the structure that holds all prometheus metrics type Metrics struct { ServiceRequestCounter *prometheus.CounterVec // count the number of requests for each service function ServiceRequestDuration *prometheus.SummaryVec // observe the duration of requests for each service function @@ -28,96 +29,85 @@ type Metrics struct { NumSocketsGauge *prometheus.GaugeVec // keep track of the total number of open sockets } -func NewMetrics() *Metrics { +// newMetrics can be used to instantiate a metrics instance +// since this function will also register each metric and metrics should only be registered once +// it is private +// the package exposes the initialized Metrics instance as the variable M. +func newMetrics() *Metrics { return &Metrics{ - ServiceRequestCounter: serviceRequestCounter(), - ServiceRequestDuration: serviceRequestDuration(), - UpdatesRejectedCounter: updatesRejectedCounter(), - UpdatesCompletedCounter: updatesCompletedCounter(), - UpdatesFailedCounter: updatesFailedCounter(), - UpdateDuration: updateDuration(), - ContentRequestCounter: contentRequestCounter(), - NumSocketsGauge: numSocketsGauge(), + ServiceRequestCounter: newCounterVec( + "service_request_count", + "Count of requests for each handler", + metricLabelHandler, metricLabelStatus, metricLabelSource, + ), + ServiceRequestDuration: newSummaryVec( + "service_request_duration_seconds", + "Seconds to unmarshal requests, execute a service function and marshal its reponses", + metricLabelHandler, metricLabelStatus, metricLabelSource, + ), + UpdatesRejectedCounter: newCounterVec( + "updates_rejected_count", + "Number of updates that were rejected because the queue was full", + ), + UpdatesCompletedCounter: newCounterVec( + "updates_completed_count", + "Number of updates that were successfully completed", + ), + UpdatesFailedCounter: newCounterVec( + "updates_failed_count", + "Number of updates that failed due to an error", + metricLabelError, + ), + UpdateDuration: newSummaryVec( + "update_duration_seconds", + "Duration in seconds for each successful repo.update() call", + ), + ContentRequestCounter: newCounterVec( + "num_sockets_total", + "Total number of currently open socket connections", + metricLabelRemote, + ), + NumSocketsGauge: newGaugeVec( + "content_request_count", + "Number of requests for content", + metricLabelSource, + ), } } -func serviceRequestCounter() *prometheus.CounterVec { +/* + * Metric constructors + */ + +func newSummaryVec(name, help string, labels ...string) *prometheus.SummaryVec { + vec := prometheus.NewSummaryVec( + prometheus.SummaryOpts{ + Namespace: namespace, + Name: name, + Help: help, + }, labels) + prometheus.MustRegister(vec) + return vec +} + +func newCounterVec(name, help string, labels ...string) *prometheus.CounterVec { vec := prometheus.NewCounterVec( prometheus.CounterOpts{ Namespace: namespace, - Name: "service_request_count", - Help: "Count of requests for each handler", - }, []string{metricLabelHandler, metricLabelStatus, metricLabelSource}) + Name: name, + Help: help, + }, labels) prometheus.MustRegister(vec) return vec } -func serviceRequestDuration() *prometheus.SummaryVec { - vec := prometheus.NewSummaryVec(prometheus.SummaryOpts{ - Namespace: namespace, - Name: "service_request_duration_seconds", - Help: "Seconds to unmarshal requests, execute a service function and marshal its reponses", - }, []string{metricLabelHandler, metricLabelStatus, metricLabelSource}) - prometheus.MustRegister(vec) - return vec -} - -func updatesRejectedCounter() *prometheus.CounterVec { - vec := prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: namespace, - Name: "updates_rejected_count", - Help: "Number of updates that were rejected because the queue was full", - }, []string{}) - prometheus.MustRegister(vec) - return vec -} - -func updatesCompletedCounter() *prometheus.CounterVec { - vec := prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: namespace, - Name: "updates_completed_count", - Help: "Number of updates that were successfully completed", - }, []string{}) - prometheus.MustRegister(vec) - return vec -} - -func updatesFailedCounter() *prometheus.CounterVec { - vec := prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: namespace, - Name: "updates_failed_count", - Help: "Number of updates that failed due to an error", - }, []string{metricLabelError}) - prometheus.MustRegister(vec) - return vec -} - -func updateDuration() *prometheus.SummaryVec { - vec := prometheus.NewSummaryVec(prometheus.SummaryOpts{ - Namespace: namespace, - Name: "update_duration_seconds", - Help: "Duration in seconds for each successful repo.update() call", - }, []string{}) - prometheus.MustRegister(vec) - return vec -} - -func numSocketsGauge() *prometheus.GaugeVec { - vec := prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Namespace: namespace, - Name: "num_sockets_total", - Help: "Total number of currently open socket connections", - }, []string{metricLabelRemote}) - prometheus.MustRegister(vec) - return vec -} - -func contentRequestCounter() *prometheus.CounterVec { - vec := prometheus.NewCounterVec(prometheus.CounterOpts{ - Namespace: namespace, - Name: "content_request_count", - Help: "Number of requests for content", - }, []string{metricLabelSource}) +func newGaugeVec(name, help string, labels ...string) *prometheus.GaugeVec { + vec := prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Name: name, + Help: help, + }, labels) prometheus.MustRegister(vec) return vec } From 5cff674940924086c543bea81213c6aad25b51ae Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Thu, 23 May 2019 17:51:20 +0200 Subject: [PATCH 52/79] working on prometheus dashboard --- .gitignore | 1 + Makefile | 3 +++ contentserver.go | 2 +- metrics/prometheus.go | 11 ++++------- prometheus/prometheus.yml | 14 ++++++++++++++ status/metrics.go | 10 +++++----- 6 files changed, 28 insertions(+), 13 deletions(-) create mode 100644 prometheus/prometheus.yml diff --git a/.gitignore b/.gitignore index 8a71575..197a383 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +data *.log *.test cprof-* diff --git a/Makefile b/Makefile index 935a186..c667b88 100644 --- a/Makefile +++ b/Makefile @@ -58,6 +58,9 @@ run-testserver: run-contentserver: contentserver -var-dir var -webserver-address :9191 -address :9999 http://127.0.0.1:1234 +run-prometheus: + prometheus --config.file=prometheus/prometheus.yml + clean-var: rm var/contentserver-repo-2019* diff --git a/contentserver.go b/contentserver.go index 55706fe..ee47bcd 100644 --- a/contentserver.go +++ b/contentserver.go @@ -26,7 +26,7 @@ const ( ServiceName = "Content Server" DefaultHealthzHandlerAddress = ":8080" - DefaultPrometheusListener = ":9200" + DefaultPrometheusListener = "127.0.0.1:9111" ) var ( diff --git a/metrics/prometheus.go b/metrics/prometheus.go index 919b92a..bcf892c 100644 --- a/metrics/prometheus.go +++ b/metrics/prometheus.go @@ -8,17 +8,14 @@ import ( "go.uber.org/zap" ) -func PrometheusHandler() http.Handler { - h := http.NewServeMux() - h.Handle("/metrics", promhttp.Handler()) - return h -} +const metricsRoute = "/metrics" func RunPrometheusHandler(listener string) { Log.Info("starting prometheus handler on", zap.String("address", listener), + zap.String("route", metricsRoute), ) - Log.Error("server failed: ", - zap.Error(http.ListenAndServe(listener, PrometheusHandler())), + Log.Error("prometheus listener failed", + zap.Error(http.ListenAndServe(listener, promhttp.Handler())), ) } diff --git a/prometheus/prometheus.yml b/prometheus/prometheus.yml new file mode 100644 index 0000000..f6accda --- /dev/null +++ b/prometheus/prometheus.yml @@ -0,0 +1,14 @@ +# reference: https://prometheus.io/docs/prometheus/latest/configuration/configuration/ + +global: + scrape_interval: 15s + scrape_timeout: 15s + #evaluation_interval: 15s + +scrape_configs: + - job_name: contentserver + metrics_path: /metrics + scheme: http + static_configs: + - targets: + - 127.0.0.1:9111 diff --git a/status/metrics.go b/status/metrics.go index ed97324..36d69c8 100644 --- a/status/metrics.go +++ b/status/metrics.go @@ -63,15 +63,15 @@ func newMetrics() *Metrics { "Duration in seconds for each successful repo.update() call", ), ContentRequestCounter: newCounterVec( - "num_sockets_total", - "Total number of currently open socket connections", - metricLabelRemote, - ), - NumSocketsGauge: newGaugeVec( "content_request_count", "Number of requests for content", metricLabelSource, ), + NumSocketsGauge: newGaugeVec( + "num_sockets_total", + "Total number of currently open socket connections", + metricLabelRemote, + ), } } From 69dec4160553cfc98f4ceba97db748692240967e Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Fri, 24 May 2019 13:00:28 +0200 Subject: [PATCH 53/79] refactored repo to reuse a bytes.Buffer for updates, to reduce the number of allocations --- Makefile | 3 + repo/history.go | 9 ++- repo/loader.go | 126 ++++++++++++++++++++++++++++----------- repo/repo.go | 80 +++++++++++-------------- testing/client/client.go | 4 +- 5 files changed, 139 insertions(+), 83 deletions(-) diff --git a/Makefile b/Makefile index c667b88..500b18a 100644 --- a/Makefile +++ b/Makefile @@ -58,6 +58,9 @@ run-testserver: run-contentserver: contentserver -var-dir var -webserver-address :9191 -address :9999 http://127.0.0.1:1234 +run-contentserver-freeosmem: + contentserver -var-dir var -webserver-address :9191 -address :9999 -free-os-mem 1 http://127.0.0.1:1234 + run-prometheus: prometheus --config.file=prometheus/prometheus.yml diff --git a/repo/history.go b/repo/history.go index 8d7f579..2b3d67a 100644 --- a/repo/history.go +++ b/repo/history.go @@ -1,8 +1,10 @@ package repo import ( + "bytes" "errors" "fmt" + "io" "io/ioutil" "os" "path" @@ -96,6 +98,9 @@ func (h *history) getCurrentFilename() string { return path.Join(h.varDir, historyRepoJSONPrefix+"current"+historyRepoJSONSuffix) } -func (h *history) getCurrent() (jsonBytes []byte, err error) { - return ioutil.ReadFile(h.getCurrentFilename()) +func (h *history) getCurrent(buf *bytes.Buffer) (err error) { + f, err := os.Open(h.getCurrentFilename()) + defer f.Close() + _, err = io.Copy(buf, f) + return err } diff --git a/repo/loader.go b/repo/loader.go index c8aebe4..ac42e78 100644 --- a/repo/loader.go +++ b/repo/loader.go @@ -3,10 +3,12 @@ package repo import ( "errors" "fmt" - "io/ioutil" + "io" "net/http" "time" + "github.com/mgutz/ansi" + "github.com/foomo/contentserver/content" . "github.com/foomo/contentserver/logger" "github.com/foomo/contentserver/status" @@ -15,17 +17,42 @@ import ( ) var ( - json = jsoniter.ConfigCompatibleWithStandardLibrary + json = jsoniter.ConfigCompatibleWithStandardLibrary + errUpdateRejected = errors.New("update rejected: queue full") ) type updateResponse struct { repoRuntime int64 - jsonBytes []byte err error } func (repo *Repo) updateRoutine() { - for newDimension := range repo.updateChannel { + for { + select { + case resChan := <-repo.updateInProgressChannel: + Log.Info("waiting for update to complete", zap.String("chan", fmt.Sprintf("%p", resChan))) + start := time.Now() + + repoRuntime, errUpdate := repo.update() + if errUpdate != nil { + status.M.UpdatesFailedCounter.WithLabelValues(errUpdate.Error()).Inc() + } + + resChan <- updateResponse{ + repoRuntime: repoRuntime, + err: errUpdate, + } + + duration := time.Since(start) + Log.Info("update completed", zap.Duration("duration", duration), zap.String("chan", fmt.Sprintf("%p", resChan))) + status.M.UpdatesCompletedCounter.WithLabelValues().Inc() + status.M.UpdateDuration.WithLabelValues().Observe(duration.Seconds()) + } + } +} + +func (repo *Repo) dimensionUpdateRoutine() { + for newDimension := range repo.dimensionUpdateChannel { Log.Info("update routine received a new dimension", zap.String("dimension", newDimension.Dimension)) err := repo._updateDimension(newDimension.Dimension, newDimension.Node) @@ -33,18 +60,18 @@ func (repo *Repo) updateRoutine() { if err != nil { Log.Debug("update dimension failed", zap.Error(err)) } - repo.updateDoneChannel <- err + repo.dimensionUpdateDoneChannel <- err } } func (repo *Repo) updateDimension(dimension string, node *content.RepoNode) error { Log.Debug("trying to push dimension into update channel", zap.String("dimension", dimension), zap.String("nodeName", node.Name)) - repo.updateChannel <- &repoDimension{ + repo.dimensionUpdateChannel <- &repoDimension{ Dimension: dimension, Node: node, } Log.Debug("waiting for done signal") - return <-repo.updateDoneChannel + return <-repo.dimensionUpdateDoneChannel } // do not call directly, but only through channel @@ -54,7 +81,7 @@ func (repo *Repo) _updateDimension(dimension string, newNode *content.RepoNode) var ( newDirectory = make(map[string]*content.RepoNode) newURIDirectory = make(map[string]*content.RepoNode) - err = builDirectory(newNode, newDirectory, newURIDirectory) + err = buildDirectory(newNode, newDirectory, newURIDirectory) ) if err != nil { return errors.New("update dimension \"" + dimension + "\" failed when building its directory:: " + err.Error()) @@ -64,22 +91,39 @@ func (repo *Repo) _updateDimension(dimension string, newNode *content.RepoNode) return err } + // --------------------------------------------- + + // collect other dimension in the Directory newRepoDirectory := map[string]*Dimension{} for d, D := range repo.Directory { if d != dimension { newRepoDirectory[d] = D } } + + // add the new dimension newRepoDirectory[dimension] = &Dimension{ Node: newNode, Directory: newDirectory, URIDirectory: newURIDirectory, } repo.Directory = newRepoDirectory + + // --------------------------------------------- + + // @TODO: why not update only the dimension that has changed instead? + // repo.Directory[dimension] = &Dimension{ + // Node: newNode, + // Directory: newDirectory, + // URIDirectory: newURIDirectory, + // } + + // --------------------------------------------- + return nil } -func builDirectory(dirNode *content.RepoNode, directory map[string]*content.RepoNode, uRIDirectory map[string]*content.RepoNode) error { +func buildDirectory(dirNode *content.RepoNode, directory map[string]*content.RepoNode, uRIDirectory map[string]*content.RepoNode) error { // Log.Debug("buildDirectory", zap.String("ID", dirNode.ID)) @@ -94,7 +138,7 @@ func builDirectory(dirNode *content.RepoNode, directory map[string]*content.Repo } uRIDirectory[dirNode.URI] = dirNode for _, childNode := range dirNode.Nodes { - err := builDirectory(childNode, directory, uRIDirectory) + err := buildDirectory(childNode, directory, uRIDirectory) if err != nil { return err } @@ -115,79 +159,93 @@ func wireAliases(directory map[string]*content.RepoNode) error { return nil } -func loadNodesFromJSON(jsonBytes []byte) (nodes map[string]*content.RepoNode, err error) { +func (repo *Repo) loadNodesFromJSON() (nodes map[string]*content.RepoNode, err error) { nodes = make(map[string]*content.RepoNode) - err = json.Unmarshal(jsonBytes, &nodes) + err = json.Unmarshal(repo.jsonBuf.Bytes(), &nodes) return nodes, err } -func (repo *Repo) tryToRestoreCurrent() error { - currentJSONBytes, err := repo.history.getCurrent() +func (repo *Repo) tryToRestoreCurrent() (err error) { + err = repo.history.getCurrent(&repo.jsonBuf) if err != nil { return err } - return repo.loadJSONBytes(currentJSONBytes) + return repo.loadJSONBytes() } -func get(URL string) (data []byte, err error) { +func (repo *Repo) get(URL string) (err error) { response, err := http.Get(URL) if err != nil { - return data, err + return err } defer response.Body.Close() if response.StatusCode != http.StatusOK { - return data, fmt.Errorf("Bad HTTP Response: %q", response.Status) + return fmt.Errorf("Bad HTTP Response: %q", response.Status) } - return ioutil.ReadAll(response.Body) + + Log.Info(ansi.Red + "RESETTING BUFFER" + ansi.Reset) + repo.jsonBuf.Reset() + + Log.Info(ansi.Green + "LOADING DATA INTO BUFFER" + ansi.Reset) + _, err = io.Copy(&repo.jsonBuf, response.Body) + return err } -func (repo *Repo) update() (repoRuntime int64, jsonBytes []byte, err error) { +func (repo *Repo) update() (repoRuntime int64, err error) { startTimeRepo := time.Now().UnixNano() - jsonBytes, err = get(repo.server) + err = repo.get(repo.server) repoRuntime = time.Now().UnixNano() - startTimeRepo if err != nil { // we have no json to load - the repo server did not reply Log.Debug("failed to load json", zap.Error(err)) - return repoRuntime, jsonBytes, err + return repoRuntime, err } - Log.Debug("loading json", zap.String("server", repo.server), zap.Int("length", len(jsonBytes))) - nodes, err := loadNodesFromJSON(jsonBytes) + Log.Debug("loading json", zap.String("server", repo.server), zap.Int("length", len(repo.jsonBuf.Bytes()))) + nodes, err := repo.loadNodesFromJSON() if err != nil { // could not load nodes from json - return repoRuntime, jsonBytes, err + return repoRuntime, err } err = repo.loadNodes(nodes) if err != nil { // repo failed to load nodes - return repoRuntime, jsonBytes, err + return repoRuntime, err } - return repoRuntime, jsonBytes, nil + return repoRuntime, nil } // limit ressources and allow only one update request at once -func (repo *Repo) tryUpdate() (repoRuntime int64, jsonBytes []byte, err error) { +func (repo *Repo) tryUpdate() (repoRuntime int64, err error) { c := make(chan updateResponse) select { case repo.updateInProgressChannel <- c: Log.Info("update request added to queue") ur := <-c - return ur.repoRuntime, ur.jsonBytes, ur.err + return ur.repoRuntime, ur.err default: Log.Info("update request rejected, queue is full") status.M.UpdatesRejectedCounter.WithLabelValues().Inc() - return 0, nil, errors.New("update rejected: queue full") + return 0, errUpdateRejected } } -func (repo *Repo) loadJSONBytes(jsonBytes []byte) error { - nodes, err := loadNodesFromJSON(jsonBytes) +func (repo *Repo) loadJSONBytes() error { + nodes, err := repo.loadNodesFromJSON() if err != nil { - Log.Debug("could not parse json", zap.String("json", string(jsonBytes))) + data := repo.jsonBuf.Bytes() + + if len(data) > 10 { + Log.Debug("could not parse json", + zap.String("jsonStart", string(data[:10])), + zap.String("jsonStart", string(data[len(data)-10:])), + ) + } return err } + err = repo.loadNodes(nodes) if err == nil { - historyErr := repo.history.add(jsonBytes) + historyErr := repo.history.add(repo.jsonBuf.Bytes()) if historyErr != nil { Log.Error("could not add valid json to history", zap.Error(historyErr)) } else { diff --git a/repo/repo.go b/repo/repo.go index c676d41..9acbe89 100644 --- a/repo/repo.go +++ b/repo/repo.go @@ -1,12 +1,14 @@ package repo import ( + "bytes" "errors" "fmt" + "strconv" "strings" "time" - "github.com/foomo/contentserver/status" + "github.com/mgutz/ansi" "github.com/foomo/contentserver/content" . "github.com/foomo/contentserver/logger" @@ -29,10 +31,14 @@ type Repo struct { server string Directory map[string]*Dimension // updateLock sync.Mutex - updateChannel chan *repoDimension - updateDoneChannel chan error + dimensionUpdateChannel chan *repoDimension + dimensionUpdateDoneChannel chan error + history *history updateInProgressChannel chan chan updateResponse + + // jsonBytes []byte + jsonBuf bytes.Buffer } type repoDimension struct { @@ -48,40 +54,17 @@ func NewRepo(server string, varDir string) *Repo { zap.String("varDir", varDir), ) repo := &Repo{ - server: server, - Directory: map[string]*Dimension{}, - history: newHistory(varDir), - updateChannel: make(chan *repoDimension), - updateDoneChannel: make(chan error), - updateInProgressChannel: make(chan chan updateResponse, 1), + server: server, + Directory: map[string]*Dimension{}, + history: newHistory(varDir), + dimensionUpdateChannel: make(chan *repoDimension), + dimensionUpdateDoneChannel: make(chan error), + updateInProgressChannel: make(chan chan updateResponse, 0), } - go func() { - for { - select { - case resChan := <-repo.updateInProgressChannel: - Log.Info("waiting for update to complete") - start := time.Now() - repoRuntime, jsonBytes, errUpdate := repo.update() - - if errUpdate != nil { - status.M.UpdatesFailedCounter.WithLabelValues(errUpdate.Error()).Inc() - } - - resChan <- updateResponse{ - repoRuntime: repoRuntime, - jsonBytes: jsonBytes, - err: errUpdate, - } - - duration := time.Since(start) - Log.Info("update completed", zap.Duration("duration", duration)) - status.M.UpdatesCompletedCounter.WithLabelValues().Inc() - status.M.UpdateDuration.WithLabelValues().Observe(duration.Seconds()) - } - } - }() go repo.updateRoutine() + go repo.dimensionUpdateRoutine() + Log.Info("trying to restore previous state") restoreErr := repo.tryToRestoreCurrent() if restoreErr != nil { @@ -235,30 +218,37 @@ func (repo *Repo) Update() (updateResponse *responses.Update) { floatSeconds := func(nanoSeconds int64) float64 { return float64(float64(nanoSeconds) / float64(1000000000.0)) } - startTime := time.Now().UnixNano() - updateRepotime, jsonBytes, updateErr := repo.tryUpdate() - updateResponse = &responses.Update{} - updateResponse.Stats.RepoRuntime = floatSeconds(updateRepotime) Log.Info("Update triggered") + Log.Info(ansi.Yellow + "BUFFER LENGTH BEFORE tryUpdate(): " + strconv.Itoa(len(repo.jsonBuf.Bytes())) + ansi.Reset) + + startTime := time.Now().UnixNano() + updateRepotime, updateErr := repo.tryUpdate() + updateResponse = &responses.Update{} + updateResponse.Stats.RepoRuntime = floatSeconds(updateRepotime) if updateErr != nil { updateResponse.Success = false updateResponse.Stats.NumberOfNodes = -1 updateResponse.Stats.NumberOfURIs = -1 // let us try to restore the world from a file - Log.Error("could not update repository:" + updateErr.Error()) + Log.Error("could not update repository:", zap.Error(updateErr)) + Log.Info(ansi.Yellow + "BUFFER LENGTH AFTER ERROR: " + strconv.Itoa(len(repo.jsonBuf.Bytes())) + ansi.Reset) updateResponse.ErrorMessage = updateErr.Error() - restoreErr := repo.tryToRestoreCurrent() - if restoreErr != nil { - Log.Error("failed to restore preceding repo version", zap.Error(restoreErr)) - } else { - Log.Info("restored current repo from local history") + + // only try to restore if the update failed during processing + if updateErr != errUpdateRejected { + restoreErr := repo.tryToRestoreCurrent() + if restoreErr != nil { + Log.Error("failed to restore preceding repo version", zap.Error(restoreErr)) + } else { + Log.Info("restored current repo from local history") + } } } else { updateResponse.Success = true // persist the currently loaded one - historyErr := repo.history.add(jsonBytes) + historyErr := repo.history.add(repo.jsonBuf.Bytes()) if historyErr != nil { Log.Warn("could not persist current repo in history", zap.Error(historyErr)) } diff --git a/testing/client/client.go b/testing/client/client.go index 1023dce..4e8b830 100644 --- a/testing/client/client.go +++ b/testing/client/client.go @@ -15,7 +15,7 @@ func main() { log.Fatal(errClient) } - for i := 1; i <= 50; i++ { + for i := 1; i <= 150; i++ { go func(num int) { log.Println("start update") resp, errUpdate := c.Update() @@ -25,7 +25,7 @@ func main() { } log.Println(num, "update done", resp) }(i) - time.Sleep(1 * time.Second) + time.Sleep(2 * time.Second) } log.Println("done") From 4e6eecc673c915111fb8d587ad202d86f8f7e8dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Frederik=20L=C3=B6ffert?= <967268+loeffert@users.noreply.github.com> Date: Fri, 24 May 2019 15:02:15 +0200 Subject: [PATCH 54/79] loglevel removed from Dockerfile --- Dockerfile | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index ca8d1c6..5ac6ae8 100644 --- a/Dockerfile +++ b/Dockerfile @@ -18,7 +18,6 @@ RUN GOARCH=amd64 GOOS=linux CGO_ENABLED=0 go build -o /contentserver ############################## FROM alpine -ENV CONTENT_SERVER_LOG_LEVEL=error ENV CONTENT_SERVER_ADDR=0.0.0.0:80 ENV CONTENT_SERVER_VAR_DIR=/var/lib/contentserver ENV LOG_JSON=1 @@ -32,7 +31,7 @@ VOLUME $CONTENT_SERVER_VAR_DIR ENTRYPOINT ["/usr/sbin/contentserver"] -CMD ["-address=$CONTENT_SERVER_ADDR", "-log-level=$CONTENT_SERVER_LOG_LEVEL", "-var-dir=$CONTENT_SERVER_VAR_DIR"] +CMD ["-address=$CONTENT_SERVER_ADDR", "-var-dir=$CONTENT_SERVER_VAR_DIR"] EXPOSE 80 -EXPOSE 9200 \ No newline at end of file +EXPOSE 9200 From aa0f6695d7fa5bf6fa7528538c1f82cfc9efb48c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Frederik=20L=C3=B6ffert?= <967268+loeffert@users.noreply.github.com> Date: Fri, 24 May 2019 15:14:44 +0200 Subject: [PATCH 55/79] disabled debug mode on default --- contentserver.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contentserver.go b/contentserver.go index ee47bcd..92bcd55 100644 --- a/contentserver.go +++ b/contentserver.go @@ -37,7 +37,7 @@ var ( flagWebserverAddress = flag.String("webserver-address", "", "address to bind web server host:port, when empty no webserver will be spawned") flagWebserverPath = flag.String("webserver-path", "/contentserver", "path to export the webserver on - useful when behind a proxy") flagVarDir = flag.String("var-dir", "/var/lib/contentserver", "where to put my data") - flagDebug = flag.Bool("debug", true, "toggle debug mode") + flagDebug = flag.Bool("debug", false, "toggle debug mode") // debugging / profiling flagFreeOSMem = flag.Int("free-os-mem", 0, "free OS mem every X minutes") From 1b7aa6475ebe9a17c7adb1651d041cc60705eeec Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Fri, 24 May 2019 16:23:20 +0200 Subject: [PATCH 56/79] skip nodeRequests with empty name or id --- contentserver.graffle | Bin 11228 -> 12585 bytes repo/loader.go | 1 + repo/repo.go | 5 +++++ 3 files changed, 6 insertions(+) diff --git a/contentserver.graffle b/contentserver.graffle index 6e61615cac53f55274301c97d9ddbc2e6885c3b1..5176c775cda5cca5051cc30c260598b8eea99d64 100644 GIT binary patch literal 12585 zcmb7~Q*b8U)~921Y}>YN+vuR*ByVinww;b`+jcrmI<{@h|D3O;YUZmtgMIO|9UTSP?k`7LntvC% z@}j}#HkI(yqC8vY*=Bjg6;8DVjrfI@_{FN%C?$;Fr(qbi^+gW*ebNh&P`=Wb5&~5V z>PFmOzmKA+HA7FK&sx8)mvg^Dszv=g))<8opa-BLo-Eh$<3X+xk~3@0qWOg9Z+fh| z$Cf|4E`#Jp`E63#aaM1$u7nCd4#i?pia3SQa%Rc+h|K*y4C*^1DgRpgCz^>!i2QK%hU+RVT^fpytd#e~@wbgK^vJkYeN$4#`hhd7YVO|VZNI>(A zu^2dI9ay_cI*>kXGekVLGZpwQ77nm_QDfrQqMGsuxhH!NePWQ`LG=*dmV4zxjo8kD zaK@k?W0A7=;3~N!{O(zkGe&Dpg=Y~2ckmv^$&S4wh<0;iB& z3v%saRYQ#%jRk=^8qAS|hsR#6i9hQ_?aEUY+*70{P#;q%3^b0gfZ1^J_9gWPbAKDU zD%O=SZv?ds4yGnpBU=F+RYviNP{V5~*z1R;y?c1+;2c~Nlc`=7dA}?xfFjta>fYd4 zYAcfC0W~kN)%KKfrUE~gJNHrTBDPXbYmJBzx-6OKSvJUxC5-j=aKOh29QP5F*oI85 zI#^3v6ci<_f28bSm%fR@xJzzk!7Z=>R_$#uHTPp<@}T`ADp}knZ?RV3NopRwFZRQ< z_Q|!}&5P9rJMM>VU`#6`9MiIjvW%zl0xwjvZ~cxWHHNG7ldhd3Mxoly4MfeC zEk5L&OD`7tIkDZDh}V6FMnav2f+w@b?T%j;6+H6iVdMIvl1{!{sWN8e3<7$5#1w?H zn)|Js^@CdmH#JL?0t9NC3G%tgeI*9<6(<_**K;4?1{}V27F)i*1r5&k!ZBz+^(D_7 z%2CdX%fU@Uq~ZOO$Mg`+k2RwpN z0_xK>_JP0Jtju4CrUTqY0)}L9&i7u;BEeHHFL%^)8kD~~=CI=r+#&qUhp!;xx)*dD z<;!`MD|n2mVc-!Shvo7>P76KQvo#It+jI!6?A4Xs6~(Kydq*nyBtetmO8k>Ss6e5W z0nEV_h~|XhkSD3iY!UciC0e`X$`PPZ;N8S%eI$L3_(2Kb zC>i?`3fd_4QEkr6mPJ6c2EgKkX5;vGOyhkz=n-MfQ zBLb(|wwzgtb1cLl};^LmAl<^k*tu(ALw6M&$ z$G}8lMN;PQddNtET+ZJ%Tv^>8QL<6-`ltA(XSk<&`Vmx^bx8yMY@4QO@;?|;qXjbh z6OyMm+O?oej2XrOoawSp$wTP@2-!d4$OA=wWMi%!LkBkv(8S*c4G5@4&sf#87P{vm z{q&QIu_ts`Vd0n#MvVg%k)B2~HV6i7h6#2-flSVTDJ4ZL*WffMA}?7UQ-;i9NMRm? zv38>%OraU%lvYwth;ip4qfW-bSwK_{=9}otEzO)NqrqTRpa7I}Yd;8=e;lm~j67gR zvz$5?P>7@Jqt98Au7hOFNfKj9Y~*WQ1=HLR5I`hR5bIJ9c8^oGZ={?O2o@I^NRv%D z?>_Od=PNZ9a1>{Sh))X-F#}}s5)>rIicsyi3bCftGgzQsluoCoaPaxbG6;Y{!43>M zHxQ{2j~$ju8zw?06LC zeP*|v3JSQo`wr1fE$|C)fVz`U^v2vmEXivn(>C_|%yMj3#2FM@2l2H${48vCcJLr} z2;D*a*e05e+%eE{frkOYODe%3cr(hG54Po$BB;r}9_ujrwWHv#Led z9CR|sJ!Zz`v-Vgo0ry7CC6I1S0qVJRf}Nvh%ebYV=G z(kB~pnWw;}v}gs;j%>}9*G229>iiY~pt|nY=1n0vcuvU-Fqh<8gj&nU#N!Vq1fgPr zG`GsMJE2Z)0DLkp^B{&BG-vGgjgxf6Fvz?}-yR%P>BuoizJj2shdAFuEpfV1nq+*2 z2X8NHFEXX=>2gF4+3KGw*N|lFYtW8X%$B<$+*FPKY?S4{7#agm@U&Uguo>Kr?&@mC zajl^gkM3d5D^>o1AZo=nanB4X@F&Bro~rbyBrbE)Ro?s2KlJC0+SZ|ptrZLf{4(8u zfwCx=u1P32`e$Hv*rLfinQWH;0a{Y?R{iOvL&Ld?)IEQWbE0h+;~q`v1LuQ6 z;AmGlcqYF>8^xA0TqgJ5`x5LwvdavH1>$ukWzw0e2g=5+i4Me_poY@*!iN)O3^|-B z2DQmF>7WT2BzN|ll)rf^&#c>xHv+C^NwYRc%~<%#!J&0Pac~`RZhH)~2V1Sd zgb*e*9byv@PQV657E-UiJ6W=|1OkAJum$)=_w7FE>iadwogS7-pntHAg1K!fi8Z0B zn#6DvRmOFK5lKy_C+J++l@a&60~A{{@E3GxRjHLUShT{bA}dmV&z~%)w&pKO*+^0Z zL(X(~+4!GHu`{zXR4pU%apLsJ0nkdyLd(p`t_C5ibq(AXFdJ<6to5|r@kUMfJ|*T2 zPc0iGy_|n{&`R1op7v(}vTJMA4VObxd#F}WIdx8^`Qy2DbQA~dD=h97>I*IL*~JX5 zy4)*x?oHX`OVVp`k_IIQNC&m+^)fbUqPufu>VH}{y=6oaJZpTAdbH0N2cN?7fuFM; z1+4PvTbZeN><%|sO>>3WvrtzN3)$|mSmc{i9NQB2%P5eR1|+U_MD8iGdL;p-1FF}4 zq5v_sW^jnYjqzb)0d8iuEs?X?zm}IayeuW^1>79&G=FR;VRE-e2AiQgFVlH1lR7)@ zAo?WAt}Zs$O>D=$=3iWy>JC#8Ao1d-<<$xd8@ILz-tLiv?P;!%W%7@E;rVT%jJT`E z1ofW5r+M=}C@oUUKXr3(;Six6)ET_}f}<*MS3XS&#ht$%4fgVX&p857G`l=A)QcJS8ViQ)&=dl|wYtOcq^pN=h?N zIB1oSH*s!l7~1*}ty*zFf|1h%I>JKTRdf}xxrllB$raZ|n~d8T-5Vf|(#j#3C4iZ= zkwC{l?3f-j#T4{GasoYPX!tDc8=}Kl{@F)8l8h z>fWopK_5~sy}!P|ZCwy6d01}Qu?VUh%OzA`b0duK50YQ}u2G$g7k|1$Vf6wt# zTwxOPIO$%S0n)ueC_(3uc2n$$MGvwQ+^AwX4~CaQ*96vFebZzH;V;)i3qNtvF>l{7 zAQ)nmnJz{#GLwcFT5Fjdq=lU^5QNpY^SAnY*o$zZUI# zv1j`en|unvJh|5^o=%ym?n6enO_aD|ZI{%W{plTORQoyG=hR0~8P-!Xwh6jPKCP2k zJo`qPD~4JQ(t@W!w%6+BYP=T=l3|)w+*bs9e@ZAwK5t<(O5jR~n(NIbaX81Zw-tjb zd=#vfRP-!1t!q6^&dXX4+Ilz83N?K#Gb%MPoIch)6<5^Tv#kNzGa9U=GsuqwgDmla zk?Y+(e=Ybfd;U;qrU&JlLu=~%gl3r*JlizM(M@BYCo_B)!xFfYi1Yq`y#73lGO&}B z&9P)dw>g;2X7px;xcm-8?lm$8AN~#{_9MHIL<`6}?^c)Fy=@eBbw>Jn-K2k4x%Si{0)Eznpu$2#t`r^7iGD zSKWTxpM}aBKq1e6f?5Oi4P<4W56`=4CB?$TbEhfV24OTRmJ`yq^doPV!Q-5?z|5z5OFLl%;@hSLw zx@XgL?^L)k>iZ2|Qv5WX?PymCo`OG9*t>Z%>%H<`u`i^=X&CeobxmJBgFl!%w0j$e zFd_i(K8`@BbM(88jxJ2PtG~pr!T%Z#B3_v}2!#41uUqM)?+x-ty*3hV4o2-cD~`iS z2*pU&`wi%H9PDxVbU$54#BBYy94o7aq*l5s|F_=BIz3E*X^M9w6i#!OHH|#WoYZP*t6*#aH22P`=CHD?Gc)Tu7%7+48-Q;h_rh3YpXR7`mq0mxXBR7 zr3|>&z~0nP8*@62_gh?TAPkndP>k8!AJ`uOuRrV_3SY@)pz<|!vcono%zb2WN|izN zDS69<&U!LFQ3;J%;}Lbo)A)XZ3wV^xxp1-LIFFp@Y#W zDmXw7q-)xD!;SiDWsQ1OfF0c9W#?LW{cFs=SjA%)h>wjP@@+jB-_hX?Rr+a-Hh@uL z8tk?nEJzZgu=OHU*umXpqbcyG;chsF?g5MLQ!{C6HJPq(fpRB2UmTil8ft4eH#`r| zrs`h2ixcs=dXTMnf^m}z-0h-IBkajgC-A*wXNF(@)!3a6p;n~E0KFa56Ve5W%}(Sj zUo1DivfeAewHJoB8!l!m#HU%e`*eriWm@a=Z6dJSDsPKRxfZ4iU~>I<(K~Zkz2svG zdma5)dHRw-uiHQ^_FkMTC{UsCl3u+c{+t5eeaMIouPbXstnWeptg!WZ%VxcoyK zalZ*KR2{EWxf-^e3hZNMmBIy3Ato7`#dgBQJY4sRX;-TA!hAd{l>-0}x8n)~RE$hh z_Q!qKP7g(X;w~>nNs<&IIkWGlaUSa($GQ5vwXe^ll$79*f@^^ja6 zvyhwvyd{-RMeDl_y zP~zs$5p`%y?#ln7Dv{J z3r{_W+3s)ayFPH+e*1%$d0&l;0XIYK6xEsltoMzaKQ}h!)0JT)F^0IlTQ`j(ENjIv zi{qTI>QaA@K0jWZ$eyF|zZ_x_$XqNBDsw$BcD&<4d&GHL5g4+Wt+8VoI#go*EY-vJ zB-PR$olekx17Nm1CXimwQ&Zx43_@@9TIuQA^TKXROuxzaB+XFYmcOl&i5bWqVyY6T z5XcmFl#63tP+|2sLg@f6>_ zE6B|fgY6E>JbS7-$R-+@4Ji5=bvl-UohbB8*sJb&AIc*Q{A|-ZyNq#ItZCt!a7)xO z%g8tRIMoTmjd<(8aa;0uH3>!OOgHKToi0{>i15Ytz0Q+m&u@F;U(j7WS;;{zZIrQh z8&lc0gHSmLVC?M|v+al@8HoT=f;qEy4j0=9z)n`IRKY&h1isof)25Xd zi}>4CTi=avhejB0_u+Wh3RkKb{Iz+TthmIh-eg1>GEMOA^zPh-WFkxJPVM!?U3oDm z*cP1#PYjKqF~iUwZ92pb9@kIw8RoZ7Vf$P^tZ1t)*L-f&Qb&XV{J5X~34Zp^G{VdtM9>r3h`gdvyZTh`Ej(Au z*U?tkM1}!E-ot}h{NBR=6m2j6wCVQij)2;)hf6t{PT#A`sk3K( zO&L^*>bG0#qarV=?#=Fd_pX}H&*#$G!KYqBLb9I0Z`IHD>sh*KyF-WX-m7EUY2Vk6 z?+pyUh!dFW#|nZm@#Bw*`$O=`9o%<33E4}jqKEM0FKX2{Gl$te*k{Enz0i}3?Ad6q zEjVr8+QNtrVQ1tQEQ$KF%0G0y%<#26p3<*NwpzrOzL0_Y7%&0q#K@f=XTLF|OJilM zg<@j7({Oi7s~>&i7Z~@GFz!(OKiV%NH9AA1x=eWN(ntSs#S-Qf6 z){v4TYKw(W{f2;rANQ@I|R0SzrUp^Y=1Qv!rmT94O~skT5%Y=N)D^R8vOu%2s@B{DzogZP7C4}g}<@M*^>k-uWnRsF7WVk6FIOLQ^ z`tAcpI@0PC)g~FGQT1JTo_)|uFo^=Et>3_mV$vaR^lAPqKxwq|4k*YzThSCgr`cj5 zwzi2S#IKM6;X9e1YG`Q}yC7}9OJIq;sNv=~ZcSKDyfFvxUJ@`;j-U(p#oX-!ydVDy z?nmIO807oEBOc_;5{b?{aC$Tu5-(LeG#xA^8T8S0Cnrr^JXKNW_nk$S6iB_cFIMXo zYUj5u>boDdFMZ-TMl2Wms-&D@mHpl;rsr|i5=(v7n2m>|N2qwO>MC{T(;PgE%9fUi z+Pk2Jzr1TOltKQVY@edEygU~&Us_oPc{RI(QZJXdm0&7rX_d z^lY?ontkKkz{o84&`^qNTX9J?$uNglxRn6W`^)Z<`2?n}#g0T$3JYg0bovR|=zQku z3j2Y8G-hmpUwgh%Bt&QH0(Ik+QVasZ7h~}~e0#BlD>3~)fd5xNg3Ur)5u8aZU7eB5 z=3L1^;__mU-(8iVUg)8yelN5Bk2)rO0J9i9?^@it>xLj|7OpDZweyq@#U8^5&Rw7z z)_(6?-fZLL$C`#_z%o)&du+^+w*cP=>o|RLF*F1HK;sg3t-?iJD^{142Bv&@vn_d2 z=|V%#FL$xl(BQ@I@T=hk+ABWi>*miC%s6clebMjw#qdW}%4W8r{Bj#HvOP!4^m%K5 z)9%*8IKl^$zH{sHs~4_3feE#nxy0F3@uj5&O5MLG|7-ZaD8EQ=XlH1smQz`_K$?fM z8)7$KCb{N>p4JqX8h)L%9dzkuatN19--0tGvJcKW~fpiQ6zB-Mo<_2L&g12RqhR^caPYht-rPVHu_e|Rr>ddji}p^ z(UD0P8;}u+70cL)phMB>9Lk3X%!rrDMR8+t~7 zXoS%`Pa~z~Ml7~-%zr!W=5{0@p6IZv%KuyibyiE{E@Od5D#|WmlZ0Y>gqLvyj3k;V zBr>IsrJy+$u|YHEC80sZE_BJr!gz(^gHTW$!_vuBC|Y7#Vc|+Ky14FJQd`i_(=rTX z9?2d{YpqCc?+t;u8erRpc3vdP^baT2Q?V(Jx`dn5jCZpV!&%7?HukGGl#-9YTkqJD zqrj`~G+Ac)_zFwYWrN`eQE!qBpL890@&=Uh$TcWc2wH7ac_~GY$Cb zLJV|-gH6Z>tD&qos#aQ$Nz2GrC*=R&)T1>HMOnQnlgDWh@hpg}y7JhXhWTMC8QKJn zz8lM0cU(@%W+@vYuQskti|FzpgOq&xpSXW6NmRk*9G%+$`b#Wg;5aHWgJ+V0V=LVs zK5to>Ze2xO;}W>m?R;3DOpI!pzx3pmP^6(aa&OVrtfHEud7U`jMkN!`Jbiw$c4jj> zH@M>$+<4^CR^x$Gi-qIji5Cfj97!>zAIv2g`|gYczyA)5-XrjsW|XMTy8S2g&%tZ{ z$wqz-f^cR&1QlSGIFhX3f$%tm=p~G~FomeYuf6zoT5Zx|db-&k#L`pnZZFaw5xMmM zmZp|m>=14Sl%8oyz0?`7_x(lPsF1a>Gh&=-wkg6&73Mv?8{rW;)z!@c(#7$BcxM!^ zGrr4CD39T_lONO%wmM(+b2c_Wjk`+9UnH5EWX%H6wLg;QArqWlJH(`s!<@1?2WXmj z&^{1Dow-}uho?+G%Y@F90FCCG@;abWc8rpD#5n6Y zz+B0IeMA??>?RhpfEj2S@>xg-H`nVfRmgC$nO=Co+km<*(GZ8Mp5D^ zZp~LZFQ0mTBQ^v^X{My1_)$i+ zFe?uGozsY;!?V#F?t)w zMlQ9%7@F`;ait!lq1BmggfcqlwB@GtKN+qN%{C#hq;^l5~D>1q4BrDh~`Uf*Y>Fg?cDr4>?c8}B5d3Clk zjakRzVYTTDrYvLDVejd5d`?tMoRP@C6PvE47ne;}6xLXH)xIUaM!RYLy9cPeqKKM+ zV=DP+)x#4UC1MQu>VOs}11F)>833UG5s^)z1Wr-!f-#(*{6smL39i@JNjgK{mBS{8 zJjh+KamDQl#IzGDO?P>G%GFh|p8c8Kl``;@1gTgn;xIfMtcqN$4rJw#cn zuDteOT{VM!Kb{ns9}SGkrD?&^Jae#mOniO@Y6!-C!V=XSaZ6Y%xwGpns9w%LL~%)` zZEQI7f|v!t@PC&9$-Gb-*eXLdG4p0|!$+=%wrj?Y*ie_%C^pc8m(F_d=MJqgpOmI? zIIQ9siw|yOh5Nt&;EYPCBaXjvZoD;l<}zAu6)SNbo_hqx(rSN^oT*m7GK9jL>dNy+Y|2)EujCUY zg82TGBoFq)y>{k)7#;x&mvS9*z38#HP0^C6Feg>OE6Di0`!o`A<47$Fg{As3C!74* zW<5vquiSr^0;oY2bD8GwyDf$Gz@lnF)Y;p) zCq{2dmsYtJz|xA@7m;@ET7v$nwSn(rfrwS?6)=<%}db&AA-_p@6V? zTjs&rsFS9himY1gLs-O;V4#PG;q+@^Ij1^=s436c&}$~jMx5yqqWP~T3@q8iPZcejw>QZXtk+b-osP*h)PdG^13L6!pkHScZ=z9wcAg16P zrFq`NrL$$Ps?WY}6ZlhhL!4uT41XSpgjOCh`$4-7mH{6ms{H*I3 z75S_>)J}~s#mPxFowkJdc}A+o5jX|n{*mwbF(A+ym)|Fm)8Cle_0`pMEg>|^b9Yb1 zvnt3PQY`0%!#!hSVjgC{_xw~?(3aun6!Q}gHz3`HWfv!|{fvy859)kIj!5G{j(@!1 zfD8w5#S?}9oSSTJpF?;ksW1A7Q$Vk?UHZICIzPE-x$+C22A(rZ?j#yt4$-I_g%A6gPF@+-UhtV;tv0m<}d|u^Efu> z*+h9u=pInPdWZvcW6SZKo5w+8%dxUa$5WZAJPh}9gn?+hdozvQ{OHj4BKLKk$-C@K z*7HbA>$w}NCkXX~|2y{--T>0>{URdIB^Rkx z+SiNaqTjb2Tm-ycztZPhv1Ch!CRp5@z4jO#@A2-gfbTQtlqi0ie?R{TB>GX*gLTw$ z4f>v87aE)S@-)ZOW81$3m6Eozv$8=As@>k}R?*k^20FwSoxm5}_zz4cSd^7^Z0 zmQC^~&uAsuSo>r&Z#i37w^NXq6winA;)^v!L-7er$@o|AJQr!#gU7(5|1LM-o z;F__kv;9UCCbCX9M6r38*{dMh-Oj`f!dvXxY;w1=SK8ifa5u|zY{SOEuC=`TFZ%OU zZd+^Tj*$y0w_!+smg#Qgr*rStM#8g{UccZvtZU;%yN36|Shi`RcwTwfalfyi9$V1- zV>POwb9GEAlX~IUwH-r$#z<%S*lf&my~(%p=-J0I!(&@aA<=G%8e@IVtUiRYOk30* z6H7iYdbXwD;q`gnuy)$&cAl!JejM-HTpcT{b-Dtm!`Y!j;oIeiG+2aFV(Y=86o+~TaYEfaYH|j;N6s{XbF}alUW7pt<3DasQ>bpR@n-IM+ z=;=$l>YRMB+c-Pb+O~98!Ol-n$vafLcIc(IZU1|==lxT_@H@@ZYggbTZp-k5FBCxR zgWl}r)A`4B-KSKkiWypL9)B?N5=u3mLN;X`dK%s!6{vdCU zXwR(-mhg%9gdwL7O$u ztKN$yFQn2>glx{O{_kkTcb|Gf#rNGNBE@&5(U{3J9;Z@>sDp!urINQZv(Vw znHBXO{h0kY5~{yk(a_}Mdy0^I5wi+1W`Gf%fM!B5XI9#ZnX?DZ6?9zMiwCvhqdD4{ z4XpbliCWsyweGhvAfALNVGwMaHbFOCO0>Bx!wqyd*`y2)K3o)3nqV!6bx!{E$f(U1 z3ijD-h>##cmA}tcdW?GW1l2^H2>B)mXIhkwcJ@y7^zH!fB#Ty-TvXqLhe&87NENd? zT{v{Ij30tQ`rwT@i%8TB7O#6Vt6m?Ijj(M~ZU#~u1`Mg}=%q=Ysp#xCX65?BU(z!& zR$q{8FnQr$hph->2sPAm02Ax$X%v-H}o?>gV<(r5uP{} zjs?GqfD+d5-&jy|q_M6r0XJ3fq7L{!Xzibmeq zeJ|Vrw+>+Y`w%0eu+jgYjovBP=+*zr< zRe!W(HsREx_J~m-e{pGyzx6Zcitz?%_-ri7fqjKMT`EW`E!+|*UBu`i-6F;1v=ny z0*!8#Nu#CSmgos87S+vj-!%M-iwKBUy7}xvHnQZn-(W!(k()^UG?_%lrG)0Lk!SOI zU`TDRq4BF`_b#5t0 zq*|Nl>UO=?{6R_@{pqbFJTJ1=G%4dH;GqugvvNEawXbyu--d@1=Xs}(4q3R2J`WG| zGY{MC7j=kaDw_sYHRtAs1H2b%*yyu_zhzo*r=tgKXA-{B zcI>HDq}C1otS`!kkszTDN+9!L4~#FZFV06cu=h74GssxutXl5WDjzXm7d0y+lXoro yBrmLeGW!^R>3t9?e@2Sf4~CTT)rZW1MX0xhlf?P``{VW-Dac_v2EMz*v^e@+ct05v2C0G?CjRo)b7lj_w#=0t3wn8 z4f4MN4sz*h-C7{ovU6Oczq|O!>|D(a_9yP@EC-`YNDLJ8_DI*+6J|^XRIFO{+ZSxe znYXs8s)v@P(mb~P4jSbhqn@bV80w7S`#pImyAXh7nELf8@c{Z-zh5kW6k7iMOw>&A zet$Ch{TWB(_e%7TrIi6!(ik;Za6!lFM5CBZFJpxY$PBO4Atu5A8b>zm>U7Ey*c~y`*u*@ zFQV&Sz{)M(IHgX%_s{2i<+@>O%CVa7EkHk#IqdEmyuo`ck@eyJqRu;Qke0rcJ#z)` zYu;MuDsPVIHsC3r&etY*Z-rotp&Rd19D*jQ5kJ-dZA9-D@>y-8S0`V$vdK=+m8#3O z1_Xp9h3oRdM3m`fgdKV)LQD@)YGIZo7k?MzMG&G7f(7S`tw<*3nDIm@!Vui2JVPF? ztB58XM6VmO1r%gKXUSs>2ZI$>=>xo04G3TuzDP1}?U~PfpUhZbF#AzErA6ORcDvD| z#KxEjDbA8~=XK4|U@ebd6{D4rha#n2*xxZ?y*6+b=^PYM!@7j;rLn9&{W1=_C2^ja zCAKLJV^=YX0I#%=$H@_4+}eZB;X{_WEK}KY@!EQ8*^&>17)&BG$l=80YyB7hB=mYJ z0zx_NhhfSfBiE9~Xt)L+msmMqg13@Kfu1K9!`XS`#Sz@gS^V6%@wKPC>H< zpZSYS7B=DDW%7f?%m33T-)~?YVRNOFa?sMIYA6TC#sMrFPdJP8YYVr_CO3O1GG==+*uq zy|zJ~8;%u*9Kg)fabc~Fe%MHuD-C~u_&FhAC;kcLC=WOGjrAt&lyMeSWTne+F1=o#<}OB0<{Q6ooZ5p%nv=L z$d<;V5e?4mFBV}3KtIDUwk=xiB6*r*HaFBy#mVC3uc+a$N1QhlanU7d%LjAgHhY|x z>(@RWFXDQwkrvN(?VSKq38<3_ILx(eWJvJDEIH8lti|5ldUtjiAoV7M);>5ev~PAr zhi@7tUf@_FN$fsVy7RI9hlH5i4*8|C8#tQ9Spa5$N}wS^ z@z?T!X!tm<__R^#y1s7oUum%9;#VzCuaP$f0Is5a2kcEj?gdjn!zx-BD<+6Z_202J9 zf)oD{DeuHGl8{3GQ$=>K5{HW|9ZiFNoD00U?qbQKD7ub!*2yWozM?9#Xt&PJ)Zt*w z#JwjlF?OWjj)szhQsS*dC=^kBUkW3uh;xt5pO3sTQpWUL5>qj$p_?KOS3wP!!He}F z?1n`?azlTE%bOVfg73lSM0XS1)&Elw+UGF$=LkBy0H@*~;vv7b(vr4-m-S>VbfcKj z0P2#|^tP(Ka4oUD7HQH6;E+tp$YMzvqf@Gqwn;(IX3j*K6xdT3ZX8XM6wv`z1x?8x zePr|r*X|@m%!M9S6t+7+_)`_`+1^@2T*;*JL(b?v--PH@C-&QvA^20o6;rd z@r+Zl>=+G`;UPNv1n6GFbgUs<{r+o`^%!Nmw8}u(T&AAF$iOk?h$_eflSy*n{(cdo!iJ}`NR;@%7K5H90bfi;6(=HzbU?Vep2hKW6w|ay zbTneroCU)$7zwDR&-?OIqP<{cX8zN2Ult_s7qH)w8KnUishHHR;lX{0%aMwHC5Tak zDxy~b_-dpC(~h3n2@`k_u2rwhO>=RB&wd_xkjxsAxR9cGHS=kJn-)gFmRHj5j2&8N z5otCmd3n1)s3#@UkS2FfF)~OW&Arf&)O!fo7~_y*Gkj^Ip7q;{avJ%*p?&IBGfs#S ziy~DNzu&@k^`$yAuisQm3J)TQ{=Bhiw7!2nf*k%q*{d-liMeKO*M|Qd#tBFH{%?5M zZLEv*RO?Y+@$>PkA!`=>q!0gCD0qi7Ru=o(9NRFC!n-^C}T7W^-~W0$*I1qLftCYN5wxmCdbt4pR^mwUGEsG#zC&%T-tN zdd1HqRC>?Jd#ENk3g|rsS;iP&DOtwqsO}`D7u8yU5k*FMUip)o>n~!E^a43|sEied zbHTmA&!|LP!^vXr3=Xg5@IhvGtN@%S8DXsl*LW*lS|Y{J1s5eLMgF;#st&`_F1N%~ zT9+RCyZ9}wBxBcnnPPrJR~2{BAo1ncp12sMGiGj*8iyqTTwbf3xiRMGwp9WAmFO_Q z5s=`c$dJ(I~k&xKzIJqxUsN!+MV1jTcI6UU32u?2@1;?@mSfw@h)K2BDJz6T7 znNAmRZtAr$&mGf~MRKRz<|#H;^svPCPSo?enq5SSEX^NK_HLwQd>Uy)L1$4S$x4MRE#@e$c*DM_#qER?g%pS-@s1I3avJ=N z)s*t#?-2>AOG}#!i3HYSrGcT>Rph%*OD^t1O5VM<&llu8as^*o(oY{Iy%q;ml;(#YiYll&?Z>`!nrwFwnB5D2_)k?i3^n7U7Hh4r~P}2u*s0V$y=f!X7$xSln88Sto)fnM`ZTOv91l zx%aV6!Eue@U!D644e$sJ}afQRQs#-^CwKFSB_*)&4v z&AXIO0M{e&v{{$~>+A&q^Y&`cAdUOc=&oX&%*bGtVVmt;yc0j<;9M~?rVFo@oI>}O z2|IT&KvzM0r159PFSRYsQ{&AXj?=`SJ^G^4FC9R$yyv^7_3h}ft<{V2(-#NwlJ2|t zMviXEJxw9n>p1!08AtIUTO z^2n~;a(YP4c`-x63gc7i@g{A}QvQ_QfKz)j+{qLdxt>-})nJI_>8suslFNxd9(oz3B z44%|5|9)qgcAQUCq7UB_fJ>hLC_aqUH;}0$$+1dyQhn!@qD;})FqN2|A_7C${Zy{_ zF6o74uI722701VUw_%c&*UQ3pa&rHGwFI1`puOdn_haH%ocafi*Mg&a zmvzJ9pndiuUl#`Dat|^+tzR=%S1a6}HNqm{ay=CKb3M}L6ZxoqGBj>36Mr|kEvaD4 zI2~L1e0>JosxkWQ4?%L8dOVaYtxkq4KFYhpecApHy247r!_}b*?Bqi~#Dk{3MTBIP zq27(seB&jXcPe<3q;<_S<#f4<1K5#S3R#!!m4A<)_OjjpF;D!1?Oy`N_l8t0De0}svq5)84#)52$K|Ly+p_gv5Wx&w6L zUquL|nf-@ys*G5tpN2<%ZvQm}io{*$?L?o29etrkoD68LG5!`~-^4i}FusD^oelTT^3IN4d{s*ek#8rTh@U^{ds~Xse;?3CKfshRC1V@7=JkcG zfZr;Y#vu@f1NwI_Pi`d3BjD`A0y*~jHiKjJMgO#l^yB%p!R)`Z<81|P5IhQidx5(C zXv__2st_TJ=h@^FdiRIs_D93@MSB|ldRT?PYPauATgF1U*nD~L3-A)!$tHeZn)<4lz_KPq;7YTSDbTbcNN!V7k_Dh%rLe^2(N7e~(h z*tnm68;C_B^yzhjUwi6r_H?ivu5ZE7W4+{Da@kl5TQ);idd)X`y3XxdK(qtldrr9A z=I0W#{gsQ~grq;Sox?o}Zo=vH^|Tn#AJ6!8@Bmc^I)o38Vx)8belS1~W{Sjg5T?LG z>z*mlTJly>rL4rTsUe?!IEEy^il$}|1f((#>KAb9Rffyz7<;U)idRSglq#}}Asu=i zyDru}P;;(3h@A_j5M(vnpDrs1xa`P;{CvwVt0iXpkiJ}d68-#o)L$o+`F=B=3;1}^ zS0~LNZQ4|<%*X0{#J4lzkd3-i-_l{6P>H8p4$F<~ee&`)?t^bYp z`?ze?x9;i+pRD9ve%ki)7J(Rh82NU@`Aq%T#9PQjXIW9iit0I~BmcFKi7b3dp&sDx zuVk_L(!yJAdVDh2kpx2difFe{efEZD}q3@voUXP zEnTDY`$g;iIagnfNrCXsC zAeH^`Toy(w?33G37>fFCnx%A3;^-lP%hyEyTlPq$weY0b*M%bJ>r%T^Ij(z*3aa)T zaC2tAO+KpBrK<+D1As{t+-C6ICS~zxsp%B>QeOw#jX0O|>IRY^JVVu@bueadjFM;P z@))kdL;<;Y4zf3B-EdeXc%ldIK9o!0t5CmgfdsVAS_hr!o(W$*ae@4YLNicWo{k&! z+^+7Wa|@f`FZt?rxJR!e<=@88{2tT_hm#3^!;BQHK93|X0!}<5U z+-#CFks_5R$(p0mEM*m4N>UH6pubqhC>YVc#z^_N-}c)YD;zk;@|GJ2V6?HQm{L z++GBej$H*f2>8-0iMianCHoYecN^tM*g&_V91^QS?WGl0)CiQRxU2ZYvsjURrxL_ z`wRF9vYGtS`0+9AmJ){q=%%aWXXyPBP|qZfe`66SynWVWyNOfXH~#yg!N%Vbbg6ao z5PsbWH=r(1{9c&+SqG2kc8?I3Nhi~w=N+x>@j5qY?_SH*LeTs9bmPkTwSF0Y%XY3k zW8cen#oGHf^8I`Fg8j$y`uTS&%nM-5|7Nbt6EhuKM*z{+yPv)-_e;E8@tX|mZ7jr( z0qe~y9pQRdq86S&<_>WS%Z&ZHn7p_7RoIgh68a;mK;xbr`>#EAD2F{+k>B|>ohGY4*#5cJk%zL;D zk>e>Dz3)&lCH?OHt^xsif3Lreu)yBDgw9*Ij6M%N8(zD>GbUmX2@GCrj{*$dE3^oa z;RzYM?yOhS?$?t()Bvy#Y_T`uMf}cKYK){bGhcDPq)>0L@1|Gr1QCE%D2wv z?At!-x3}rtNb0W|UA=F6)h3#rd^!5UuiqH_9gi&BxITdJhui+@xbO#*gdQ)^4qyM$ z3<9UobL<>l0ar9`|GOR55yG3>1#8wCxCXf_;T(HT#Roh)PJaHRJCsI#qow6IO=rt- z$;q$|(RT)nk`cx*Qir|H(egM2A)lf0IL^nv`-_QMeO(g~T@B8@$*=nK8VEPLgU?9n zL`XFL!Ig0ikADX-Hr_vFhRUrCVetxVEh=c0U!dQ``%m^y58@0SP@;Rh7`>iQ>g@63 zuKaXwpG5TXy}!>(YyCfKLp>fFjqG3L--DXh>1u!Ly}efoT6AM{#6vw)2bdnpBNOV2 z=C8-md>=BHNn&~YN(OmR3Ee1yFusDzOTqSt#ERPQ4XA#3$CDvc#>c)3>i~3uF6<6d8J!hUO}}O4sKC({hG~mPLDrx$WC4G~ zXhRCghMY2}11Ia@#Q9-B0#-4UOyzlaaDKzvG$4RE8YKWk=2;)6s3bT+_$@5K#t0|C zWJNkS;X71}JizrUN*2Hv%3(-VR3(axD#)Ce9{*gJ&iZMLzHt= z1K3LSr(pHco3l)IpHeD9!7_O=GSZWm)p~Jaj)wZzZgy37IG)wV*w$J?RTr$b@(Mx$ zEl0B=aGKj3EFx-kY*trMZ#k$NDsjsSPNZzVBAt^s-OeW+$%IVqy)6F%_`c=k*Jr_@ zgcI@-i2v#HIw#_P)H3C(%A63ymKDEjV)%PG48Jk%dNAe(GPweNx#f~)k<0(mvvXzL zHRB013xAz|R+YbA&x&nBrxD>9n@+)oiFWA{UZZ8MQ|6?|LgiVx_$?&6rV*yEJ%ORL z#vI(lHmMn&t7oC?Iy(YppIS9tx5_+Fgv_PTIwfFEaxK)L7`1yD+GD|-wKY<3;l_`^ z-V)65QRj4%z zFvR}$AFFlMan1exDt)Cne&oWy=nD=d*I0+d-oxJLgX8FaZoQ^f1P#OD|3H1y7L&4+ znQav~wjoNO?L${{{lc6sOwWQSV|8;!xwPKd*xR37mX2TKMPJ@7_6chD+xhQHEBYF@ z$r$@=Is!|O^}3Zsjv-C(-S#-5THb-#)I5NwI5)|w9sgUi&g$u2)BZ)N5T{Z30-~Ie znq?HwD5P_i2Z`wLsuZwgwvAT!uG{=IZ#%z-5~S`*In>HSTb|U;ABY-mE&+DE2Km!^ zBw>tG@dq}r0S84ffpu{$h(D*OH*GAfkqSY?Qjo&{nC0RVAA(C9j}Q!|0& zgy7^~h4T3!!Z!?>AJCsUAD9-{RG<|Ho&Pn2AriAfR8>4?Qc#yKJ>atatgA0ImP*5@F(ZvMNjLb7+S z#Nb!F`q`d{Uq z)&zvox$eGvDi@uYqg*;iUa0&xXWG$DS^r;Sc3aDw|77<&BYe-t;EYZdMOrToU?(+n z&a+0?pkowLSkbWPpRiK@9`Bnj5}60|NIdfn515TrH94dP4L2ut#+t{ke84FPV#E$r zH|96e@J}-#ak#To`p2WlfOpFx%$02_EwLt`Yh8#5m%lt0JsO;r4%&CH8aj!YLfr#L z6GVrO`_7wLN5K@v@Up-{`qaPDjb!_8g1{-Dg)hRqts>!m$@yn0MG2b-qAO3z>Zm!2 zR+BnvQ=pid?-fgHbCt_686MCTw~UEGb@HV!B|%&uspQMO5%(&p&H=l2upmO1m1X!r zB5z86aKfqp(%lZFi_2{&)l1pGi#%;x9%p{?fwRc==vd$+h!Xv1%8{b)J8Hs^QXi{0 z^L`k~Wx?>vE41i=6#ZAeFH1=hPaB>fLb4dpL07dGa;HsK5*%P;SydUMC&RS;@>D#* zXq-hqg7~&18&=Ens+Q%V^%;4IrGYf6l%q`e8<1s_Zd9^YiZh*MF7pbpkIAonHTGlD zrFm(4Tu@$>mPovF1Z8_*)$g%Lpn%V{UZ|&5_bjs zDY(jHI7X9KtN97XMy2IAZR2MESC3h!gGj$r#wr47?m|sk+$>B{QLa}Bd$Eal%L3CkR7<7+Og9#7u#clRl&#((5Sl+RL_I)uZPmj zB5YnA1~c|rZNZvIhFF468FL0?SzPF~$R9AG(r{s{r|JCD^5^huLD7ksU<>4vaX>c( zms1{NFD1-)a{ssG@5m=T$0KYso&AXnj-+WPtsU0>8>wv@n^sj@a&n;3iBBV!bb?J^ zzBv^E3@WCRwhC_MS@?(MN8vhZ99@Ai8aql!M)e>n#h4$0PdH@Aw`?Wk`4re{x(>>F zUXuS(Kk=!3T*|<&gwv&YgsE(0spTmUDzui^9v#7U#mPoi=R7AuAY_}dFu@V;vNMLx zyuBG&8|Jbi?jSfH^$}iawKkVF2zO} z>q*|q%&wol$(+cgsy(ysVTgUU=A0B_*xZ@(NFs?9+N$HeZxa|2$|i7WLRVjf(5c@3 zIl|rH$s@Bu((JV_JPMT1MwCt(gyum;!elqOKy!R()8mmrR8cAiXM6b zfRa#R3{ZNZB!5I};<5B5b(AF>sf#;l>*=p?uM`K;RUK~3sEiURFIV_x2&hl_pJ+@i zp5himjU>L#P;FShE!ELk`(z zdEMeN?}@V=sITC`3uxno>)_0TQs3bfmUhh;HQ`>#yA&5Vd16~OeB6NX3vL9`d|8q; z@#{u)@eMUtpfWA)O(?%QC-l&hsW&S*Nu=gxI}Tk7xFfJCdPvnso3zO&vZOXR)Y`D8 z5-r;>hl_Mg6aS0p>yU^PaE!GLjZ)I(uBuUAQHiT*YPaFhLuzFPUH(VaUr|p| zyRLEFIJ{^Vq@+$--!-&C`14^xFi5g7axcy{w#x2>RpY7_rd-h$4B!RqEAbXi&88*u zhk3T>rf9763h(_}*nj%XG_p>@-!4qZ(!7or3pKh0%gNGF7L5=QHzcP@XldLsD^EAm z)S_~-kkojXJW!$lJwqh%^}^d7eosmQT)!;B$GGg-=WQ}ibQWirOi|(HbQ`}6VLjV} z&YlQM#36aAdyF0SkR}~-@}$8)$tHL-ZC9H0Ff~X#&(mGgky)?OHI1J1I=EuI@M!Wa zH+mPoVn#K42#Nxv6zclVTANF8D;n!)Lju~{h9>uyF6f>9=!>C0ZH(EN!{;Ta@5*lVb)Z5Kux{RZ5Gmv(j+dz6T zD0F^KhhJn*=VY|34)03JdLI^>+k5d5(W&xdzy8S)vvs{Pz_rL^z)5as=6c#Qcrz_V znLL!1kQJd@($dx6Zvmrdq6HA5$b=Z9DF5SKFt0{N(Z4^VNN+-8m$*IS(Ok-|6?ob<9HYRjDT)3%zuA2@b z3F*MU0tO4pLe%sGbMm{}IT17pq)ojdAnz|RWEvxJWf`_CzQg1P$sRfsMqYUbwnwUE zh#GQ4ZG5o}Yujk2;NVwW(2)1URo}a?qsn3q@6?Rd;$P&+@BRlJwOZ z&l9aGpLewCxa3q***2IZE*iY1TvZqwnYSk33I`58@fH!I@z3gRM>r5#33qxLP5oA&`VbiIp z-VLgMEms|lJ2&&C z7?$U@CN1_Bz$JJ5?|2Yf7?g6i}X~AsH*H>@x3*ai)z?83nhA-oM11CIIG^JZEDBNbVH59K{r7Xi?%Vz zYE$;l>Ziup&H1K5%r2#W1Rr&kruU13HsWU4U5)Nk43@OOWL7Jo+0)|tpw)m0N!vsy zf7x2vw9VCx-qv_|7tQSmaib8)t;JH|E==rH0$9tK@S^+nyEew1;HRdoaSP2^YdqKp z>P!^KZW1A7xN9vHmnWvIH3h$a;W{5CK;9Jzpgp}c$4-0gRcqn-_$t6W`YLTJWO(B@ zng2TOkgHffsj@p~$9*Wzw26VlA%qwx=knVn9355MF$3<*$?Nsxj69pga9`BxTR#%vRwX&{Mn<9sA0+<6~(mJ{RZ1vkiDL60A*4~4yr8-2bv@zH)X z?~!+Ibj1I#6#l}~xA&>_T{fmX4jH%foWQ$VZCcl027Ped&+7!l@1mk3KdnLDIr2<{0p8phYkqIQG{33f{bJ=p-^_uFv|9H|&hv9a zEZ$3H@yu{S!}I*cmv)~a z%36J?c|OlAkU_%XIbg!r=_dJ8H!NAhb9G7`Ea}G;a)%yh*4>q3(mIHocsJfUnp1YQ zX8Hc7kWTeYS`+-e`I(QB8Mtw9d8X=f#Hq2Cmv6;Y{ITMh_^&u)|6|%uJP7t)g|2J< zqPPNTJ^$7pflUg8q&KmB3CA;pE#fyiMV+kg_4-^xdSCP)Sybz9-F9Ry*UZzUO$ej< zNR?u*0y2|d1(p`o; zU@`C1^P}gML*j$piY6Z;to)}>-_Vxz);-K$=^yTy$?JuBns@OOmM6!nhR=*)>N<9| z?DfycThtD6>Sp%4n=Gjc-Z`P~+>@xAkx1^bZZkK2crRSvJ+G_a($TryImLIigqWRx zqj3TvzNXc6lku7_sW;DVyvQq`_mC*g)hl>+%7kNBDFP0DaF|s<4y;&Q_mAl<@Gbbd zz|JoSgn-#+V*9JP>z%_OjIhI3!b5JG1Is8UsG3l~aRa~H*n0hKN$5M{9dDxTz>0Ar z{*}^pBAz#eyCS6{i}_>RV0Xx$E@2pDD*iS?ORnFxL8q5L_uY>h#Nr#8G`sRPmLmt> zwIZtvOfr6YDOBy!)*GOQuCG!WbY*iZ(`@Z-w-B7t!1|0H<{c^KURfW1SBgDgKayfE+otSW?p#k9-Tv?#yVjrlT7 u3mfpR;_I$uhi0>e-)$sTQtKfgx7$4iY|)3(z6tof^gf9YIW4z Date: Fri, 24 May 2019 16:41:57 +0200 Subject: [PATCH 57/79] refactored GetRepo: serve JSON from fs and write directly into http.ResponseWriter --- repo/repo.go | 11 +++++++++++ server/handlerequest.go | 11 ++++++----- server/webserver.go | 8 +++++++- 3 files changed, 24 insertions(+), 6 deletions(-) diff --git a/repo/repo.go b/repo/repo.go index ffe6146..df9720e 100644 --- a/repo/repo.go +++ b/repo/repo.go @@ -4,6 +4,8 @@ import ( "bytes" "errors" "fmt" + "io" + "os" "strconv" "strings" "time" @@ -218,6 +220,15 @@ func (repo *Repo) GetRepo() map[string]*content.RepoNode { return response } +// WriteRepoBytes get the whole repo in all dimensions +func (repo *Repo) WriteRepoBytes(w io.Writer) { + f, err := os.Open(repo.history.getCurrentFilename()) + _, err = io.Copy(w, f) + if err != nil { + Log.Error("failed to serve Repo JSON", zap.Error(err)) + } +} + // Update - reload contents of repository with json from repo.server func (repo *Repo) Update() (updateResponse *responses.Update) { floatSeconds := func(nanoSeconds int64) float64 { diff --git a/server/handlerequest.go b/server/handlerequest.go index e0f4c86..2d14a34 100644 --- a/server/handlerequest.go +++ b/server/handlerequest.go @@ -51,11 +51,12 @@ func handleRequest(r *repo.Repo, handler Handler, jsonBytes []byte, source strin processIfJSONIsOk(json.Unmarshal(jsonBytes, &updateRequest), func() { reply = r.Update() }) - case HandlerGetRepo: - repoRequest := &requests.Repo{} - processIfJSONIsOk(json.Unmarshal(jsonBytes, &repoRequest), func() { - reply = r.GetRepo() - }) + // case HandlerGetRepo: + // repoRequest := &requests.Repo{} + // processIfJSONIsOk(json.Unmarshal(jsonBytes, &repoRequest), func() { + // reply = r.GetRepo() + // }) + default: reply = responses.NewError(1, "unknown handler: "+string(handler)) } diff --git a/server/webserver.go b/server/webserver.go index e12b7f1..64c4f29 100644 --- a/server/webserver.go +++ b/server/webserver.go @@ -35,7 +35,13 @@ func (s *webServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { http.Error(w, "failed to read incoming request", http.StatusBadRequest) return } - reply, errReply := handleRequest(s.r, Handler(strings.TrimPrefix(r.URL.Path, s.path+"/")), jsonBytes, "webserver") + h := Handler(strings.TrimPrefix(r.URL.Path, s.path+"/")) + if h == HandlerGetRepo { + s.r.WriteRepoBytes(w) + w.Header().Set("Content-Type", "application/json") + return + } + reply, errReply := handleRequest(s.r, h, jsonBytes, "webserver") if errReply != nil { http.Error(w, errReply.Error(), http.StatusInternalServerError) return From a5ff003d8ff9a11a1d4b909bf16a6d6068a4403e Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Fri, 24 May 2019 17:40:26 +0200 Subject: [PATCH 58/79] added flags to testclient, added optional getrepo call --- repo/repo.go | 2 +- testing/client/client.go | 54 ++++++++++++++++++++++++++++++---------- 2 files changed, 42 insertions(+), 14 deletions(-) diff --git a/repo/repo.go b/repo/repo.go index df9720e..152c94e 100644 --- a/repo/repo.go +++ b/repo/repo.go @@ -100,7 +100,7 @@ func (repo *Repo) getNodes(nodeRequests map[string]*requests.Node, env *requests for nodeName, nodeRequest := range nodeRequests { if nodeName == "" || nodeRequest.ID == "" { - Log.Error("invalid node request", zap.Error(errors.New("nodeName or nodeRequest.ID empty"))) + Log.Info("invalid node request", zap.Error(errors.New("nodeName or nodeRequest.ID empty"))) continue } Log.Debug("adding node", zap.String("name", nodeName), zap.String("requestID", nodeRequest.ID)) diff --git a/testing/client/client.go b/testing/client/client.go index 4e8b830..4e45469 100644 --- a/testing/client/client.go +++ b/testing/client/client.go @@ -1,6 +1,7 @@ package main import ( + "flag" "log" "time" @@ -8,24 +9,51 @@ import ( "github.com/foomo/contentserver/client" ) +// https://globus-b.stage.mzg.bestbytes.net/contentserverapi +var ( + flagAddr = flag.String("addr", "http://127.0.0.1:9191/contentserver", "set addr") + flagGetRepo = flag.Bool("getRepo", false, "get repo") + flagUpdate = flag.Bool("update", true, "trigger content update") + flagNum = flag.Int("num", 100, "num repititions") + flagDelay = flag.Int("delay", 2, "delay in seconds") +) + func main() { - serverAdr := "http://127.0.0.1:9191/contentserver" - c, errClient := client.NewHTTPClient(serverAdr) + + flag.Parse() + + c, errClient := client.NewHTTPClient(*flagAddr) if errClient != nil { log.Fatal(errClient) } - for i := 1; i <= 150; i++ { - go func(num int) { - log.Println("start update") - resp, errUpdate := c.Update() - if errUpdate != nil { - spew.Dump(resp) - log.Fatal(errUpdate) - } - log.Println(num, "update done", resp) - }(i) - time.Sleep(2 * time.Second) + for i := 1; i <= *flagNum; i++ { + + if *flagUpdate { + go func(num int) { + log.Println("start update") + resp, errUpdate := c.Update() + if errUpdate != nil { + spew.Dump(resp) + log.Fatal(errUpdate) + } + log.Println(num, "update done", resp) + }(i) + } + + if *flagGetRepo { + go func(num int) { + log.Println("get repo", num) + _, err := c.GetRepo() + if err != nil { + // spew.Dump(resp) + log.Fatal("failed to get repo") + } + log.Println(num, "get repo done") + }(i) + } + + time.Sleep(time.Duration(*flagDelay) * time.Second) } log.Println("done") From 3b4a55f18ed27b83f2defe40a5cf36abd7d69094 Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Fri, 24 May 2019 17:40:26 +0200 Subject: [PATCH 59/79] added flags to testclient, added optional getrepo call --- repo/repo.go | 2 +- testing/client/client.go | 54 ++++++++++++++++++++++++++++++---------- 2 files changed, 42 insertions(+), 14 deletions(-) diff --git a/repo/repo.go b/repo/repo.go index df9720e..152c94e 100644 --- a/repo/repo.go +++ b/repo/repo.go @@ -100,7 +100,7 @@ func (repo *Repo) getNodes(nodeRequests map[string]*requests.Node, env *requests for nodeName, nodeRequest := range nodeRequests { if nodeName == "" || nodeRequest.ID == "" { - Log.Error("invalid node request", zap.Error(errors.New("nodeName or nodeRequest.ID empty"))) + Log.Info("invalid node request", zap.Error(errors.New("nodeName or nodeRequest.ID empty"))) continue } Log.Debug("adding node", zap.String("name", nodeName), zap.String("requestID", nodeRequest.ID)) diff --git a/testing/client/client.go b/testing/client/client.go index 4e8b830..3641a66 100644 --- a/testing/client/client.go +++ b/testing/client/client.go @@ -1,6 +1,7 @@ package main import ( + "flag" "log" "time" @@ -8,24 +9,51 @@ import ( "github.com/foomo/contentserver/client" ) +// ***REMOVED*** +var ( + flagAddr = flag.String("addr", "http://127.0.0.1:9191/contentserver", "set addr") + flagGetRepo = flag.Bool("getRepo", false, "get repo") + flagUpdate = flag.Bool("update", true, "trigger content update") + flagNum = flag.Int("num", 100, "num repititions") + flagDelay = flag.Int("delay", 2, "delay in seconds") +) + func main() { - serverAdr := "http://127.0.0.1:9191/contentserver" - c, errClient := client.NewHTTPClient(serverAdr) + + flag.Parse() + + c, errClient := client.NewHTTPClient(*flagAddr) if errClient != nil { log.Fatal(errClient) } - for i := 1; i <= 150; i++ { - go func(num int) { - log.Println("start update") - resp, errUpdate := c.Update() - if errUpdate != nil { - spew.Dump(resp) - log.Fatal(errUpdate) - } - log.Println(num, "update done", resp) - }(i) - time.Sleep(2 * time.Second) + for i := 1; i <= *flagNum; i++ { + + if *flagUpdate { + go func(num int) { + log.Println("start update") + resp, errUpdate := c.Update() + if errUpdate != nil { + spew.Dump(resp) + log.Fatal(errUpdate) + } + log.Println(num, "update done", resp) + }(i) + } + + if *flagGetRepo { + go func(num int) { + log.Println("get repo", num) + _, err := c.GetRepo() + if err != nil { + // spew.Dump(resp) + log.Fatal("failed to get repo") + } + log.Println(num, "get repo done") + }(i) + } + + time.Sleep(time.Duration(*flagDelay) * time.Second) } log.Println("done") From 735a0ab3f8a977ccbc45eb0d7e1430a7530d1cd5 Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Fri, 24 May 2019 17:45:44 +0200 Subject: [PATCH 60/79] cleanup --- testing/client/client.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/testing/client/client.go b/testing/client/client.go index 4e45469..99ea991 100644 --- a/testing/client/client.go +++ b/testing/client/client.go @@ -9,7 +9,6 @@ import ( "github.com/foomo/contentserver/client" ) -// https://globus-b.stage.mzg.bestbytes.net/contentserverapi var ( flagAddr = flag.String("addr", "http://127.0.0.1:9191/contentserver", "set addr") flagGetRepo = flag.Bool("getRepo", false, "get repo") @@ -56,5 +55,5 @@ func main() { time.Sleep(time.Duration(*flagDelay) * time.Second) } - log.Println("done") + log.Println("done!") } From 9e8a0cb6d341d7bf1b61fc67c3499aa8ed278f5f Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Fri, 24 May 2019 17:45:44 +0200 Subject: [PATCH 61/79] cleanup --- testing/client/client.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/testing/client/client.go b/testing/client/client.go index 3641a66..99ea991 100644 --- a/testing/client/client.go +++ b/testing/client/client.go @@ -9,7 +9,6 @@ import ( "github.com/foomo/contentserver/client" ) -// ***REMOVED*** var ( flagAddr = flag.String("addr", "http://127.0.0.1:9191/contentserver", "set addr") flagGetRepo = flag.Bool("getRepo", false, "get repo") @@ -56,5 +55,5 @@ func main() { time.Sleep(time.Duration(*flagDelay) * time.Second) } - log.Println("done") + log.Println("done!") } From 1d3405cbf7d5c9383a0318f190ca7c96a4665a17 Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Mon, 27 May 2019 10:23:28 +0200 Subject: [PATCH 62/79] updated graphics and readme --- README.md | 28 ++++- contentserver.go | 2 +- contentserver.graffle | Bin 12585 -> 19879 bytes graphics/Overview.svg | 262 ++++++++++++++++++++++++++++++++++++++ graphics/Update-Flow.svg | 263 +++++++++++++++++++++++++++++++++++++++ server/socketserver.go | 18 ++- server/webserver.go | 5 + 7 files changed, 572 insertions(+), 6 deletions(-) create mode 100644 graphics/Overview.svg create mode 100644 graphics/Update-Flow.svg diff --git a/README.md b/README.md index f8e63e1..91d36ef 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,12 @@ A Server written in GoLang to mix and resolve content from different content sou It's up to you how you use it and which data you want to export to the server. Our intention was to write a fast and cache hazzle-free content server to mix different content sources. +### Overview + + + + + ## Export Data All you have to do is to provide a tree of content nodes as a JSON encoded RepoNode. @@ -39,19 +45,33 @@ All you have to do is to provide a tree of content nodes as a JSON encoded RepoN There is a PHP Proxy implementation for foomo in [Foomo.ContentServer](https://github.com/foomo/Foomo.ContentServer). Feel free to use it or to implement your own proxy in the language you love. The API should be easily to implement in every other framework and language, too. +## Update Flowchart + + + + + ### Usage ```bash -$ contentserver --help +$ contentserver -h Usage of contentserver: -address string - address to bind host:port (default "127.0.0.1:8081") - -log-level string - one of error, record, warning, notice, debug (default "record") + address to bind socket server host:port + -debug + toggle debug mode + -free-os-mem int + free OS mem every X minutes + -heap-dump int + dump heap every X minutes -var-dir string where to put my data (default "/var/lib/contentserver") -version version info + -webserver-address string + address to bind web server host:port, when empty no webserver will be spawned + -webserver-path string + path to export the webserver on - useful when behind a proxy (default "/contentserver") ``` ## License diff --git a/contentserver.go b/contentserver.go index 92bcd55..dd41799 100644 --- a/contentserver.go +++ b/contentserver.go @@ -30,7 +30,7 @@ const ( ) var ( - uniqushPushVersion = "content-server 1.5.0" + uniqushPushVersion = "content-server 1.6.0" flagShowVersionFlag = flag.Bool("version", false, "version info") flagAddress = flag.String("address", "", "address to bind socket server host:port") diff --git a/contentserver.graffle b/contentserver.graffle index 5176c775cda5cca5051cc30c260598b8eea99d64..9b9a0e04007a0d1492fb4b7b2e9f9ad19e855ad0 100644 GIT binary patch literal 19879 zcmdqoRcs_-nxJd5U1nxxW@eX}nVC7w%yyZXnVFfH*)H2h3vbW_xG$_DCl$ zLaB_9QjsA=DF5es2_m6@{_z3@y6m=g+mo6ne`|kNcYhn?y5`whKZ!Y6-E34+Clw_8 zMLM)RsF;-`Ff;uuSAh6R#Ym@oRsVPCKsbZP5}{jDEoVE5{C{g}SH8DWwfiaxBtW=gR%0TxC9 z0uzMj0|7<*vZEel&Ymnxbbux3i!H&)`lH&!L>!==8*dEKr=Ducec&&}4de6ynwv43 znp+>t0$JO|T=Bjy!y2OoTw9gEV5VGhf8(GyH${WJDRsBtqi|c(>A=Y%4H$?Z^9c!Q zWY&G8rIxdtVVzXQXF=MWnW(Jzk^#O>ZDh&Kyu+gLC`VFfl}LxA@sz~%Iy5b%b~|;B z-k@jD?r-<)FviY@vBXqJ?tBn2HcTb6S8WI7z_+$~&7s+b3}W7d?e;b6&I|BBRQ1_K zfi#!k2wz~%-c$|B;na}Cm4~Mg$_t0M3WM0IFN1z}x0O9C9-`qfpZFyQO+z2EU$I@^ zuMBx(xp*wJ`aKayksO?iKy36~klRF!rxi`*QH4;~ZxT+0mw6`lzah>cw6p zv+{@PkK5*VQb`GK~CJeMpMk7OLhKwv$i0AHPN}njA(;@N4A5jC zM8MjBgMt9ShN)-)O#uoM*n%^HT6NODwF6!H)INdU=vRPafI(q!q2<6M;Ohcm`V=5l zfQ4*L0^S47{m+=QCgl6v$CFMwzzINY2Vp^SfKURlDhGftk&P>U0?DL0fB=Ip1pa`o zM=(Q91T)e|HfKnWrcUTPOb=Sa1GO7Rv~mMKLUtmMz=eWV14@7-j=aP`22SDCE?I!q zgk&0+f%ZU{kAf4jgHi*|1n-8^(6-5cdM8}g1bJ9CC=~`Hc&NO4LZ>JX!o>xZqf0^( zWC5U4Li(fVpu)-uN^Wpt{yk}~B24hQhGg8$$_l7a5C&j2^ESBJfcpM0&@I)`=O~pM z$vjh;?Y(CMX2THPi{Npe^cY2?e3%C|mT@pyHrf$q@LEPuGqWF+eIi)u7z!@fEKbrU zAd@D<>a+_~;5J3>RBQcJ(0wfd6< zeL-B1KIL3*R|b#NuL{yV(JwF3bhksnQ*i)+%JAWHp(!2!foH{2;rE4Pb*n^xJhyu= z9>5q)(xnb!5o()q1DdgpgtYfthQd@&(}Z8wyL>V5huD&o{nR)M6Ia0!S7g!N3Xy0T8h+ROc_TGo~i);#; z5J4oOu^_*A>6-s!LwhL(glg%Z5WO*Fp$=#;qmD2JTN)&tCUmbNc^eGera(;=pE5p4 zXh@S|b0^h_mz=Fcf*Wz^hC0bww5PNo(6pn8Zbl4PP*8Wc5YU4vE}K8`V@1aB431(k zh^1G@s<*WfUwpcEBRy9rm*8C82TPdDkZ-K&61I2S*Sd7#aI6zRZpL4^2sf=~gD_z2 z>nt;P6OmTlbx0nK!n$m4(jR~1^jsTFdQ5G!c06kmTELQoi^kRAGDHimn1V}YoF#9V zX}M8ObHsbc!BHg^Jmz0Zind@jND;@M6@-ZePyL<@XD>P2G7A^=WScpVL|m`pO4jAL z!mUyXfg-@S(b`7ZBUnXrjuN2*s`(n%*ly4mpCLVjLSlEt;0itpm_z+?Os6OU?KMu3Wy~quB zG(v_Dvs9kCeG!S3S5DJ3#$dw1RRRA;R7RULIH>!?bg8fynKL~$EF|!_jFBs6@i_Bl zjP&^KKmqU7E#t=gyF;uSe^`rC)hKXcqmDf zooS?6Q3G_^txW+uUr8xUY(=;Ensm8>a{BZme+|C+Vq4BR2_powfLPO(yWqgC4d(u; zR9~tZs~qnwl0(Gp+{xP*t_9*48!5JWTcx|BdMri;-Xv@8u2#g;Q*mFOlz`?{!H-Vt zwn5l@+l$W= zCK2!iG(1IY3yT^!SWLu*pA?zMQ@rulkx6<};SP~aX@K96nCvJ^0g_IFWpj`CbNvjOgo~2n%VEWH z(-Qy;Bhyqwp+Q9i(?k0lgn*8MvstI3z(UH z)aCr1#_b&V@mg*A% z%^$h7*63E*Y`;Rtm!!`#*J7u<>uGmgzz&pIqp4s`wBr6GWc|E4%~bJb*R8g%!=LB9 zesLHO_8Oynpz#ak)4oMjL`l2!!nMBG(Y$d}iJy)u6#CfBxUta!rCYaTvvm#6YB>YM ztFU z13Ju?H#)8|t_#9blO6wa8zs5dUwYAYW+7~0Ni3T`h%j-SpJ4LDz+C3;6f?S%yu_W} zb5@hZG~V#2{>*Jn+i(SN3?@aiGrzWvHZ?R@n0xpDU8)t(V9N80|3|E_$IpY#U>5dPf*WXe04gsQ`omM{*1S5u+b}R-|HO7@X@)3itTQm(x@fw?$Kk`1#S&G zx=R~3lip+k?U^psRovd@oDH^Q2J^-S-S2^PWv?*qp38L!NKw9yeZ{s|w-0!(QEymL zs+S4d_pp6uh3|>;Kir++*nW!p3|1ea(Y?{BLC+PJ8m$#Va2rGRT)vu zkLAg-C;Jhb?rDC0MdZ6t-l~JW(nxM>lH539tv@75w)m#|qKY{*X>Gz0Z{IJZhX0~` zq6l5ARJydy=`KQBexg2LcFTOR{9*>sNI41IEc=O4=^v+wtEcA2uP*c9e!n`2jCU?eu9*N#_zg#AMZuSVD^ct<`E8e;KWa~faFUxhWeD6h2X zodk#}n=;Y&sj0M>{anghu3*13vV3cs@_+y}f-gxa*jXaW z3WZbZ({cIA;QK=+|L_Fo-PEQM0g@}Jkf7#$R1xt@``s_0`U`JW;CzO8x!%rekd@yX zpAisn8;ypub2K+|y$=)_z`5zx8mD^GeodduQ-I{+0o-vCo!X`XpOhm5C7r&CHchQgu$)X2r5`W0fXQ!w|CQ^Q{1Uc`aOVD;r*!neRCzM|jT!ePO-GVm+6_S<&~NKMV;5Q<-}Uq!>A z7oxc(W5c~d6hb9p5#YrJT`%S0?Gz@hoBHPNjunjKi*_=B0k^(%=7`wBHYsKA$ON^E zz3BUvM$&mFO=SJ@Jl-XKGSf0%|0<3{S=Tc3n3YmU5KEBg$dB5TDi5*4P6fSFb(MCD zoGPqvCq=N(uH^U5L4ndQdCNRJ)Ni~{HD{Mwkr~lURalDT8occ?lsEs4>buU~O;wEd z<;4mzTji_8Hi<9Tsao-Q8qD?~qy)-(U?F^Nk<6msu8{puAfwc%LOckZQmzIp3wXGq zOna$38SdfOt%m#MVbF|#{XsZ?cYtMrFECOo(_V;wk65^MLF>?qEmx0NL~Guj8kO0> z;clc-#jd|DY^1uvpN?Sg+?ogJh-3OMpX}MWZClEYMZYU*K_=w)&FPIPdVNLP2j|9B z?wcIJIDzl|XOEPlc9d^^ao^lhqz_kN94#p>$K}WU?A$@i*gdxeVIud8cy`-(j;`G=d=YZ|Z1&zx&X=z#A*A~hdH=ht9E zkw;OILyQ=8Quk8g0Fs6e;`j)#68c`sCgS|HxS}ZzE#XYx&X)acg3UW2J!~0Y?p9sS zPO-0#;|fTVp_Wbq%)z@rhe zmTfE%vtZfrMeDBPH}9u2jqd(3i~sHk4UkvzyY|X*Pq@z~t^Z==B-4v!%17HBY*(EU zy~DS(ui5++M&R7@z0S)K%v)?OXs$&n|huX>yV-I%aZcpU*X7@0IWm;Vx>xX^c) z@oX1l7YX%FrunpTIO^$LcB-cZ?t2lMjk|JAGg|GHstNjQ!4l1)u#oeoGgM$fonw9& zX&fXng=9tn5xXlC_VN>)tGztq6AZBhhiRNarWsarE{Y z)~hI*m9tu5BX;)UN!9ErZNb7xRcv{q!Ux5IW^T2@f9w!5_ko$+UxwsHewf8Ok=XCu z$S3{u*A+WdB#9zJ7TtD6hiCI-8!Ox`VRDNay< zoB&!YAy}Jr=(26vNRCMLQq>=VLixjUDMhxX-%0Z*4R+it*YhKCMp}o5Ay&`LLl*CY z=VfEV8~x6Yz0SRLvu8j2<|l5hTnF_m^5>8PDVyZjm@pm^qC`uB z5@>q*zNE(`DLrSRL(@Vp{W{va6@Qtj5<2M9v3Fo5-Feq%oM)G$|~O`lUH62Hc@A$MssEI&*Ge`CEm#?Wpv78C+p_i=+hX9CxMEU0IKQt(edEF;bD#el+w<0bHtVsh|v z|5b>->jYC4csbf!kAN&;y^Ax|F<6(RADbc)yS;llmO?__P6W%e@YDkbhh z$Qt031ggufH_p0=U^QbcL;APBo@h%jlpcH5FM@5`oj#WXu0_`mU|LsG8R8a*xRX zjgl`EGByp=YWqW#!pbv;TG)t&WmNLNZLAg(>7&ssQgz(b#-9EjH1?nS-8j^!VGe9O zvofzjz(HgYfCUS8H-PIz!B+}?r*N&9;d>dm+uM^gzp8f$E^eKiipl%Et=OY_S?q)P z#xM=kJ!wl5f(7s+qHdDUM09-SL}1&Yx)hdWF0A^!EUbPX3~;9!)_!>&*ZS$cS6(cC z+aZ&Ek+&u;et*}kn4BVnrEz@LZDTOiLoneC*&sFwO~nSK*}eMrFentjDXW9V9ur{$`N%wGb&<8$$>Pb1xMdf$W!$Z9F-CGAU<6xHp zjTC^ReQE@PF7bQTZ(vusYIxi<;eNf2we3`Ra49p#5g;P-`Rr^#q>@DLB_6JS7MU;; z%@dk{pz^~mV!e38!>Ined)&S1y(sosj%Gic_WS09Ddm_|c&mXab#H|I-Dk4@@OGQ6 z$FU^4dFcs9@CnpUhWWHtS~9u4Lpn*IQ;5WQP0oZXbduN#>*2G)D9P)*bATJ1^S?d>V%r@!>xp!!fV{bHB6v@J4#uRHKxEBKs$Qbtv^HZPL({`(a;hJatMKEdgHuOs|TyvCS!XKum@Mfa#gAxbNt4!1cc zU8#NSL=@A|{i8qwu?^j!BHRdgfhRj;R?SGrLIOIBJWE;4a}u`3Js&SeFc@U~Hn|8r zf$EpHuSf>d81O*6ebh ztT?6o-hDTd$+M2apS2d5p5KE^8Mzj=akWA{xe%{sb+J5j>ovOHGY{F0ZYl@9Ho}M` zJhgirzP5!32g6lqBK55xofdo9eGRXR#SpZatGBTNJZ7O2ztqVQ4u6mmU}GwRv`6@# z3ogVJts#)kk@JwQ3Ej9teqYr|uVvX&e;<$ngw1$DSv2GV}cIiEbtGm}=57PlLaKAYm-$+Zd z^{==uXE1W)qML7@13%3|*1{}FO6)$~AVPABo+f>L_VJ2aG#>y(PZTL@+geYawz&;u zQpn5k6vg)!T*okg97e?;cZL~w8uU@lXK?r$@(v7`U`ExNZ!2GqD8$2kRq?58DHxpbC9OGGK(;d9iLOL(4uK99ds~^MKZzURk$NEO;%7oI4zm$ z{O#QD@$jC96)Jf-m8wPudk@C~XHi2Y7K1f1?;sp;`HKVdhy@;N&1I93mzl@Q*uN`V-V93L9uY$yb8_K7k>GU%H`w~?vTt{(`~*g{BM|Tw0|vQLONXC zl|t&D(3UBJJ4(;)en!Y&EherIlY~;-up?<|co-D6(t9F1En11>jz9hegd>xZhD;Rt z75LqObcbFH4zoQqlcV_2KC_TdNY^SjIhCUB!J=4@L*?`{ z6aoE7b*oD4hz)wz@3lmkWYDI9HU}nayp9f}lSfR4e*Mt_;Tis*qJ;IYw87}rxo7W* zc*wc24EO%dk)L<{Yi%k!|6y$^(xZaLFX>z`L<7VL3Xk^#{ZBl0)Zl<%15duQH*6=` z_(#zI_1C2&#^>2XFEK4Xcam*w7eA(m_dwj)ysxzM1EUhAqwsllxDEF4rX&!G|A*QF zzYp$Lsz*Pnk*5A)o7-P(duqQD3y0F^%Ers%H9DBjiOJU^0a+)d#-c7hrWNNfJ(=Dz zsY}>?8#ZAh_9y(UP5DFu9J7jIiZcK^2>S*^3DAN=+_kJ4#D%CbT=Y0wly;LdvapfO z)rKz1#h30@d<1WPyilMJJjAvnQxw|#;~5yUkg7-BgJhsWSkYH|ZvvSIISg)(tN{I+ z-g1Y5w;A-yl>|o_}z?R;jFzPXH2F3=o7(xKc|M8H6GOL0HHIRrZhX!WU9r*!< zDg#4KImDkPITg~w)ZXJL&vmpZHvAH zn8sa|+8B2jlB?R7SDe{JI5Su`kp{R8`UAUxZGG_Cb&!tvkKg)1+3?MydHTogt(=m0 z2RoSdR{2pt84ittTtHbn1qK!%XF6~HiEqGdUaDxjVY_5`N3gBCY>e!3Nl~0Tx-QE)!KW8n}9C&+mI#IJNmw^gX^~v@D6BF`xcla586V}Oax{Oo<2fn& z7RA2ueJq^TvxIC~o9_>{W!?r$rxf|j&EILQGyes*+-ENI9hT?@gXlIgonVB zUBs7jKN$sR$erI}&9LPkYVGm=V($Tn+W#c_uL74#DLR_HyB~#7IJGoNqoafLYW3rLN+)#C2(-Gw3g@jn<`^nKt6 zV}b_QkICD4s9(BjaYg0$|F6MWb0siSX%W2TsF*|&Rgxl?(??jSjKOwiSr}_w*;;`O z66>Z=2-!3qskm;oIn^m)ou%Fw?BB6Vh%Qyq(O}Kaj}C5J0+Zt44fxbjm{NA4dL8Rp*=&MMQenJaHY5Q}ih!50@f2tD>8 zW?v)z2iX2Ycmat;w96<}jZtEYtNCH8SUFPXT6qSlhcBe6fUSQ`45O&?v}jAswuXx(~rQdct|{?�>m=;T#E zQB`XqQ=65c*z%=3KX7e$67#@ zh~=_yOgWp2Ju34yLvPm7jN?}jy=4pR8gKDD?u7!D)~NSF@Tk7N6>;k(1d23FO{YQt z+4M#GQjBKVioMn5v9#)+hgHTH@sjdsZw9P%SHAOf+d*fOx5QPStmwM2rqK7~2(lJF z41Bg*2f%fg%cj%{d$V?SbO4@f4E9kOPI}N zolO_HFmN|~a<0$8IB;^BjQKJ}CZP+FQ%nqpU{kGj_ky*YRdG6TwG#+T%)6)aTY;5R zPPX{8oSRf*#IhPo$(eEjrjwk#m;iD=xNu27EyyP&t4$ML`&$m1;|cl9WzhlDWPnhS z)bizf)%3LzFB{*2s4?{9a7q8iUx1^P_>UEq#sNL|*g_Msl}WzhW%iW`Ipo{9t6Eo$ zGv1jX@%+?FFX@jtIsDG<(K+L}%J@*E*0@buBk7WPGA~o^EXTPxJL^|9o5Huk9aAlzt6f9a znoIid>@x?lSitCJBfUq{gW`KII=uN(*v0&=>4$DdcAq3#I}3wm^8J!RGk@{YqIYL) z{ke}3UdN)m0_Kw|k-9=&akz{^eajJSQ`tP?jpn@6;{Lz7+@jV9?~H|AuwiF53T3dr zv~{dHBML&SZ^{Bqmepc)k%b^qLZNzLx}?d7{y={oR?YG~g`Vm*FXA%WTe(X)|b!RPC+N z5_k3`F|`qY+`cBlVr=PTf(LSWLNgr<6F&@C0|*^B2V*R)nuN=E>v7 zfMt5Gx|!@Ly zp+2h4;VoqPbfgWMk3&Z49kY(OZ$_`g>k@jPa)d4ZpjMB@HD4ePbB5i57;TQDO+vIiggl-D=Q`S)likryvm-} zHp=`O%IRQ)m3peD2K)*bwtR{jTng4n=deaMNoKnSV@v(Nt8G0YAvq$x8WK}avq-Nx z!sDhAe5zEeQRFS%t-0Y}j~%gECk1_ztZG!=B`CI%sPc}4fe8OjnS|62xj9m3LKy38 zc`R4~AL{f*m?_;ActK8Pk;RmOs`*g=9XYT7Zf0TTc_ld%$YqmRB@Z-|EU3$_kIGlI zJSlZtG$G&~H~TwwRL^2c-3@z=2wFpIl9uNxmcu(boS3Oxw&Ym>4QnL}US!o>YSmu> z3PHxDzUkIijzF_b?rn42;k{&kBhAeO=7j7R2scwvPaPd(r}O>CPToW1@A*z%HK_Nx z&lKvXZSI-k+lWJ1-u$7*=(_l6{l$&&Mjfl%!A!u2^u+3(9@^Jddv740lfK*q^r417 zc&m&*=Jc(3W2^8~ugcz~`y|%dWnO&8Pj&6J$OnAZoFe=xLzU~G`FhcF*6+`Eul7i| z4W8dro1#Bd`gTjpiC})iDyQe0bN;>FxPWR_oU<-nWLW8af!CUzntU0h5P=yiPau_*TK!=Ud3N@JCg-dv01Pf=sUZ z`|n$g%`o!V`!!D|!K;Vie&^-*UC5p~){8#0y5>)*W7}DdFMX>va`wSBRqmA&h3fNX z!V#Bz2?)wp2cP`YMKIgD4KSQ?FDs~k;X-sH$dIDF^l?6iGd`)GX+lnuHCD%0l}@E# z86tI=WYYG0%%HBvi2D>I!NXe7f11`?};&%3c#*j5c!23A8ZZZTRv%vP~i!T-UyPykA zv5sK!ej4kG74ODEc&)>K;LRWVNlj~a-?w%;v$avST=`Km@UUf7yw2oVYsOGZGOh=y z*BwLR#S(SlqO*KX*kGsDSGjr+iP^9X7*6w{54=zYHc@XZw)QZ*h!&V^IDiSgE^o|* zr{5AR{P4WU4W*1hJ_Jrb5zq%6ql)EFChpxJvMZS{*oUiogNz_L^vJRwxGPU8t}@H{ z4%QelyJeUBhI-0T@@D!N@F2?az}1sGvwoU`JOCtbe_qyftn}8X(V1s{Ek4{WD1ULz zMD$#~FtyI8<);7X#fs6M!4*J?qA@v*9Dk;p^T6ZtA`!V4n>tGmC#3YlJun@!lGOYM zi3;sljK9e`_^#D^IkB8S=6S2BF9#IW5|x~~d1^4%JHq@s?OUkAozBpFNI2A}DD%3)v zw-m@{A66exqS_Tws5d@y4gD=8{eIWNP3+4jxTf{!M9sf%a!WSHYL@{gvn891Z)jge zwtIz^t+i?c+X|gN4$?m0>(Q>(P3x_{0V0K)U;EALy@>e%A0}&XMAAc|r%#~^=K7Vc zz1jTlD(l|hh0ze!tSha*4P?Q@3;TQc1A|UJbg-)yh-!CIsh52fz+NrhE)Xc!>`MB- z1lPTh{`2@>;wZa9e%%n(-zM0wKGE}-5#CMiv6(hTq<1-Q7n9m&G%J86?ZV9-QRlJc z+FYNrP`ou%7h~L9-w9xv)-z#z;!J$>-U=(4}8Ot^CsDq z3Ct5r-;8p9;|~%rg!0UnR1(;d@E};=oaS-DkwKEkLymN%?rwjWX&b+AgnY-^y?oEs z_fFssSb*x|*AA0Qrwd=%eZ65iHgJ3$_4-12)kYU? z4{r@(_gMqg0(TMM-(=aMdf|$6-(BvG_VGsK^@8r@!|kz!Efa1b#JZR3Q>Gsaq#-_t zJ$2~7eBRWBs+jZa4r!m`a^3}4ln3C-_N(NmRzx4 z+^D+NH$J~&x4*q11t_X~EIoVVzPEe7U!VQhaV#?6ShvkU;hrA74sLU8LE*&y@&tq1 zrQGp8p32hKVNFoa`M620O*DgH^(y6pP0}j>$5i2LyFn2`RH2KP4_&;cx3PGYYC(nu zyY_x|&n1Rso#ygAwkKqZi2HE25bWy{xNent0muN3JhWYX;F8;Gg$4a&~+O_N=|xQ3)?;0sqfEWawh&rWu1`b~K^0QNPcX-x~Nd zOGFh^4w)lTB=ynVDA-1_3cf)h1fXyN5v231U!>vJX2}M!GVUayvNA97=qNp(2KhU( zynD0Au`iFknGx{<>b%54NCZrn^+uI%i>+>vq8}pC-oVv>Q{2^5lsERjbYP%n27pSe#!<(efUtWZSS-Iaul zN{g%uopB0Czovace-Zyz2NIWZsoe^Um;kwqkc!5<5w z!X3+i*=)g!xga*lsu6LszWLAL{TlG)mKpA0Df6#g(R_V8zl|3g?A+=17n>tQ`WEYN3dcZo zhZ)`8KxOeY!25*O+m;PPSW?NH@F2 zMaoh!E+5dtKVQ$6GQ98e+b-tBYqZAsXv^Rs(O6U(8J>`xqV1Lm#gY$~4wpEBYF zpZ!fmh z(jBYf_q)c-BPhGWP&XGF{M(F#0-m~DEi$OLr82yU9aw}aV#%yM8TCwG&s~ z5)8+v4}nS!Y?c2mFg7-qFl#0PY3>KKqo$;rjS#W?Q8ea3i4GzeSn9Q;#%=NhN$v2!+?U(kaSJa0MckdpJ*AU9UtdB&2qXWeZg$$f2|)?@GR)}GIpwx zOcJ?G0|M$py%DHc)Y9rx#sYJGbn0vaMgc(wMoA72CNxYMdK$R}d$R7pMJ37Vv4y8x0q|E)j~ugj3REASb< z&=(D=#`;od<{eJFGUQPp-3z>@l#T3RKt&a9RFhm%!%FzYkcPXiDS7IgghtyP^1eZx z@lbyNguezPlIj0dZtqYrg3;@eRwPHog()chc5J1*T#Ek~XgcI3oInd6p7OU$y@3PZ z^3^BiQpsIcnd*|*5OlN=BWNi6O|l?p>zC0F-CTO{$^`ioL15~D?a4s|VTqip8=Pg; zIU5f^%zwhL9H1O(fam}@Cr^@P;@Ihc5>b4b;CdD??k#qM0byoQvibHpB4-@#l8llf zainxw4QZ6bgTu-KA^LO614V{*0ZppJ(8tP+Fm(kG$Z~L&S6Ud0fS6BU{6`)+2Au4S z4rZsY_2Ql4XtO)j?^VKDQ97D#fm_YYI79tv!j8{RFGbDRJEq*4@bSP}W>xr)IC3(r z6|Hven{#c6`WQvdp7e8bjo|M3LuV%uk-D&rxRf&Nif_cIW`;zuC zc@@|Qry2gMmaGIO=8bkq8}w(Z;+F6BJe`q*jjld*k_LWUFfQk)=*erz!8-#h(;=xQ zr7p|VlK<`SHZMKQ0TQJ(!ualza{=^_*9Ig9A_crgcBx_N>0g=;ZD1xycs5@hI6jeQ z&S7Bxn~v~LBXJx%VwWc97?uP~OA6;dn@N1ZZEzl-=*R$Yc<@pXQAiD7)w925D~pE6 zD~{gA;0{qnKUn?&7L_W~x?`MYnLyyKpWi^5%xnj z$O=v&>o?e#I22N1xHmQ2e=l&LJP4if8+tEf;t!SsLk8I^EHos!B;CB)bV+iH3PYw6 z8uwr-W)^7;aPo<^4*!aUwizctxQs!%%t3?{Fb;!Wge#*pizZP?MYl0~UduV+q9!9x z_ay376qij0$s!ID=JumM1M@*cCxmb1Oe2Ylq>Egg0f|fGPTyUM3B+znGAR{&fq09& zgiB68v8gyt`hVL^X2`qtoZeccG{6*@`!AAY-gblW0#>892K(Q}&@(#@iw%SuD7RY| z=?up6dVzVuyN}-B9-#?DOkS zQs!fk_8)Gp7!m-64eQOTk?E&VA-5v0O~gyr(mv@CcRD=X-kab!uual?VoLJA z1(eK5x)I%ybqxsN%lCc8(XfgkPYF9rTvg)TKecESd(Ps=u;-xjXX!ZPoWrbitJ=6m z3uOGP4r3vCFG81G2b665E&=OE-4})CJSOGM9iQlmlqhB_C*|O-2h9OFmqEc2a7Xj` zt#KI5({*dO3V=LV#B)vo_;8^}anDhQsxh!Yhc}Tqq^6OM{Up^Rs1jUr$iw))vhYq7 zM*q?cZ*CKnpT`AjH8}7f3F$)@_EL#F^kkthk#}vQgZ9#`?}^@gMGdae_d4-0BH4Nl zqr(!3+)H3E;{0u?oXnVffnI@y*5^jQZmVtPB2?6XS#o6mbz;h{CJBjXPqrE1Y@{i9)vJT)Oti z=NyQbfx$%~+gd1!OjK37VrzoglaBQ$t%?n2^VNMZwbh#q<|xU+AG4m+D4EWj$Eg-W zZsyPN3;r1~4U@IOPpM(#^&FVkj2F&Ux2+BUnN8+e^~^{gJdZe>lk|`TMN%vK(or^F z`~9%mrWAc9hT#PGl}U4K>E?j$QK%}nfetF)-;DAX6a<$dey8V=CWMqAeYa}NrW@aa zH4T^mTEROE1u=IpBXFi|Sh=)z_%gn;wt zQs>15bp@(fqG>rNhn?+)#glZMDyB=owf1Y@_}0KDI$DQt0&J0yFI8%Pvy>G_! z6E8-*IF(?7!$5__NXytW$9g6F;Wp~EVbowbxK@PWkPeHH zfkKA=#VHT5bQue9j(BaL{e!dPh6%?0`#kdHHq!I~YB8X@Z2KqwA(aR2wA^Vh>*FlU z6ryI7SwD}fN~G9;UY}WMVfKXoGZ|xn#T92X3x#{0RZD0+aZuN#-Wa9^Z6>5$eoa^T z^nxt&h!YMilor%2GrQIfF*V(EhgKj- zF7+9IN0qpd-fDx>QQ&vDNcI5vqT_ADZ?>n4&T9KKgudb2<%mac>Lw>!u#KEsm+41& zHvl!WH*cghJk4KSD{~F8SY$9sMMv{2mHW>#&}lsvk@KyrwS{baq@qp|CISj93qxb+ zw_k--dU$xi<%X2Lhp;FF{g7Hgz1}8Or3U#Vij{x2RvMfT6 zrNA6l#HfjATx?+>K-8A8=({9!XgM-n1|f=`j;TWi$KU@Sl-t;QWFBuNqaUsydJW_j zlM%WtjCT`X*QT$F`EM!&MrWmMX}yZ_0qk^o_c=Am`CGbnn2mYagS%@%g8-1)Q&?}; zYP*-0Uz8+#2Mwc|M1V2d-a*SN;rKn!`B%0@eX#pXAL+Ka-@9oey&JND#^jYqZgj$> z3p|7$6!{JD8Eh4mRH?IVKAYV-uj%g4`LOZn`ld}^&&bSAsRO}=j~#V?OjgLjXWUA3 zWXMw$^R?R(cjI#d%Up6Ay#Ec0W9!S)GP%`!+aSj=+FNiR`FFRqV>YUY8pl+Mr(QRD zBw4Y)d}%%I%al!Ba;s7Ir6^#znt8X^%jlhr4+>WL!&9e9St%|ns3EbnTZfr90mCb=J#-Mn(@Z$wOaGr$KHAG-XUCP z%ZxufK$Idihzn`$RB1#6zPQ2AbB|Q+!29a zkXiCu`wfR9{sI=RWKv&ybfNAK^q!I`9MBj9c8qIwzi&@3i+ctiS{4@%mfrR~0b&?m z91c6Antg0*?;bdw-}h_auHq7%1rE-7z}M*aEM9#AZP9eH=uQ-{FdaNEdNXNp5?U?# zv7v?cs$Y=Wcu%JL<)qD;IKlW5^$+1IE5(X2e$R!!(mu^kV)47t1A4x>`yL%jdLRA~ z{K&QFSpn4r7c--t%Mn$k(D22};L?AIHYBHO(%r{lXcAu<77Tsn!7kR)dt)GzPFl$) z373c;cAnOEUQn^D)0C0Rkr__7n^v9@dg_8o$6@6RHxeeT- zpJZrOGP1t?2`;t%Uei@vYa7b2loDilyTQa-$nXEB-D*RxY7J@)pN!uhAjPV$VQRi) z4Kltvf(%~d+hXCn;2I7N{cT;1Q2rZjucn?{LhtYPI!&r9LXJ&W1tHX116u-*9YP>fXxu3@64ca$|pO8H>#d6J;qp-fbR8^P4) zqP#F0(TqO)Ui(fEeY$G*U>@pViCebQLROR3Qq-tZ=DDuwj!Gsc;pJ7?NMo5&ec%Fr zr98*ys=Z_Yyv6<+@K=RZNBPv%Goi5uEc5plexmgQwkDQz)ptaMNI+*oDRu0nJONeC z5{_<3w>kq?8t41ZVjLZ<7$-Dta2qQuM{mYjpWW3vAK?E#O@p78apf?beN%B*%!sGz`~e$4gX literal 12585 zcmb7~Q*b8U)~921Y}>YN+vuR*ByVinww;b`+jcrmI<{@h|D3O;YUZmtgMIO|9UTSP?k`7LntvC% z@}j}#HkI(yqC8vY*=Bjg6;8DVjrfI@_{FN%C?$;Fr(qbi^+gW*ebNh&P`=Wb5&~5V z>PFmOzmKA+HA7FK&sx8)mvg^Dszv=g))<8opa-BLo-Eh$<3X+xk~3@0qWOg9Z+fh| z$Cf|4E`#Jp`E63#aaM1$u7nCd4#i?pia3SQa%Rc+h|K*y4C*^1DgRpgCz^>!i2QK%hU+RVT^fpytd#e~@wbgK^vJkYeN$4#`hhd7YVO|VZNI>(A zu^2dI9ay_cI*>kXGekVLGZpwQ77nm_QDfrQqMGsuxhH!NePWQ`LG=*dmV4zxjo8kD zaK@k?W0A7=;3~N!{O(zkGe&Dpg=Y~2ckmv^$&S4wh<0;iB& z3v%saRYQ#%jRk=^8qAS|hsR#6i9hQ_?aEUY+*70{P#;q%3^b0gfZ1^J_9gWPbAKDU zD%O=SZv?ds4yGnpBU=F+RYviNP{V5~*z1R;y?c1+;2c~Nlc`=7dA}?xfFjta>fYd4 zYAcfC0W~kN)%KKfrUE~gJNHrTBDPXbYmJBzx-6OKSvJUxC5-j=aKOh29QP5F*oI85 zI#^3v6ci<_f28bSm%fR@xJzzk!7Z=>R_$#uHTPp<@}T`ADp}knZ?RV3NopRwFZRQ< z_Q|!}&5P9rJMM>VU`#6`9MiIjvW%zl0xwjvZ~cxWHHNG7ldhd3Mxoly4MfeC zEk5L&OD`7tIkDZDh}V6FMnav2f+w@b?T%j;6+H6iVdMIvl1{!{sWN8e3<7$5#1w?H zn)|Js^@CdmH#JL?0t9NC3G%tgeI*9<6(<_**K;4?1{}V27F)i*1r5&k!ZBz+^(D_7 z%2CdX%fU@Uq~ZOO$Mg`+k2RwpN z0_xK>_JP0Jtju4CrUTqY0)}L9&i7u;BEeHHFL%^)8kD~~=CI=r+#&qUhp!;xx)*dD z<;!`MD|n2mVc-!Shvo7>P76KQvo#It+jI!6?A4Xs6~(Kydq*nyBtetmO8k>Ss6e5W z0nEV_h~|XhkSD3iY!UciC0e`X$`PPZ;N8S%eI$L3_(2Kb zC>i?`3fd_4QEkr6mPJ6c2EgKkX5;vGOyhkz=n-MfQ zBLb(|wwzgtb1cLl};^LmAl<^k*tu(ALw6M&$ z$G}8lMN;PQddNtET+ZJ%Tv^>8QL<6-`ltA(XSk<&`Vmx^bx8yMY@4QO@;?|;qXjbh z6OyMm+O?oej2XrOoawSp$wTP@2-!d4$OA=wWMi%!LkBkv(8S*c4G5@4&sf#87P{vm z{q&QIu_ts`Vd0n#MvVg%k)B2~HV6i7h6#2-flSVTDJ4ZL*WffMA}?7UQ-;i9NMRm? zv38>%OraU%lvYwth;ip4qfW-bSwK_{=9}otEzO)NqrqTRpa7I}Yd;8=e;lm~j67gR zvz$5?P>7@Jqt98Au7hOFNfKj9Y~*WQ1=HLR5I`hR5bIJ9c8^oGZ={?O2o@I^NRv%D z?>_Od=PNZ9a1>{Sh))X-F#}}s5)>rIicsyi3bCftGgzQsluoCoaPaxbG6;Y{!43>M zHxQ{2j~$ju8zw?06LC zeP*|v3JSQo`wr1fE$|C)fVz`U^v2vmEXivn(>C_|%yMj3#2FM@2l2H${48vCcJLr} z2;D*a*e05e+%eE{frkOYODe%3cr(hG54Po$BB;r}9_ujrwWHv#Led z9CR|sJ!Zz`v-Vgo0ry7CC6I1S0qVJRf}Nvh%ebYV=G z(kB~pnWw;}v}gs;j%>}9*G229>iiY~pt|nY=1n0vcuvU-Fqh<8gj&nU#N!Vq1fgPr zG`GsMJE2Z)0DLkp^B{&BG-vGgjgxf6Fvz?}-yR%P>BuoizJj2shdAFuEpfV1nq+*2 z2X8NHFEXX=>2gF4+3KGw*N|lFYtW8X%$B<$+*FPKY?S4{7#agm@U&Uguo>Kr?&@mC zajl^gkM3d5D^>o1AZo=nanB4X@F&Bro~rbyBrbE)Ro?s2KlJC0+SZ|ptrZLf{4(8u zfwCx=u1P32`e$Hv*rLfinQWH;0a{Y?R{iOvL&Ld?)IEQWbE0h+;~q`v1LuQ6 z;AmGlcqYF>8^xA0TqgJ5`x5LwvdavH1>$ukWzw0e2g=5+i4Me_poY@*!iN)O3^|-B z2DQmF>7WT2BzN|ll)rf^&#c>xHv+C^NwYRc%~<%#!J&0Pac~`RZhH)~2V1Sd zgb*e*9byv@PQV657E-UiJ6W=|1OkAJum$)=_w7FE>iadwogS7-pntHAg1K!fi8Z0B zn#6DvRmOFK5lKy_C+J++l@a&60~A{{@E3GxRjHLUShT{bA}dmV&z~%)w&pKO*+^0Z zL(X(~+4!GHu`{zXR4pU%apLsJ0nkdyLd(p`t_C5ibq(AXFdJ<6to5|r@kUMfJ|*T2 zPc0iGy_|n{&`R1op7v(}vTJMA4VObxd#F}WIdx8^`Qy2DbQA~dD=h97>I*IL*~JX5 zy4)*x?oHX`OVVp`k_IIQNC&m+^)fbUqPufu>VH}{y=6oaJZpTAdbH0N2cN?7fuFM; z1+4PvTbZeN><%|sO>>3WvrtzN3)$|mSmc{i9NQB2%P5eR1|+U_MD8iGdL;p-1FF}4 zq5v_sW^jnYjqzb)0d8iuEs?X?zm}IayeuW^1>79&G=FR;VRE-e2AiQgFVlH1lR7)@ zAo?WAt}Zs$O>D=$=3iWy>JC#8Ao1d-<<$xd8@ILz-tLiv?P;!%W%7@E;rVT%jJT`E z1ofW5r+M=}C@oUUKXr3(;Six6)ET_}f}<*MS3XS&#ht$%4fgVX&p857G`l=A)QcJS8ViQ)&=dl|wYtOcq^pN=h?N zIB1oSH*s!l7~1*}ty*zFf|1h%I>JKTRdf}xxrllB$raZ|n~d8T-5Vf|(#j#3C4iZ= zkwC{l?3f-j#T4{GasoYPX!tDc8=}Kl{@F)8l8h z>fWopK_5~sy}!P|ZCwy6d01}Qu?VUh%OzA`b0duK50YQ}u2G$g7k|1$Vf6wt# zTwxOPIO$%S0n)ueC_(3uc2n$$MGvwQ+^AwX4~CaQ*96vFebZzH;V;)i3qNtvF>l{7 zAQ)nmnJz{#GLwcFT5Fjdq=lU^5QNpY^SAnY*o$zZUI# zv1j`en|unvJh|5^o=%ym?n6enO_aD|ZI{%W{plTORQoyG=hR0~8P-!Xwh6jPKCP2k zJo`qPD~4JQ(t@W!w%6+BYP=T=l3|)w+*bs9e@ZAwK5t<(O5jR~n(NIbaX81Zw-tjb zd=#vfRP-!1t!q6^&dXX4+Ilz83N?K#Gb%MPoIch)6<5^Tv#kNzGa9U=GsuqwgDmla zk?Y+(e=Ybfd;U;qrU&JlLu=~%gl3r*JlizM(M@BYCo_B)!xFfYi1Yq`y#73lGO&}B z&9P)dw>g;2X7px;xcm-8?lm$8AN~#{_9MHIL<`6}?^c)Fy=@eBbw>Jn-K2k4x%Si{0)Eznpu$2#t`r^7iGD zSKWTxpM}aBKq1e6f?5Oi4P<4W56`=4CB?$TbEhfV24OTRmJ`yq^doPV!Q-5?z|5z5OFLl%;@hSLw zx@XgL?^L)k>iZ2|Qv5WX?PymCo`OG9*t>Z%>%H<`u`i^=X&CeobxmJBgFl!%w0j$e zFd_i(K8`@BbM(88jxJ2PtG~pr!T%Z#B3_v}2!#41uUqM)?+x-ty*3hV4o2-cD~`iS z2*pU&`wi%H9PDxVbU$54#BBYy94o7aq*l5s|F_=BIz3E*X^M9w6i#!OHH|#WoYZP*t6*#aH22P`=CHD?Gc)Tu7%7+48-Q;h_rh3YpXR7`mq0mxXBR7 zr3|>&z~0nP8*@62_gh?TAPkndP>k8!AJ`uOuRrV_3SY@)pz<|!vcono%zb2WN|izN zDS69<&U!LFQ3;J%;}Lbo)A)XZ3wV^xxp1-LIFFp@Y#W zDmXw7q-)xD!;SiDWsQ1OfF0c9W#?LW{cFs=SjA%)h>wjP@@+jB-_hX?Rr+a-Hh@uL z8tk?nEJzZgu=OHU*umXpqbcyG;chsF?g5MLQ!{C6HJPq(fpRB2UmTil8ft4eH#`r| zrs`h2ixcs=dXTMnf^m}z-0h-IBkajgC-A*wXNF(@)!3a6p;n~E0KFa56Ve5W%}(Sj zUo1DivfeAewHJoB8!l!m#HU%e`*eriWm@a=Z6dJSDsPKRxfZ4iU~>I<(K~Zkz2svG zdma5)dHRw-uiHQ^_FkMTC{UsCl3u+c{+t5eeaMIouPbXstnWeptg!WZ%VxcoyK zalZ*KR2{EWxf-^e3hZNMmBIy3Ato7`#dgBQJY4sRX;-TA!hAd{l>-0}x8n)~RE$hh z_Q!qKP7g(X;w~>nNs<&IIkWGlaUSa($GQ5vwXe^ll$79*f@^^ja6 zvyhwvyd{-RMeDl_y zP~zs$5p`%y?#ln7Dv{J z3r{_W+3s)ayFPH+e*1%$d0&l;0XIYK6xEsltoMzaKQ}h!)0JT)F^0IlTQ`j(ENjIv zi{qTI>QaA@K0jWZ$eyF|zZ_x_$XqNBDsw$BcD&<4d&GHL5g4+Wt+8VoI#go*EY-vJ zB-PR$olekx17Nm1CXimwQ&Zx43_@@9TIuQA^TKXROuxzaB+XFYmcOl&i5bWqVyY6T z5XcmFl#63tP+|2sLg@f6>_ zE6B|fgY6E>JbS7-$R-+@4Ji5=bvl-UohbB8*sJb&AIc*Q{A|-ZyNq#ItZCt!a7)xO z%g8tRIMoTmjd<(8aa;0uH3>!OOgHKToi0{>i15Ytz0Q+m&u@F;U(j7WS;;{zZIrQh z8&lc0gHSmLVC?M|v+al@8HoT=f;qEy4j0=9z)n`IRKY&h1isof)25Xd zi}>4CTi=avhejB0_u+Wh3RkKb{Iz+TthmIh-eg1>GEMOA^zPh-WFkxJPVM!?U3oDm z*cP1#PYjKqF~iUwZ92pb9@kIw8RoZ7Vf$P^tZ1t)*L-f&Qb&XV{J5X~34Zp^G{VdtM9>r3h`gdvyZTh`Ej(Au z*U?tkM1}!E-ot}h{NBR=6m2j6wCVQij)2;)hf6t{PT#A`sk3K( zO&L^*>bG0#qarV=?#=Fd_pX}H&*#$G!KYqBLb9I0Z`IHD>sh*KyF-WX-m7EUY2Vk6 z?+pyUh!dFW#|nZm@#Bw*`$O=`9o%<33E4}jqKEM0FKX2{Gl$te*k{Enz0i}3?Ad6q zEjVr8+QNtrVQ1tQEQ$KF%0G0y%<#26p3<*NwpzrOzL0_Y7%&0q#K@f=XTLF|OJilM zg<@j7({Oi7s~>&i7Z~@GFz!(OKiV%NH9AA1x=eWN(ntSs#S-Qf6 z){v4TYKw(W{f2;rANQ@I|R0SzrUp^Y=1Qv!rmT94O~skT5%Y=N)D^R8vOu%2s@B{DzogZP7C4}g}<@M*^>k-uWnRsF7WVk6FIOLQ^ z`tAcpI@0PC)g~FGQT1JTo_)|uFo^=Et>3_mV$vaR^lAPqKxwq|4k*YzThSCgr`cj5 zwzi2S#IKM6;X9e1YG`Q}yC7}9OJIq;sNv=~ZcSKDyfFvxUJ@`;j-U(p#oX-!ydVDy z?nmIO807oEBOc_;5{b?{aC$Tu5-(LeG#xA^8T8S0Cnrr^JXKNW_nk$S6iB_cFIMXo zYUj5u>boDdFMZ-TMl2Wms-&D@mHpl;rsr|i5=(v7n2m>|N2qwO>MC{T(;PgE%9fUi z+Pk2Jzr1TOltKQVY@edEygU~&Us_oPc{RI(QZJXdm0&7rX_d z^lY?ontkKkz{o84&`^qNTX9J?$uNglxRn6W`^)Z<`2?n}#g0T$3JYg0bovR|=zQku z3j2Y8G-hmpUwgh%Bt&QH0(Ik+QVasZ7h~}~e0#BlD>3~)fd5xNg3Ur)5u8aZU7eB5 z=3L1^;__mU-(8iVUg)8yelN5Bk2)rO0J9i9?^@it>xLj|7OpDZweyq@#U8^5&Rw7z z)_(6?-fZLL$C`#_z%o)&du+^+w*cP=>o|RLF*F1HK;sg3t-?iJD^{142Bv&@vn_d2 z=|V%#FL$xl(BQ@I@T=hk+ABWi>*miC%s6clebMjw#qdW}%4W8r{Bj#HvOP!4^m%K5 z)9%*8IKl^$zH{sHs~4_3feE#nxy0F3@uj5&O5MLG|7-ZaD8EQ=XlH1smQz`_K$?fM z8)7$KCb{N>p4JqX8h)L%9dzkuatN19--0tGvJcKW~fpiQ6zB-Mo<_2L&g12RqhR^caPYht-rPVHu_e|Rr>ddji}p^ z(UD0P8;}u+70cL)phMB>9Lk3X%!rrDMR8+t~7 zXoS%`Pa~z~Ml7~-%zr!W=5{0@p6IZv%KuyibyiE{E@Od5D#|WmlZ0Y>gqLvyj3k;V zBr>IsrJy+$u|YHEC80sZE_BJr!gz(^gHTW$!_vuBC|Y7#Vc|+Ky14FJQd`i_(=rTX z9?2d{YpqCc?+t;u8erRpc3vdP^baT2Q?V(Jx`dn5jCZpV!&%7?HukGGl#-9YTkqJD zqrj`~G+Ac)_zFwYWrN`eQE!qBpL890@&=Uh$TcWc2wH7ac_~GY$Cb zLJV|-gH6Z>tD&qos#aQ$Nz2GrC*=R&)T1>HMOnQnlgDWh@hpg}y7JhXhWTMC8QKJn zz8lM0cU(@%W+@vYuQskti|FzpgOq&xpSXW6NmRk*9G%+$`b#Wg;5aHWgJ+V0V=LVs zK5to>Ze2xO;}W>m?R;3DOpI!pzx3pmP^6(aa&OVrtfHEud7U`jMkN!`Jbiw$c4jj> zH@M>$+<4^CR^x$Gi-qIji5Cfj97!>zAIv2g`|gYczyA)5-XrjsW|XMTy8S2g&%tZ{ z$wqz-f^cR&1QlSGIFhX3f$%tm=p~G~FomeYuf6zoT5Zx|db-&k#L`pnZZFaw5xMmM zmZp|m>=14Sl%8oyz0?`7_x(lPsF1a>Gh&=-wkg6&73Mv?8{rW;)z!@c(#7$BcxM!^ zGrr4CD39T_lONO%wmM(+b2c_Wjk`+9UnH5EWX%H6wLg;QArqWlJH(`s!<@1?2WXmj z&^{1Dow-}uho?+G%Y@F90FCCG@;abWc8rpD#5n6Y zz+B0IeMA??>?RhpfEj2S@>xg-H`nVfRmgC$nO=Co+km<*(GZ8Mp5D^ zZp~LZFQ0mTBQ^v^X{My1_)$i+ zFe?uGozsY;!?V#F?t)w zMlQ9%7@F`;ait!lq1BmggfcqlwB@GtKN+qN%{C#hq;^l5~D>1q4BrDh~`Uf*Y>Fg?cDr4>?c8}B5d3Clk zjakRzVYTTDrYvLDVejd5d`?tMoRP@C6PvE47ne;}6xLXH)xIUaM!RYLy9cPeqKKM+ zV=DP+)x#4UC1MQu>VOs}11F)>833UG5s^)z1Wr-!f-#(*{6smL39i@JNjgK{mBS{8 zJjh+KamDQl#IzGDO?P>G%GFh|p8c8Kl``;@1gTgn;xIfMtcqN$4rJw#cn zuDteOT{VM!Kb{ns9}SGkrD?&^Jae#mOniO@Y6!-C!V=XSaZ6Y%xwGpns9w%LL~%)` zZEQI7f|v!t@PC&9$-Gb-*eXLdG4p0|!$+=%wrj?Y*ie_%C^pc8m(F_d=MJqgpOmI? zIIQ9siw|yOh5Nt&;EYPCBaXjvZoD;l<}zAu6)SNbo_hqx(rSN^oT*m7GK9jL>dNy+Y|2)EujCUY zg82TGBoFq)y>{k)7#;x&mvS9*z38#HP0^C6Feg>OE6Di0`!o`A<47$Fg{As3C!74* zW<5vquiSr^0;oY2bD8GwyDf$Gz@lnF)Y;p) zCq{2dmsYtJz|xA@7m;@ET7v$nwSn(rfrwS?6)=<%}db&AA-_p@6V? zTjs&rsFS9himY1gLs-O;V4#PG;q+@^Ij1^=s436c&}$~jMx5yqqWP~T3@q8iPZcejw>QZXtk+b-osP*h)PdG^13L6!pkHScZ=z9wcAg16P zrFq`NrL$$Ps?WY}6ZlhhL!4uT41XSpgjOCh`$4-7mH{6ms{H*I3 z75S_>)J}~s#mPxFowkJdc}A+o5jX|n{*mwbF(A+ym)|Fm)8Cle_0`pMEg>|^b9Yb1 zvnt3PQY`0%!#!hSVjgC{_xw~?(3aun6!Q}gHz3`HWfv!|{fvy859)kIj!5G{j(@!1 zfD8w5#S?}9oSSTJpF?;ksW1A7Q$Vk?UHZICIzPE-x$+C22A(rZ?j#yt4$-I_g%A6gPF@+-UhtV;tv0m<}d|u^Efu> z*+h9u=pInPdWZvcW6SZKo5w+8%dxUa$5WZAJPh}9gn?+hdozvQ{OHj4BKLKk$-C@K z*7HbA>$w}NCkXX~|2y{--T>0>{URdIB^Rkx z+SiNaqTjb2Tm-ycztZPhv1Ch!CRp5@z4jO#@A2-gfbTQtlqi0ie?R{TB>GX*gLTw$ z4f>v87aE)S@-)ZOW81$3m6Eozv$8=As@>k}R?*k^20FwSoxm5}_zz4cSd^7^Z0 zmQC^~&uAsuSo>r&Z#i37w^NXq6winA;)^v!L-7er$@o|AJQr!#gU7(5|1LM-o z;F__kv;9UCCbCX9M6r38*{dMh-Oj`f!dvXxY;w1=SK8ifa5u|zY{SOEuC=`TFZ%OU zZd+^Tj*$y0w_!+smg#Qgr*rStM#8g{UccZvtZU;%yN36|Shi`RcwTwfalfyi9$V1- zV>POwb9GEAlX~IUwH-r$#z<%S*lf&my~(%p=-J0I!(&@aA<=G%8e@IVtUiRYOk30* z6H7iYdbXwD;q`gnuy)$&cAl!JejM-HTpcT{b-Dtm!`Y!j;oIeiG+2aFV(Y=86o+~TaYEfaYH|j;N6s{XbF}alUW7pt<3DasQ>bpR@n-IM+ z=;=$l>YRMB+c-Pb+O~98!Ol-n$vafLcIc(IZU1|==lxT_@H@@ZYggbTZp-k5FBCxR zgWl}r)A`4B-KSKkiWypL9)B?N5=u3mLN;X`dK%s!6{vdCU zXwR(-mhg%9gdwL7O$u ztKN$yFQn2>glx{O{_kkTcb|Gf#rNGNBE@&5(U{3J9;Z@>sDp!urINQZv(Vw znHBXO{h0kY5~{yk(a_}Mdy0^I5wi+1W`Gf%fM!B5XI9#ZnX?DZ6?9zMiwCvhqdD4{ z4XpbliCWsyweGhvAfALNVGwMaHbFOCO0>Bx!wqyd*`y2)K3o)3nqV!6bx!{E$f(U1 z3ijD-h>##cmA}tcdW?GW1l2^H2>B)mXIhkwcJ@y7^zH!fB#Ty-TvXqLhe&87NENd? zT{v{Ij30tQ`rwT@i%8TB7O#6Vt6m?Ijj(M~ZU#~u1`Mg}=%q=Ysp#xCX65?BU(z!& zR$q{8FnQr$hph->2sPAm02Ax$X%v-H}o?>gV<(r5uP{} zjs?GqfD+d5-&jy|q_M6r0XJ3fq7L{!Xzibmeq zeJ|Vrw+>+Y`w%0eu+jgYjovBP=+*zr< zRe!W(HsREx_J~m-e{pGyzx6Zcitz?%_-ri7fqjKMT`EW`E!+|*UBu`i-6F;1v=ny z0*!8#Nu#CSmgos87S+vj-!%M-iwKBUy7}xvHnQZn-(W!(k()^UG?_%lrG)0Lk!SOI zU`TDRq4BF`_b#5t0 zq*|Nl>UO=?{6R_@{pqbFJTJ1=G%4dH;GqugvvNEawXbyu--d@1=Xs}(4q3R2J`WG| zGY{MC7j=kaDw_sYHRtAs1H2b%*yyu_zhzo*r=tgKXA-{B zcI>HDq}C1otS`!kkszTDN+9!L4~#FZFV06cu=h74GssxutXl5WDjzXm7d0y+lXoro yBrmLeGW!^R>3t9?e@2Sf4~CTT)rZW1MX0xhlf?P``{VW- + + + + + + + + + + + + + + + + + + + + + + + + + Produced by OmniGraffle 7.10.2 + 2019-05-27 08:17:48 +0000 + + + Overview + + + Layer 1 + + + + + + + + + + + + + + + + + + + + contentserver + + + + + + + Web server + + + + + + + TCP Socket server + + + + + + CONTENT.JSON + + + + + + Content Source Webservice + + + + + + Repo + + + + + + RepoNode + + + + + + Dimension + + + + + + RepoNode + + + + + + Dimension + + + + + + + + + + GetContent + + + + + + + GetURIs + + + + + + + GetNodes + + + + + + + Update + + + + + + + GetRepo + + + + + + contentserver-repo-current.json + + + + + + Var Directory (aka History) + + + + + + Logfile (JSON or TXT) + + + + + + + API + + + + + + … + + + + + + … + + + + + + contentserver-repo-[Timestamp].json + + + + + + contentserver-repo-[Timestamp].json + + + + + + contentserver-repo-[Timestamp].json + + + + + + Log entry + + + + + + Log entry + + + + + + Log entry + + + + + + Log entry + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Client + + + + + diff --git a/graphics/Update-Flow.svg b/graphics/Update-Flow.svg new file mode 100644 index 0000000..ecbefa5 --- /dev/null +++ b/graphics/Update-Flow.svg @@ -0,0 +1,263 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + Produced by OmniGraffle 7.10.2 + 2019-05-27 08:17:48 +0000 + + + Update Flow + + + Layer 1 + + + + *Repo.Update() + + + + + + *Repo.tryUpdate() + + + + + + *Repo.tryUpdate() + + + + + + if updateErr != errUpdateRejected + + + + + + + + + + + + failure + + + + + + success + + + + + + *Repo.tryToRestoreCurrent() + + + + + + + + + + + + *Repo.updateInProgressChan <- make(chan updateResponse) + + + + + + return errUpdateRejected + + + + + + failure + + + + + + + + + + + + queue full + + + + + + queue free + + + + + + return updateResponse + + + + + + + + + *Repo.updateRoutine() + + + + + + + + + resChan <- *Repo.updateInProgressChannel: + + + + + + *Repo.update() + + + + + + *Repo.get() + + + + + + *Repo.loadNodesFromJSON() + + + + + + *Repo.loadNodes() + + + + + + + + + for dimension, newNode := range nodes { + *Repo.updateDimension(dimension, newNode) + } + + + + + + *Repo.dimensionUpdateRoutine() + + + + + + *Repo._updateDimension() + + + + + + + + + newNode.WireParents() + + + + + + buildDirectory() + + + + + + wireAliases() + + + + + + <- dimensionUpdateDoneChan + + + + + + *Repo.history.add(jsonBytes) + + + + + + + + + return updateResponse + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/server/socketserver.go b/server/socketserver.go index 5d2f34f..b395eb6 100644 --- a/server/socketserver.go +++ b/server/socketserver.go @@ -1,11 +1,13 @@ package server import ( + "bytes" "errors" "fmt" "net" "strconv" "strings" + "time" "go.uber.org/zap" @@ -15,6 +17,8 @@ import ( "github.com/foomo/contentserver/status" ) +const sourceSocketServer = "socketserver" + type socketServer struct { repo *repo.Repo metrics *status.Metrics @@ -41,7 +45,19 @@ func extractHandlerAndJSONLentgh(header string) (handler Handler, jsonLength int func (s *socketServer) execute(handler Handler, jsonBytes []byte) (reply []byte) { Log.Debug("incoming json buffer", zap.Int("length", len(jsonBytes))) - reply, handlingError := handleRequest(s.repo, handler, jsonBytes, "socketserver") + + if handler == HandlerGetRepo { + + var ( + b bytes.Buffer + start = time.Now() + ) + s.repo.WriteRepoBytes(&b) + addMetrics(handler, start, nil, nil, sourceSocketServer) + return b.Bytes() + } + + reply, handlingError := handleRequest(s.repo, handler, jsonBytes, sourceSocketServer) if handlingError != nil { Log.Error("socketServer.execute failed", zap.Error(handlingError)) } diff --git a/server/webserver.go b/server/webserver.go index 64c4f29..2bbf008 100644 --- a/server/webserver.go +++ b/server/webserver.go @@ -4,6 +4,7 @@ import ( "io/ioutil" "net/http" "strings" + "time" "go.uber.org/zap" @@ -11,6 +12,8 @@ import ( "github.com/foomo/contentserver/repo" ) +const sourceWebserver = "webserver" + type webServer struct { path string r *repo.Repo @@ -37,8 +40,10 @@ func (s *webServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { } h := Handler(strings.TrimPrefix(r.URL.Path, s.path+"/")) if h == HandlerGetRepo { + start := time.Now() s.r.WriteRepoBytes(w) w.Header().Set("Content-Type", "application/json") + addMetrics(h, start, nil, nil, sourceWebserver) return } reply, errReply := handleRequest(s.r, h, jsonBytes, "webserver") From 0aed28b524a9573ce5495e8ddfd7b32bc228444c Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Mon, 27 May 2019 10:45:32 +0200 Subject: [PATCH 63/79] removed links --- README.md | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 91d36ef..bbd416e 100644 --- a/README.md +++ b/README.md @@ -12,9 +12,7 @@ It's up to you how you use it and which data you want to export to the server. O ### Overview - - - + ## Export Data @@ -47,9 +45,7 @@ There is a PHP Proxy implementation for foomo in [Foomo.ContentServer](https://g ## Update Flowchart - - - + ### Usage From 8d85fc5f81c49a778ae5272b9cd54a7e7a92eb0a Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Mon, 27 May 2019 12:11:16 +0200 Subject: [PATCH 64/79] GetRepo cleanup and testing, fixed unit tests --- client/client_test.go | 14 ++++++++++++++ client/sockettransport.go | 1 + repo/history.go | 3 +++ repo/history_test.go | 7 ++++--- repo/repo.go | 9 +++++++++ server/handlerequest.go | 7 ++----- 6 files changed, 33 insertions(+), 8 deletions(-) diff --git a/client/client_test.go b/client/client_test.go index d7e46ee..e52df3a 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -68,6 +68,7 @@ func initTestServer(t testing.TB) (socketAddr, webserverAddr string) { t.Fatal("test server crashed: ", err) } }() + socketClient, errClient := NewClient(socketAddr, 1, time.Duration(time.Millisecond*100)) if errClient != nil { panic(errClient) @@ -79,6 +80,11 @@ func initTestServer(t testing.TB) (socketAddr, webserverAddr string) { if err != nil { continue } + + if len(r) == 0 { + t.Fatal("received empty JSON from GetRepo") + } + if r["dimension_foo"].Nodes["id-a"].Data["baz"].(float64) == float64(1) { break } @@ -149,10 +155,18 @@ func TestGetURIs(t *testing.T) { func TestGetRepo(t *testing.T) { testWithClients(t, func(c *Client) { + + time.Sleep(time.Millisecond * 100) + r, err := c.GetRepo() if err != nil { t.Fatal(err) } + + if len(r) == 0 { + t.Fatal("received empty JSON from GetRepo") + } + if r["dimension_foo"].Nodes["id-a"].Data["baz"].(float64) != float64(1) { t.Fatal("failed to drill deep for data") } diff --git a/client/sockettransport.go b/client/sockettransport.go index e236ed4..aade59b 100644 --- a/client/sockettransport.go +++ b/client/sockettransport.go @@ -111,6 +111,7 @@ func (c *socketTransport) call(handler server.Handler, request interface{}, resp break } } + // unmarshal response responseJSONErr := json.Unmarshal(responseBytes, &serverResponse{Reply: response}) if responseJSONErr != nil { diff --git a/repo/history.go b/repo/history.go index 2b3d67a..fb14d25 100644 --- a/repo/history.go +++ b/repo/history.go @@ -100,6 +100,9 @@ func (h *history) getCurrentFilename() string { func (h *history) getCurrent(buf *bytes.Buffer) (err error) { f, err := os.Open(h.getCurrentFilename()) + if err != nil { + return err + } defer f.Close() _, err = io.Copy(buf, f) return err diff --git a/repo/history_test.go b/repo/history_test.go index fc06a6b..914fdc5 100644 --- a/repo/history_test.go +++ b/repo/history_test.go @@ -21,17 +21,18 @@ func TestHistoryCurrent(t *testing.T) { var ( h = testHistory() test = []byte("test") + b bytes.Buffer ) err := h.add(test) if err != nil { t.Fatal("failed to add: ", err) } - current, err := h.getCurrent() + err = h.getCurrent(&b) if err != nil { t.Fatal(err) } - if !bytes.Equal(current, test) { - t.Fatal(fmt.Sprintf("expected %q, got %q", string(test), string(current))) + if !bytes.Equal(b.Bytes(), test) { + t.Fatal(fmt.Sprintf("expected %q, got %q", string(test), string(b.Bytes()))) } } diff --git a/repo/repo.go b/repo/repo.go index 152c94e..2a7c22f 100644 --- a/repo/repo.go +++ b/repo/repo.go @@ -221,12 +221,21 @@ func (repo *Repo) GetRepo() map[string]*content.RepoNode { } // WriteRepoBytes get the whole repo in all dimensions +// reads the JSON history file from the Filesystem and copies it directly in to the supplied buffer +// the result is wrapped as service response, e.g: {"reply": } func (repo *Repo) WriteRepoBytes(w io.Writer) { + f, err := os.Open(repo.history.getCurrentFilename()) + if err != nil { + Log.Error("failed to serve Repo JSON", zap.Error(err)) + } + + w.Write([]byte("{\"reply\":")) _, err = io.Copy(w, f) if err != nil { Log.Error("failed to serve Repo JSON", zap.Error(err)) } + w.Write([]byte("}")) } // Update - reload contents of repository with json from repo.server diff --git a/server/handlerequest.go b/server/handlerequest.go index 2d14a34..3562ad6 100644 --- a/server/handlerequest.go +++ b/server/handlerequest.go @@ -31,6 +31,8 @@ func handleRequest(r *repo.Repo, handler Handler, jsonBytes []byte, source strin // handle and process switch handler { + // case HandlerGetRepo: // This case is handled prior to handleRequest being called. + // since the resulting bytes are written directly in to the http.ResponseWriter / net.Connection case HandlerGetURIs: getURIRequest := &requests.URIs{} processIfJSONIsOk(json.Unmarshal(jsonBytes, &getURIRequest), func() { @@ -51,11 +53,6 @@ func handleRequest(r *repo.Repo, handler Handler, jsonBytes []byte, source strin processIfJSONIsOk(json.Unmarshal(jsonBytes, &updateRequest), func() { reply = r.Update() }) - // case HandlerGetRepo: - // repoRequest := &requests.Repo{} - // processIfJSONIsOk(json.Unmarshal(jsonBytes, &repoRequest), func() { - // reply = r.GetRepo() - // }) default: reply = responses.NewError(1, "unknown handler: "+string(handler)) From 8197ec0931962f7463d41404a876e7c621795d2b Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Mon, 27 May 2019 14:35:06 +0200 Subject: [PATCH 65/79] removed version flag: git tags are used for versioning --- contentserver.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/contentserver.go b/contentserver.go index dd41799..3058761 100644 --- a/contentserver.go +++ b/contentserver.go @@ -30,8 +30,6 @@ const ( ) var ( - uniqushPushVersion = "content-server 1.6.0" - flagShowVersionFlag = flag.Bool("version", false, "version info") flagAddress = flag.String("address", "", "address to bind socket server host:port") flagWebserverAddress = flag.String("webserver-address", "", "address to bind web server host:port, when empty no webserver will be spawned") @@ -59,11 +57,6 @@ func main() { fmt.Println(http.ListenAndServe("localhost:6060", nil)) }() - if *flagShowVersionFlag { - fmt.Printf("%v\n", uniqushPushVersion) - return - } - if *flagFreeOSMem > 0 { Log.Info("dumping heap every $interval minutes", zap.Int("interval", *flagHeapDump)) Log.Info("freeing OS memory every $interval minutes", zap.Int("interval", *flagFreeOSMem)) From 2cf28f7217436d24e89bf1eb316e8cdfd2775abd Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Wed, 29 May 2019 12:23:55 +0200 Subject: [PATCH 66/79] updated graphics for horizontal scaling --- contentserver.graffle | Bin 19879 -> 34699 bytes graphics/Horizontal Update.svg | 327 +++++++++++++++++++++++++++ graphics/Horizontal.svg | 391 +++++++++++++++++++++++++++++++++ graphics/Overview.svg | 28 +-- graphics/Update-Flow.svg | 61 +++-- 5 files changed, 755 insertions(+), 52 deletions(-) create mode 100644 graphics/Horizontal Update.svg create mode 100644 graphics/Horizontal.svg diff --git a/contentserver.graffle b/contentserver.graffle index 9b9a0e04007a0d1492fb4b7b2e9f9ad19e855ad0..d380454418d8d5d6c2f12c5e501aa46c31c0fc69 100644 GIT binary patch literal 34699 zcmaIdbCBlXg6QF%wmogzwrzJ$+qP}nwr$(CZDZPY_nkTC?A~+t?(X+bs@_y3UnOsn zN zMgs<%{{78Ej&E68H__yg(sGH|WR4j&IyoxIg@BHp_Im!^5K)j()uIk&b~p3QcOCoP zdW%Tv6fpRG;$v!Xi}{KBJ>&EJx$+%w?O@=5w@Fo?o3#ba)J*BX`h8n92QXoD$c*uk z+Jo+RH1`5}JZB0%?z8zZ!Znfny<~dm=56v*$wdYOHSl~`H;Ja}SMzoMWOpC1i}mpr z7}as=I|%1Z6(-x-pFP<01aE)!MC=2OjPG8*#a*n3+~Ry$Rw*mr9@+F%H~c*O?$NVz zG0O!0qS&Ike!IzNtT%2LwfI^u6`L-V5AGDS8-=qyZ2VoFCh88PWO)N;swT_kHwOQAATbmXC)PrCO1 zNIe!p!nseCI5k%Hq&%NFzG2VIe$hm|W?PefC@tg<=VBX+Dj22iW-w~Y+nQ&rcv-)Z zeO#~<%;Ag;2Kh5ac?MNM=0`hm$v$UWh|+UEbnRUu#wtv^lO}7bizB3_^D;cFHXl|{ zC}z^6)I<1~%s?CaS$7>ngxx}nZO5(@YprJc(=*WIS2Z8ig@#_r8h1bu&8&C61E zrMmqfQc1^L?69(mFU+Av^=_kQ&{4%1=LKeRLQ?#?h2SwOkm8}Bl%|t*6~v`8QFGCA z;->Ar>&k^2p;f09J!4jRT>{#c8m&6O;JC>!T8K4rYkMX!w+y>l^l6BNa+Vz>Y|ZN5 zDI!y81KIr`;9Y0RxFyKmKRTNZJ@M5*-`@hg|7^G$s)y9D%l4%h%8zZ71p~KxO37?mAG)KUlY(7=VdpMSrMSWqe zjDB`W1%?PvCLE~mFTX-SI9u*nKSAsp208m#s?uPJjURwo#S43QpaefAGo}3k0Y*&h zX2=9#E;3e3x8x>;@HNiRXRbMsN;BwD>sdv?88T)une&w11Ot|4bF!LS3FM^V^ zPJy75!cJ85qd^Nci>-_Yfk!S|&Y*(Rk*wxXJo(Zz1XH~A+Ny4#%F~Uh@usxvQZ&@j zPEFiZ#%fg{LSXrARaD{I3MR{GNYW*&Y3LmLI-Yp%_P@WE2#VxX2 zL3Y;VGJXfV2)eb3i~faB2h#YHQ)VcxJfm_i)Y@9?Vi7r|+pj*FB>y*&zDe=mTX7#~ zv{Ip>{i>Kenc7$5SaDlo>2k8xK(^8)NXL@8<(1UhXFIdp){h7XYE)Mvj$~*J%3Wqv z5$m|$k-NR>)<<92NK$JzCzvwHN2738Z8Z3VUj%!g=gx650D6N_3r}n@0@jGlsBT#H z&g(pf;)7Gm*!r}s|?O@$v>i3p84ifh3?lU6GrUy7}qusQ$jl=L$fpafm)wc+ERH2-) z2znDhH|Es_e49PY;`C29%@8}&fFZ>w(c8{)ek z5Ocx@X-FHKm=i$e8Bd97)diS_yt1K&)K}NGTTuKLE#o70q znBcbib3&T-7b8@EH#b3;^hbAF>FrVJ+hA60ETW21QA)PqI&|W}^>z-0)pN~Bv`LFr zb#rNeU3zJxD6*?wc2sAUMDUahGL+HW>BhFGabU#+J9ad{*6~q}sCit7qL@gnM%*5+JuV6#)t=V_8T4S&tAk64d@JmZPY$z1G= zyf5V@Oi&V!CSxP2;TLFL-0iKyKJk~SAd?=jCtk`-UUH|G~9qlE71eh=Kd?#+_q0g0PY^YZzxe zL2Brur$FP=b8T~K?2=jarnFx8!i6q|(RBzH(I;j+Nza1ox~qIF-Xq4^3f2NFI|G(F zD~1}ycZ5v$v-QtmvlK;F7rO8b-VaL4ystb;+7aGYifCCZIPz_fi zi2ygHi`FsEt$#8QuzeScj>eCNi{Do}s5NBO(r8LwpU*;F_(I0d*&(1SSA-d|p*u|G>cov;BYP}4gFt zJN=+*=!oSoL8Da3OK;O4mn5>@h${>W*0iL`7HmWnHFU zXEZhV9_Ps0(0pYcX@ywPSLtN=XS=9Q*=dL(>-SH?{ptQ)ge}>NXdIf!Gf3Mm!xLIe zW=Arqmnq^gPI8Th4hsr-at)(-MMc$R1;B@sTE?qZ6U>{%0^kX+!{?WyGKekQGeyB; zUS>INueE0T8rH*MjlArWCh4PvNd8$vmm6GpsEqonc+$;BE`Z6BLrK@VMctQUS#dqs zw;XdN$8viwcdqm+T({YD@Arj-d4UU=2`vniGb#Kr=Fg$67a_VLPQ`cSNLa1biPmR9 zi`SQmhApX}5&?JzL+Kb~XH}?xl1(X{2+=kZcFsG=A$s<_1Y*Jv8ObCqqFJVN^Q4D` zFrqneu`;CnQ4n+4&->PNRxJmJCkJcW_3M<}A{rgO{%68S6xxmMF0> z6iNEH=b=@L%@UcWJ1&t-Ehtx?c-rC0tp!Z_WE?MwBE&K1v zC*E>*X)emuO>`jQDO%|7;Q#36+z((Iy%I1kp z&!w|W9BH(A*;Xg$D7!)~JX>Ul?CQju#&vlN9$jm4IMf^Oh$k(A0cuQP(=6CqEa}iq zR#LtT2PKf}NPg{Z=J<(&I`g3Ea z%*{6I+(T;xA-6(fSb&_B7k&aeh($fVvWwCrv!Pp9FsU>Vcn7yMn9XzB%D5&8_!!dA zQg+x6$*d*g1VfR6YuAk(gSD5-Pf41CM@f`qseUtX+UawWndUF!!3N+~T)Yn(1_T>4 zlN){IVxrBUdIdJ%g)t*@pzDiD@lPy2q|KfX&uFd88P(W7ts2NT(suP^7*!LU#giW z(83jNNK4M}o2PQOgI3Dy?2(qf^4E2g-XqL?N&5{|{90%}s(E2b_(ESn+oEbZ0dWp#Q0Pn=cw)w&{7t?xtJseTXf9Mi)K`}X}Rk_+w~VgN44N*>=!pZIZfjJa+p zpqD#KB60u?(L?wBc696`7f5i^A*vAgdK7G`KZre)%O*~U_T$gW_Sn>LAe#E+6|_=< z7EuSXj&sgwiRR@uiJ|Nx()*LNu34#HE++4zoF(3tnd6xm8rJDK@|v+&W-m!eIrrZ! z=LhFDKAbv$o@96$MS0I9$hNm#Vq8zoYZgh=-eB_SFF=$Tl5^hdU}20tOo%g*&Yg!4 zr`pJ_Uyp2fE89%c2dvRM1}67E8o41m?7Wx|vNM*x@1uxP4^lg^)J)nsQ&C=zK;7t< zf~sAm0uX`UpqQ%c9-0z^S_SZW6x1%%jTTIiNbw*)EcThdX_1g>K+?puFHBdQOr{j71AMWEGpp~ZOgNROb zti^!a3_?jD>;nCE0T2p=5E9!4oojR74AP)g@W}(XFmp{Nj>C`?eL5|V- z*r3SB>rs<)obbskiqR)}%1GcS`}HBjJa;hQZZKd1$p7`l+cPy!G;wj-f_}F_1+lAR zV$I({E@-W;-T27B2BFXy_1yCXVwR=P2}|^)(^JKTN;tOjk#s^lgvz<@dSfI0=<|Ji znQXhi-v;+^^Jyb#arBjZDF*5H-H)&Qm3O)i7mRR(i$VUi>niTgNDP4$H-w8wF0T7~ zA1)}FNC_z(`4yDXbk9av`C+3=v2zsieoWFh8R%qvbmv=p**eOAwTGSYFgOA-<3?wu z{i{m#g95Fm5;EJFIW(W?TD=+dw!+bE?u?OA-6M)s96OW3qh>TSRa<7TI;PA%XX!fX zhLk4Rfy5JcAe-fcUf!Gg#~o-JrE1Um1su@5Ih?k6EEv5I?a@u!VN-71n>(e}{tBV* z-YFx1NAkPm>Plz0*VbalQbaiY3x}k((i=p!D!B>8zM6`5VaG*g2i0FZR=%*e{NBv!vroPy{za+*w12jMXPw+huL2!&A zGyfOvCZv&7-qx_h~u|#cy3DSck&w$Q5uPgYg?#;7Q&O6mlKC6}cyOWjQMC2y1 z=ZMS@g&{EV;5#7@P;9p4ktJf8yg!(D6QkdP^%l3#`w7Vo)6C~%Pex?#KS_*tHiEyc zgUqvjomZ+?ctTY9`>2|dtx&|8j2>cp zf0=ccD8E*-e7I05StO}feE6o#ItaGq`{a}2D0&Lkl%N%!?-J0R*D;J@`{T305c>_o z2N%kBig$5`Q91U3S+9 zG@(>C-Q?s4Swy`w8App%HSiox#+kDC>`&f`z7y^!<*A!2aa1g;HBG;9X9?5&S)_76 z4fUi>zimPOYel)!a?A~vw@)P@zt5$rD8}~@DDfHR1s7e+n;hgV`NSeE>rU;l)_tTw zWTChE?g6I8%}%9E6^w<|3t>K9j|O^oGhe?m=avxQ*AtbbexA6WPQpfw~VsU?eY#{<+0Jt~p(6Gf&HA3U4)EpdopZ6ZqJ3&H% zB_SRVf_#4CI(OPwi)GOPA6LDP;o`XWSK@h| z1aZ1p-sbn5+-&M_hM@cNJF-auZHPm)*0c98n=Uf-4KijbZ*tozYd7UW(8Px${#ST%al{_Hp}J?Gad+Wdk#P+29%zN%%fEx+l(lc~ zzm9R0O}85Too;V(a_m5oim$_q6oEz4(Sm1ry?*?xNdYo%t<2KN*?Dy``dnPARK8gn z(c%0I)|0)eC4+q?iEz*~1D@^?hZco@J7pHk#?|9Pbb-z+G?@LeL<#9;IOOyr9bS}a zQXk!sA0?*66d3#R?3bD!#a7p>9%e!hHYB?Pf^_h5GpC-sCZAlNZ(_Jc8fOUK$zzL_ zSK1>evBmmyt=Ft^UaJ3?4p$S`viW>1M7Lp(J$yP4XZ$7UKs|L$IkhhL71%F@T(O2M z{A(TE3=vm0bxb|gLeE=A?>M@<4vu;TEtrcl3R*ZdM2tSM|0=X$2jaL3xEYqaOa+_= z6y7`0iifCwlODKl7sQ+y0g<0qQ;suuML|Q-%BtJb{XPgzAFg=@Os9|X`OOnlbWM{fK{|7K_5n$S+!f0QnTL`shFb*HTX_`3tce6@mA&MOT@371FrwS zy>RWG=Ejn zPmAeRMpq42Ogq4coAc|V!JxV(62!<;B~$1w&kun(HUgXtG(DJTsgp2_WGt2}U{@OS z`>a-Maj&ZBrMuuIy6p#(ZRYHRb+kthZx%EXnww~B?UL>R9dfV|V^0qGpeyxY6MRnr z{6N{5N$j=_vKxK^+m9f`A(|5sG4k%ogdncY8`A@q>*k5V?99c4g!P2`UtH{{Ase0_ zMqHT!w98jGsE@Y6-GUJVz3`C)wvFOl8oL(Q`@abFyCP2Z@tiYzaex!}PD?g+cg)J= zZLGf|flkASrDBJS#-U&1&onaezhwAya8Ff%yr@gy?8iUs2bj|C70WZA@8`*v$h2DX zcU3S*O|eSKTsVW9vTK+BoXkt86notl`OqC_aQzaMaLuwr zv*VS&JJM<`Eb{TVvsZj6?x|)Ar@c*@H5Nw26N{qaj^pa7>HU75L$wr1ZN#^-*1gV0 zJ$6mQAYg(K0uY^vr8#G!j64-r#ijo)*c)bcworoM)Xb*rl&$Emy~H9PiR<4$OCL}m z;6%r9jKZ?Wn~(;!uhgGhAKo}1EFuhxD1%)HzCqRGZADDTB8%=x)B8ne$?9TTv~|Eo zbKiSyH)!2R_LV%7kqx;>NKE(R1jvI5Pzd!TS}l=h9(kLrwRf#8?Cg^8NDj0Ujwxxz zFaoz48ykOQ%azTA*bc9h6CGfdQz1HVYd+dc#<#{YKq#|xO95z^>0!~t3=jZU$qi7I z>kuS0*xnY{kt=c+=-)I&==G4X76u^jj`FXj0DVCvs_^tkvNPE^U-KyZKHB(CP06`f z9}02f2QfBrGdIq}-gz^!Vj@kKg~9dCct5(2vkzCc3F!2+$S&=#r?R5R%LuP`rO^nU z+S{o0)JC1jydZIB@;XE!zuPBbjV5S$8BfFtbe5c{+_xCuJy>9QXag!!m9vL$kMFJx zS9pmJC}+(wWmABn=+3?KP<*&s3+%xqpV{f}46TRR#l$obCyhp|C{p%XjB}SvLmmBE zeHM+6-iVKG#VYT&tNo{^T*l33*E(?eNfW>jf}L|a#>OTV4A6><$0?|R7uM4?X!BiU zKOB$pPawErjH$z&M+r(n#Co~ld+va{g((Dfp=IK{1QholQ;Q~QHI?9tBzrQK6a;D> zn<>U7>*NX>?+I^P!dH-v{~?u#Ag4IiKcIr~4^SzK)#H#yi2;$>sHOI-U>3sS*>bi+ zb$LFl&u4PSM7qzZbxqqBbCYmk9_pFPhMRZ(u#`~Rozn8L)ZePp6c@D_gj6)(2my~6 z)nA)|!rZXh3cKJW_BUH9J&#@$^sAJG>>O>$4{ZC^?j|i<`ar$wQhp{aXr*i}AYa5) z4A7Jkt+Xp_gvM@xW*{%@jSi7i32%1prP=$wTv)9d{%As&ycCJO9u0e(GT3?qM5z7C zDsF>+StU=K&~pM$0-d7pgh*7y=yYh?u+D$ycgU?hAAo!ztK0<`0ak!x!@0$dbW;|WYC*XU~z-D>`|C=3R z)B=1v6Ly_k%)UCFZ5E<${N(8`vG@)P&j7z?s`%yqpbDXAs`=%nzz#g)2Vw2t2U0$8 zS55~y5NmZL04!qBd;%qubdstYQoah*K8aC&3N)Z+r6bfwWb&!`+rMlQhulCX9f;C! zrCbP5RQK830(>);cySqV((q|HrGf|0dpeBInP1m_OOu3;%(kGj$`5?6*v$hhK9N&l zn!f}tnJ48fi&-^Iv)Z`>Z-EWaQAM|c|8nkJ+3iY7TiFAh?WHrF63DmsV^QS^fb^yJ z8q=2}srx^f#S-94`4=&pJ}`PI`b13-0@`m;3FY4q(HxydN1CCZ^5f3ZdSJr_&9uGR z7(4Y81UTdxd>VBJN?2v>V;Iq=$H9X#(spN~{ek)smyXvi~71%>ezOm?f8XI6<34mb7|G;ugSn@=v)xeSH+mu8xR{rR0pa*!zibTk} z9_3F@lf~?oifv?lCngh4D7`IAgbp0q3l9S)FuRD;sFVvnWa}o0?X|Y_P=QjldwG=$ zymdXE>7l$z%1Y}{B%K%+P=`WfZrxsbj0io`)kRO_ftV1sbvz6!dPVgLt8rH3E7hZmb%Z!-P%tb zB&ZU86TpGGWfTqEnu@h>o#TucpKUc7hbXX9wHwJlrcV~Eb};X(zMpZ;{R{pQWWv?e*~A)F<6Ly7nf-4KzUh7 zmoI^QcSg4zwMNvWsanlz`R_`^UGCZ zvJ><9YC86X_EvY*J4P#jU_ON4(Ne~XI5}uXFE*`kS5xIn_DHQWoH_@#1xvS4Xc=`U zw?N>5xcz1g^+q%``K64;*!=?F%L~Xj!6a)e>j3(e3--Nij0!6wWc{T{$3N;~6}1_u zas>b`bVZLwxvHfaLDec*G}tv#x=dK;4QyV1M>HZr%RIz=Ej3cCW1y87>Q4hcS* zEu-MAX3YUBia@$Fo6ha%3nIH;E;9~gdiJ>ggS$YUAi2{M2E)u7YIu|Cwkc`8!VW(k zPLJ+HyJnp#COPsgolG1IY2FWd^k2**LELzMOZQS6?&e8W`Hq%Fhi2G2^E`i061HVK3bW2- zrPXZU!b@_TB4wSx^?&k9$dW`<{S(g5bP>{f;>m_ft^vRgF>%L}g(!l!f?xesD3}xt zD;Y3HbM;*<|I(K@`;2I7OzGft7Lwkx+w;xTI^eDBqqSsKcOWa$=0J0*s>iEoHL-!F zsq0u}3(5D%ZV%Jv75X+EDh~9n>SOHp;w{BYZg(`<#7X?xfpHwN2Wr2Yu=X)vjJk{@ zN9g|zV5%A;?NaBj5C@tyHdO2pQdFvQbwPP}UCl&j?aJkw;_LU&1TbgdKA%!Yo1yNV zYY8+0YUfyi=SL1q50tvZ6~xy!fSHQcBnx{Q%ccyE9$Kb~ElvDG*r#;mrwAH&fYKlE zSwTI8WZOGL3Kj}n#`M7$<9 z%YS|`o8ddy7qvaaV!;t3&_Djs01BHz0*|ULlpL<$q?5zLpN)q?A}+BQ(kJ`ca8 zA+5{Pyh-hGy`=+e>DT=F*c93q#F3?g3Q`&{#%!y@BV(hs&26kal_tjxdm;Jm`T%Xg zPhWE}a(qi7(h?wrfP32%U=6GA==nqe%%+8N4}24j;EwdH!k+$tusAjM_5Z~6Z+oYv92wqt(w|d#ZR%{0NeBJk5 zex9OuccHVyJezG*8~nUj@gv9=J`| zaoAVhCspW}C^tS~#3*T5TvtsucxvLhUe$wDOv+R(B?xynP|2nh%e-(Bh5z(eQLj5Z zIsQ%13((!F$vUQ-F`@a`JE4>BYV{L1Io-N`Wrd#)aZuTL`azOX%0w-?D(Tv2rTxw9 z!8mOcLRn@V3Xw;tb&8J%^idXeO| zlIpS_S&R0TcDYvuM6V=CD_DdMi*h>EN5M921lN#dV!`O$Rqx)2pePs zIfKf-$+Ldj!yJ>PKK^EUrqdNYNpP<+E1P&Fc2N$f`m z6BMQ(3rnyy)xg$9YqpCZ3FZ8n-J-odefDsN82Jhb&OG?BAsZ^rgi#g>)T{h>o5B$x zb_WtmNIKjPU)RbQrWixNu?x0`5Lb_LnQ@Rr#ZQTJyS|9yD`^9ajd)PD$2=@6^IZ1D z>CuPN7|ZX^LcMSn1SiGh!Kp-pNBOj1Z!(oA7Y9ylo8lL`dQ!`>6g_N$-jqP9bCv1Q zf)h3$TZix?by+n(?V>VY&FbewYE@^x8oZMU6Hu)8+6Sxt73Jb4Sq#{(xEScZ%)!Xr zj|6in$%8%GXqStY5Uz1Yx}4gd&gHl`_ov4qls@TTBcTo$T#Xaxv>R>g9@$`CaU=9v zad2o6oEThFf`(ZqZnmN`lMdPd-qbN)H~4Ueb>C7i6*>=fYD}HK9z-f!j}Q0RX|g$w z7=M&EC41eg7 z5VNcDlJ);C>ju0qVQG%kT7)~bVx6Hqw6P3@n!rQAR8de%g)*@ME-Gls#n~vG>=7&B z*fgCH$qGsffiuuFz3&3yOBB>Njn?vRyY+YT@oMa)`Fs!R%+fFys0);X)H2vGWwPHR$dc33B`1MP&Q|th^!uK* zy-9nPHuJ z{MWlU;JWhf@m?7d9!j+$#GXWt)IkNN^87Kt)|f9>cTh~~kU>`!n@WlMK8HEs+T4@y zKZzNi(}$M>oODIAvq7y#Hh8Q0)O^Pa>~7ABk^nNPR^5yC$p2VmhXS2_k#6mn!Y&QV zN%Fcu(lwX zB6O=-_m&MN=m8=7LP&r7tg;oo2z;U7^tiz*TLtJw=7?X$q+yhD;AYt+Gs&{*G0B~4 z;+lZ!x=6VVC*j$$`Hsu}!Yy~{zO@c@U^sikjvEf+T@QpSgPSJ|{ryp0YVBG$vI~XI znVz#f5pG%CV}U7ZhfOmk>(kcy76W51#o&{Ps7Fu9js|Ov=<5-SA^QJ1A{m>eU&(hL zI9759OPGox%6wY;Hs)N2WPx<%2SOuvu`A@h*2jJEN4JNp5FIxBdq z`IGmA)ZZT&lKmu4*#K%b1eL5TA3^X6o5;EMRVQtIBwrHXYto;X;Kmi7Cj!Fle8P!- z3FD$lLOvW6^N&Gzf3e^%3gqeAbz%r;Uuc(P@Zy4=BnysC{#!u$123d*0(_)9;9bC&H?#;GF`gOA4m8eaIi`22R*+o`fR{+`t&ykcUtB=(xZfe$An zpYB4v5%Kb?{UZyTfp%P@4(6c7eCQJa3U# zY~?$2xjlz#*i8EKL__aO)3NT}L$gD83*Y;G+hQxZclO>s3dmoSOzS#L^=X;=a)rsZ z{Q!DASD9x9f%jda@cPj%w2HMIwu^D-#JywxB#Wqn_Z9{(@a)>-8D2(v3Gp%4J`0(D zJ-`X+0dsAh^{g`M#`d0doc>@v=W1VeQp zvwH$}`Fi=CKNi!4HUa-|;ycJ;T@*ZsfoLK2agUF{k3XxYbiY@=&5?L37|g&j_C|2P zad`D9c-71IwF@8;FWXWh2kb1m&om!cj1x=_k9#hbcF_KL>KBQ9gc4o z>`4n1Q##bvEVMm`7}sfeO1&UUI$?;^L8(~}yBamv=Ls z3a#04@5`&-?H+@lEd1|VblWNYC1XK_g#wX7?s>**QVhLX6>q5 zZ{Oi})~i(F)0(Q=aF22M`0DG_PeO|8s3_$fdX4kWoPztWM9Q#kgw-JeAn~6n)CJJ? zuTN;Mpv!>f zEEMeT&@?~%HxkSHLLVZ8Kp~;`BHHtv$!9)TiO?W2`Sx|si-}N1az_r2P~$SX^v75A zqe6YUYY+-*_a|3I4sc72y|ib+xM;y`mOXCH9jvy$H8KC#DvPiD;XShoS3kNQ9#zE@ZLGeaus7&1PXm#;xH;4z79)Q^pXFN26Pw zX6l^?fBYcr19u4``;%u*9SAL3mWrRAkGBIT-FmADNP>943PWKkY*9?) ziG#Mld1EOwm3U#C!2;;eC1E5(v223fNUr))mw8n|AJ{>fclB^Sii^MWNX!mljZ*ht z?1J&p!#{lcrB?biq3$^$8=4)fCf@_=Bllg%q79IR_vQTU{z0p>S<8QEwXOBSPVbS5pPw`<0;rC$#ea|0X%}#mK55u6cn3*_v?zHtXq9Qx! z*k8b(-cx+n2jYVJN~y@nQVw~K5TOJ&@=Xs%Op&$-$9_Zvjdm-?!p+IUz`gt zga!-?te~D)!rPfZZc@fZQUc4hLei{a(- zo3D2b$8LT0_>y-G?ZI<9o9gca+R_7L_qy<}*3M|g2kdY>v|wFizXIvsTLQD^x)T@( z=bw(K&oR5ePk5=o0=(S*`&lH$k9~+ab^zRItnf!owa~BgLVvzlwK{P0N;QXhXv*Zt zERogJ?(Y9%th0ldF)JfC6BQb<&xrb+**(jD|EPrwM=<}6>l)%;Rkb_^_gA;Q0LndTH)ZI22(m`h>cA8NytRNU$sQZGZt+cR@*vwlgTvBJh*L@cPtePzE z&VcA>CV`wkaDypjm;sdjec@u zUuQ=L$QK`-Xf|SIXKAyJQC7x-VXbM4)~NTLPw`c%`DJf? z4#9piD#Sqr5zN0m;Riq$;gQ%X^=0ZY@XI)h*kIT?HM6d?psZIi3BV11(w|QT1hsw5 z$iPJTMh3undS}EI+}esFgF~TZ5@mUSHURj@dI&s}mf3cD_}q@}I}!zvOrEa@i?qAB zrHDUuk8d*6ZpNy$JX2AXbe)q+jEC25t8|t{{sw9{do67`Jh8O}YC}6g_|Xv>fS|_qe+c>d6SQ4S?=CyJW|GvO0D1B~k3M zmP9wdix@8-qO(@&v(Sk=3M?>6hj~~d{xux(O9;*uq*bDMI{Vr`jT(eFJI9B9Ar^U3 z5FZ5;<2lISmAFkbRwI^QNnGV)?KazBBEXspYX5UoYzmq1YDC|eAFnBe<+i=h+Aj}` zzm=K;hs3B+HVcT^-%1Td`)~@h&OLt;9Hv`U71hn5fS{%i>nouu4HCWhZ>6@|2(!`O zqAobNw_t_pV&@`DLn%?vLQnq<5s^Y3omk?iqWFP%`HE*XGE*!Bvc35@Lb8lKk>vQV zQZ2MLlq&hNv$k$)Icv0Akswt$s{JaI8+r7tus6b!QX=*J`p82-eaZ)a$HK{nA-qGM zGhN`5o_Tc0aS%gPVq(^A0xv-T^1`*%vB=Z{>;my?hbL;wd{PIqZ_^t1H9Avuhih4j z7XC*B%K~?zXe4yi82oSMKcQNFVxEAzjJ@NpVC@TO6v~$>Z!}skV0dA7pdoL(etM5W zOMz6qHEmRYA@T7a1U!*Wse&;JQxvjQA9HDOXc%b;Bgufj&kf2L$-eolxVKawW}Sl? z{}HY6@QvKYD3f*n(EWv1)?0CR=$;ES1oF>n%{1h1wFdJGfD?p3KFRvn5|Rj*jVB^N z88$CF!M8Jbcj_!R%b)0kA`B=CzS7egt{rw2&{Xq5s&>ZhCmBYseW}8`dJ<`p;0;#l zE(@w^E(cW&qyZiDnlg=$ZlkeBH=mMQG=y_$ICt(uWnMTB9oF)bFtU{W@=Os;>i9(* z!&4D^t$=ft1d%PUb4_z_FN~Ui6?LDKv5i;TZ^GS^OmIU~V3L(7BfvmF*6|IDPI#DW zxcdJkT`RxKOhY`+0L5gBj~^J05E#2Fk!G1H5^8F`z)*6=&R>Ueu4r1_Osju)vdC=* zLP1iexgq*zx>nAG-X>hm=nrJkmh|o+LuWLQW5cGFOvu}M(jKUJbViG4U26@67ujmJ zj&mYED&7A?5FMxIB-F=4i(hk19!g=}}xBgz-go=0v-SqM-Kfy9RBkD8Kjs9GMIpz%0{ z8M~OAS1Q7FNX+vbhqf?q*HwyHqb8{nn%HSD1q7L zuxlE5UDX#eQ?WVVKvrZIyFM7)jUlxjRgSH=%?f=kBA0F*zoqe&MID+O;Fr4)>dtTb zyIGZ8;9EksUB_UB_LM6Qv$86}$O6-uah-->^RbT)G_l6p`+oHFaojV#Uf92W`%SYnkibK z!o?)UUrOETQHb%In+*`tP(xU-YE~6xnLoR%2@oi}i?wCjRH4LXO?$GMH!aJwb3Uj8 zO<7INDO&(;x`wLE&)wO}W+vqREYTiRXsMEBLvntwf`T$Z@YpcjD*ibgL71FWv>C&$ z9Sf$ry)UGeZF=vx_PuZ*rFK<2hHSyLtHH^u=a)`SR#t=P{dP)|1?bb-i0q44+eK3+Orh$?HK&?_<5hmN z$418!TB35+gy$?_5BW9diAS6x4+%JWxCX(TQ)b{o2Q(3h3_dz-OiB!YWmT6{+i~c= z!~w^eO(ZmCX8{l(*Zvt2FJ)pTp3;kU?-|lHyTM0NrnV!b`W`y+l%AbuR>ua;X=N`8 zD`mEslQF*eQ(_-gXb}d3=(^WlpcV$%w*U`Y$s0!_Eoi#DI2sJUadKS>+q!j^wP=#< zisfjlVEby4OUMMqs)&jZV%~qR*oOF4LKccbLyVWQ2Ue4=5I92^r;Fo+ zkz02Z2}i{J{M986Mb0BJJ6#RWu5>oM$AacCk&Sd!R=2J!g0DO!HF8zGvphQGdIam+ z+c;9l(CZXyN-e*Y>tA?gNsLXu9^MqhmL9g1+^jf^)aX;gT%3*(1==2ehAm&CuI;i8 z8Y6@K$-{uQ_|vhT8(+xBqEXHvCOBD7XaQ1iun`2e(x_1YMyQ3*3sc&4D@_#4fWcl% zcCZ3N`rk@6?o_^Kr^pUgu#Q)>1VD#kl`19%j_5WcREp)C)E2c;2|}y&n0!CI$l`2F zIOvM>M=92OLtWQ{Gpj_{+TPl?j-v3PWM-sm(fQBG_4bPLq*`fx=G3O>%a&%287T8P~AAfqyK2x=Kuee zO~vV%D+Pn(IW7e*gQWY&I8*FBVZgIEOo{`4Rc*7o%9iK+TEHCnj+jPOJmUkUS*?Mwz+Dr`4wc#QZ<_Aij8?H3{XU#TJ^i!4pz&Z`$<=>OKU`9FD$h&fp7ZNu#snFeu6R9My) z{T!L^TGFzv8|=<`eby)4auWF14!_pOdhhevE_>T*YMw}H)gi=IQeEpSwvfMHnqCK$ zW{a}<8?;eAXm>1oFJ9Q63)-qyEs|MW%jcOnGU)VkZA>##^~NOrw@I6a2PadiaKb4T zdCS>p^PFL+l>uNTPS11{y5#JoTDFy20qd-@zGUg}y!!Q3W)PF+#s{ofQ*-0YMMsFU z4|c-)P=3PPx%qb8e-pKV=agnh~v>#pCXtfN9K zT4}ApDQcJ?usr#HdU8fGOU;bRaUym}o`pu8r=&k+ z$QH%3)9(_#%WpMjWan~A!cK%i(RI^Qz>+9o14i0ivg|!wtAVuTfe2AI7CMelv&9R9fE~%Uod&n;WQgJ)n1sbx|y()04YEF<%dOWA`yfj%r5f>=H(e^?k0?k|)0UIs*j zM+p?j#P`QP*mN<%}AeB}T3Rbo{AeN#dwBD6L84)k-Hqn)hWpBbLVioPMYJ}_Q zkOyyVBg$&~FpP{QfR9I8#5KY0{`<%e@V5Ztw@`-nfC#n2$XDMcrYet1i|B&8n4s+shX8CT9C$P^D1 z^?JjTv-#l!atzyfWlbTO_k~#&hSI8kKnbr*|EB(BQg3Hg+kCT*=Cu*?ODK7rVqkXLlr_C?i*_60dPOf&N+)%v&2lCreUM+po{5em zIiX|AThu@-+zh*27F#Y2iS=;s-0m1Ezy{Phxg}~~IZcAkLMDzSFRKM>KUXQ#-$mEY zKLNPs2jr)3Yec8gr+xY_!T96a2EZ#CRzrqMU`a*lw>4f7nQ`69t;NDWb<213$@)pa>DiGK3D}dqX26!CY&(6nxZytvp76BbH%+A z#2s`HYy za=D0~1pTw1_R!B!;4)a$-?~}@?s?ItzU;^MV{Biu`Hg^UPI{}$l+`cD_U7(ZrE~Dw z^)3c4X;0@p8=|!61*$4*d5PXhmM6P$owGB#vl^e8(oeubL)1{0I=vntK61g=%_5Kf zlOW^7CXWlt5!DEnJKQ#B{1FvM7S=L{!|8duXA*fs00nk`Ewab5v|iGG*+z+FgX?2Kpn44UMlDeUgrD7qitd+0_w5_AUB}|6x&@jnCiA%5jmEjve!urojCCnn zwCJY#kzQ3EbKZqc`0Y(f%;Y&y5v<}Rm}5XN1wol%8!Xo8Q)q$(A)l-sHS5jAZUJ^P zz%|?JDxXaT+YOGie!(vI;JCVBHsM%**1;O%lSdtNS0*_D+tlcsCKMK>H*H*qbq^)@ zb`y(#!JX-=W>Hj<9npF?*gCU!*}CXN)|A2Z=~LP$*N$)qmNcrxlet$DzE?Kk#22Yi z>g%RwNom%kA#zFIocCQvhT@vkOPbE$`X}QUCYe<Q4qp|2Min&}3l>R6_MD{A!+Pb9K=eL$XW9etUwC0+GVZK&o-2^3WVGX@* z`t{yvRriarp*;aeqQdJm3aAtAvlitf8GAgcUUG0>cg zjWo0wxj*aypswELQ%w>+_ws5+7O>>|SF&jGk7V)pKa<6+%Mf_r^S_csZyGA-_oH!P zC3sH~_U+0X`SP3(L`d= zEq>k2%A6~j;+#Lkw{(Xx)-oA9YW$kkVJx&t=6}@q-yvWE7BEjLD{`(59?eP5!c0ly zyDvmii9|wqMWFgkf;;+`OYtPNWrrA;A$2=h+(Tk7|PH*e_|MZp017;u5oePEsQd%y7RpI9_XYD2#espTvICrRO`3!x>I z;veoF+R0seOaq={u~?Azc7wWlE&}>|oKpWvW^`b@);3iuUZeWiC6_!8%0F+5RzE9s zbq%SeBoH7zKd9kz2Rbo7&E8(`o(?l3YdLYT2Z}#r+^disef4bd_rc{MiTZ49S@rKRtC|6amm8HI+KKN<`ih(S$se2;tb990^Z4C z+L;elCfq0zZtELOn7hA#hHwKhS1bzxxGeFH#AVv-6x73}yRe`On4y>HUKRCj#l4Dr zJdAL$zp>+)HoTVsO%Q1rRN zN)x&`6y|@$z~xpQ$1Y5x6lsf!424k5Ch%eYp+FR!6%soBH*PYYp?xLW1Cq~cF*UuQ zbWkK(Tk4`F?epbud_>ISlGsQ3G;QZX+_Vdu%>+@Xuo;cMFX*_CK zz|ZqD()F#>GPm30F7oY*>ZirKT&g*sVS#?BIE-&!!L-&b-=LJYPJA{^Gn6@zUZ(UT zAk|9$6-LOjxh!{?SX_iszxZ70l94xO_v7^=uQG=LR32MUY>NjE==7YP31|W6X&*>N zqrldEPMMn(en#>o@;78`7eZjyF|iVQAgSoPhep}s)LfIw?v8S|;N~bdJ7R9X92OrP z58lhkIwRBF{M4gj_GJ_1L-%y0CZ&oZXqOZ{%l8Q;)No@9w)3a`OMJy=RXOZ9uq*~;{L zI#nc|4*UXq7q5f0f8k!gTFdqiRiZ?R$n*(A3sEkUUwXSYs`M$V?Z8;f+XmmmaPko3 z!oSKBK-tFo?co zm~w_#x1bk^`zeYjJhE@_y_0{MSd0Zvfne_W|?Kd^ql%6BA&?C9u(tB0Q;_A=m zNA6r=MN^TRrfhkO&et$Aj?rMIcKwmU7Z9liRbR3LNvI;zSPz0#!BE{)Rlyfy_SgM$ zze(LK7_ogqpG`dS{nDL+CbFML>XY_KvC9hp zjYe!;SJK}^j!07fO1q{@;VqIl$0iQBPac!m*{7J%Eod5=PgfgOz8Wv0c<52b3@Im> z*f*9akb#s>kLk}s_KWhjc}ZkXdWmw)LG3=rx0l4pMHKLtdW}H^I!ywRle}~NPeP6} zujI$4gaH>U<9d*0r=Ood;5Uz(TUw=mDsyu?KI>4RixuZM{*b1c00iFe=+*1(N)l$J z6{RYl^F^Rr`DKBPZJ;A{0(<^a7Q}ZDVtnTnK@^mID$sZlKsbX%%+a182>>`_+nu)- zd$L^awz5@VqoT#Usk6UW4R0wFffh-FeNdkKt|-V>iz! z1&3*lDEnpK{Mf`dFM8jrCuUBmuD>^!8JnI{zJHsL%heiw9j)B^`tM1Rg$W=d-*m}{CsFB&-pyF>6k{KmUXPs8gG^~qijoNSo+S2<7O_h zZJRa^Rq_D`=pq!2c2&Y`)@_mYcxj%bSlL=swI}TqVjeWwHALfhLM*Ccy|p ze0_xvp{9gFhU}o8k(9Zqa!Xn^%5lD%y=PukZpy~c*ebj;|#sC&5 zY8xjjJ-1g8w)k(CAd>qG%R52l!oowgEVvVzqSz28Qs05@BI5~l8|(B_k+vxQo2F7R z69^T~^;=;MHug6hIKn3&9zw~({X}|90(w?i?q_HV5TKm5&@)-pUGGoe{e;O&IatZ` z_136(x911iggd{KUTdRnl2)f{T`d`(W!22TB$bT-Rjl!l1Nq*d%>%%*M}B(l{OV7d z%~X<-DoXJ*v7^FV%FNM3UHd0y%B;nH29nkgTuHoVfzY;pS}MN+zkgUNC!gwIJTHEe zqkf^575Ow}xyT)R*(Z`Dn2~e7p)`NqHjvvrn71#5M{XNwX>u(<{^6iabIjk6 zI{>TUd3cY>-)+bzqz)>v!7dvU<%kC}bkDCNIa`y`i4*A^>UM*}AtKs8x(bW2%U3`J zE~XqSLxXQ6X=k&WvIET&s`<~RN)|q?Tx4CE??vDr*~L|^qX>q|=cd6izqH4XN%50h zSCI4f(M)1gU1C_U`c6XV2^pXCbT#+JBSPE0+doM+7?B$!OU3u=`^++1{Ma;uK?ss= z)LiRd=V~q^jo29Idn2LLDrA+*oY9oLwyrz$tHa?*Lqu`0U&KpOai;uw!H-qlGnYJp z-$PKN@^ITT;H2(M@G2FeqsDa7Liix_^E3{a=BAuf_o6fD64I3EGr*&A{(YGABJ|t@ z>V9zUC)zB6IW`#;{~(p01Qc1-?)FSVZaGes_>&qVU41_btqn7jZcX4IqNZhFpEHYZx=3vdWVg>Prb!2*gcDYazL@LNxm zuxf1nZc@NEn7%AUeoj!a>7_)l4t!NN&0I(9IzO^)r~jp{%n>^`SIks^?Y7tPUDeR> zIjL$R5AUWmP;Z9!nNB@_2A17t`}gij;gO}JgD$A$dh(bqgr%>;2{4NRWPLp-LrVMo zeo2(~f3a7tky@MoP(Rsg{4cwUPf(E|C#3s1;=D}L4vZ3jD#n))c zFZo<2pZ};z8fYvF4Hh0jol-huMG-EsstF`)X~wUXm)CgtKEj?FToU(Z#NJ4(%o@{T zOGFG7iw5e~XcqG_V-n*0N{zY9vSY9r%~JV-p1twN#bUY739O2QNpn}63cW?S_u`)$ zgFV#Z2?*G-eWi;_w(fWsua0d4)yTK4%u__WgrZK7rtTIhZ`z#!Q}=XXd=|*nFiu!D z4oke7{GGV8at_~A;x*voBOTYt0zQ)Ok62dajy6^7HqR|)uFKkS*yIjWjv%ZvmlfSg z2UYA&-Yx@K&Uln+_5;gtwol{m;A_$MMCKISgYXldyU@q|GkAmZ23((>kZ7?5=+yKRFvt0 zXO?-6X-cd#kt#}wc030zytv-3VMvCqx#P34r8JWZyzZM7dm>W%I`x+=cA3;K>4>BG zEkl018%D=AoR~A`6D@qSlv69`)p)Aubfz3qeP`qOF!!S#*4yV1St9A`s!IRA$WpLb z<|O>Rpgw9n^2Y^*0fbx~((B_jfaz%OMhvpXFi1dYh!m~Wqp88qQ|4H&f7k`quoa1+ zb#wRjVr?y)E%dY^U6LWYm)fm{zehH2-V(l<-e|EsGF2a3cW!njqci%?hQx+5>bamH z1LdN7{3RJzALkgXXYL|=^n@-tnI&+yi)EP!gqrTUZV%bl^8xH^yPdSo+!7=T^cEyf z+MQ~`)KhlN>AzH#dx_FRA3gaRAArrxsqqU|_50#F)oso!5)je6q&HnU^x<%}fdde@ zwGqx^xQlM8?%4!Kx-NgA786kNBJbj^ezQ$Oj#QUQHI4?1k>=Do3WI&S%Hwi3n zd>May`V(k(B4iLvJgFoC+*Cz(SbgOd;WO+BMs-|zzc?jLT>NgE{HF6nR4O4Qg?r+E zx-9qpo6Ay#i2urs``RN8=mMRgZXJiiBb=9$h^)1d&B0+ z2`9s^@VdkEhyW%qcVR28;x8nw*YKHh+tavhO)yUB!hac%a;E|=2$z-!h3_jE86T9G z*ismY7%lFcqT5KdXp;toFL8iX*+|GCsxiNQt6`SBCalG6k@}RFW=n7`4SSnHOn>aJ zV{H4{(DUrCH5RPv$GDuh38_MjqTJzU%hk+q(@3aA@}lqyN9O+d-w7>C#4Zu88Ixr* zirY>=!7H!|)ZOs!-CvQHsS15&ehxi}G^@yi{A$1F()itgM28|W zm&a)?;yUJbi)2N|F=ICh@v&z>un-HlZ-ljW*R4lVP$5k|B+hFc)6G;X8(a|CozW6u z5?c+*ug(!K>(&w){$~E85b9Y**dVtGwepBVI9?v$&J&!Ji*i9)UeleZgf0H4guHgA=&*LY0Dp8XI&ofHp;q1jL^#Ok8iJ=)2#zarm}4<+Gj7YvH+UKto=B*0F_=?;UJaqj!4x-dZ?#9Av9Y}6 z$e&-C;jX>JU3qtuxIhb&5~?h99qcfDL^h7X9YFeJ4{=si0}kaW{?jAZirz=)%;*;m zNZ}~KkW1TWpk-a-{NS=dun-c6yasst{&;bX2nCfscHzGhTPl?2Td)RB4czi~auL^@ z_q5@Z4K4h%xP$SzyJlTsRkV)%U(A*YwL@i--o&0m5#R8p<1Q+ukM52%?Npc*{Er=F zEoqAnApW!4lE^9pqku(en2I}Zd0@jLJa(jI$zs@wkc06|HE-MR`VUi)r>b=C2uC9w zOkGV9v{h5puj?(H7v0>L-D?41iM;kqvj8tiwV`wkQ)(`vL4HvI3D1qi=n<^qAuaRD zev&BNeKs~%Fe7g_k?|J}h=TIlNBd<$ z^y#Gpga&zyE9zKyD7Ew$Rlw06RY2IlZrpzcNVbmk4)TE!zS&g7gQ!x7Ku;n3PPIok zV%;)$?)BFM-t#_$GTzeK4|;=_RbxZEAyhjCsjF#srBY->sdvdxvhcQg7Oi+9UpN@o z5cRN_P(RpTi{?mUveHiPC4bz?`t-1I=V=MZqET%}wt4kvK2aQ*$0DV{zCz2qrkJ=x zTX>qlc{B{)?7C^M?avQ#kbVG5>cg2-kDQIs&!~jU6u!gBdoF8lWS_jDT)1%!e;}`w zH@`QqymNJNVsY%*#Fs5wYQ^l*yt21Zak-;(d5~Yrq|Eb?yXe0b0c#tm%9G%FMaIC{EgJ!8As<9$tj(iFo z`x$Mdt3Na_Fco3gw-qMztYE#*_{r3*rVTEHmirvBgUp& z;*qcK+;bpviw^`Jb^qlbqNGj13HpGywq6%)_O>h07G7kyXAztI7)Gy_|a9lKaIz|=fnAs1?hDk#?2fS*ciHDf-;$|!8zD( zU*tjjyMCdDW`nGsgPUq&J(26kp5#j#(YeL1p^cANtM2AAr^;0%u|${Kv(ik!@g zdTSyAKLr<5;hIbTHWh6nM|3ESzHtEv!t8Hs)hXLg0g30k3^>Jn?Vn-PL)`959*t(fWgB|FUlw{ z${>o)Vz~XQeTH`xtJ0tBBb)r(<%3e>S*>8>`k5Ax#65=YxaZ3Z<`2`rCnQ?B-#heX zVP>%fe7qzNbCex$iM)(raqB6hGAI`5*DPRB=2(X+Ob7KvMjLlgcZyPF#eH8RoN-qk zN^loGg9}Q*PrD*HxnR~J8@<#*YS9yfU$+*;`M|C>ABMcwJ_eBI(Ewp%*!hkPE`SD z)6p~+!yTAEo-ICZIQ1@Fy;pBt+R=LNGY2QR;1>SIQ%ZA(^nLM^l$)3R(e0>wPITHW z$Y}rMDczzn#=)zT%5um$0*#*cr@WfPOVMLUeLqgw0NA%KbiZy!VQ%zFM!Bk@3Z_}H z7tll6QfuJn=k7Ox)ip3JclM&PFX{TzzM)S*Z?EZpI;d$^M{VGghlEi%AA+D)h zbw|k`NUT-M{YpOXvdv&0gb&sQijnwSAGjazUs-)NWTSqIB6EmAgg-r_ht8p^>6=94 zc*#N=Di`mwFR*j0pVYyWVS$AUXTeF@`oA5Uh|w{nE|{q09cv>?25F^R@k9x#y=Y#L z)`iSS-ZaI!-+?mI?L_8=fuzgCxAY$06pl~2$n2AFzn%y1ubJ#Q!`ixwGQ1Z%Lj!)r ze|*SwfI9@~Uk@)U8h1;0>GL|-sGkop%JijGO)FRxc+03N;wo-?d8O5!cLWAQ5UHnz zL`p<#H$7Pm7O9kXjvFfC3XgP!C&h%6>TsV!%(s#wKUph6F~=%4)Cz~K3(=Aw+R}P@ z5ro{gQ;s!CVaxVUen}HC3&$+UN_wt8n7Q&hxOIE=+BHxM&9NTy*-U7JOQIEtblDLP zvLP(6s3qH$CIlujh(HCqQ9=qZ^ITWy1#*_g+1J#CP@8Akf>}kRh83$wN9yKiDz z5^cC188l6n)U*UB#WRml%X^9`g9>_2V>6vBa&f;9v0v()b~D3WM$_TQfFf^T!MfN; zuOhQxuYm~lrlR;0Y^SF@YDlZh2I=IF@Ek?wA5C*Q=TQ7SIe6{KjLq|*Tiy>hbj>8! zA)r-<4X|JM7+Lu|_=(pW>>UC{0o=N+6NYT8=Jv(ZwU`!^ZUJqoOiNY+0b#v#T1A#t zumo_INqdw_d%RbE1yL8W7}q83XwCLqBnd3KH-;^w z4ZBfH;)7?882t5#8={IdFwZ2>S17_OdcRhg`TAbfjQ8?x&vO)(`zju_Otn}mh^8g?mGrF}<=4;=yT zH3oAZ^d=2)1_n$uLGos5MS2r3x=RDc_C~lU$g|qvT;s9W9hV1zcaJbFKw21*O%$>X zjom6uck8k?b1l$gOgtABw5nkmu+si;yny~8BkrZB`POdxraeyRyn^bod+zJlP(k=; zKDaJzyN17c-b9#Ex!+_?Z?Nr6FcgJaD!n~~wI0GVX8LIMSMa~0tBfV+xYZA5%k#lz zi)O?AZexKLsGG%DwL?Q!T@cOQv16{Id9M0iQrnyWZW@PWT|Yz4bJ}B1KlwgorQ}O| zNt)hG2bK4u6GiI}D0FA9NjR@)Y@z9Q<3D+6D6PGt_S)Q~N%sY861U(SFvpiy7_lO8 zye80jI=d{k$Ry@j+tRG=`~2=*-2G+>jwU}ryB!o?Yt6hCCa;VU`W|0vhiFiPh4fUm z`*=-XdM!EGs|OgTAAbnAw{vauX`C;3*PcrlDWX8n;dFR9Ee?(k}*6!lP)41mXz zmu?pg_<&435p!>gX;zb;PdyPfAun4{@N$(5@a!z?>HJe@254L7^7&aRkz?fAl%5cfH>gN8}kFEZT z6rwxYwt~MCq1pLE(xTO; z(BAi(`8q7V_zimKJe~~4wXgm`HG4_kg5DD1)Cx^jf(8reBjprN(;hS9*K z>F?YLe##Q+M0hg+1KdCzKv;{vcGT=R&10mIF2`tg%9XM}zMICrK>iF~{Hfxn1LK>8 z3wo=w??YO(u z?NUxxInXv42Ywk)r znsjcNb2m8|7CiU*ApbZdtzYXI`MjbyA?3%7^2`g#it@RqPQ?T!gY!PJ4TACA{;Qe*!}7Ok!mo*CP7aA3Y}kE+Dly%HgUx#^1tc>V8M%Yq1wNM zu3sy4|Edq<1y4VHsq=59smmyq^^zYs&t4M}T|m?wHS|0kC*gu=1*&;Sdf-X6nhF3t zs^Ka)V)E(-^^;c=Ptf?5H`7*ouAb+hgO~8w{j`BSqOtkIUzs;_J3qpbH)#(ogjmcz zwevp<1YjS%?Lm6cTv*vC31s0Fg~8NdcLK0!?R5#8ksXU5PEhAh8Ho&Tp2nL$HNjP* ziIvY>_9C-Iznk-|7K?5(O2$eXw2IJ4FqDdXN*H&x68}$iLmEj3EtN!)JAH&B!~hZC$tKbdV9Y1j1SyvE*Gq*Ei$$Nvh6tHYR?|Px29sxz zy&L{n|A6gL!SjwX+41^*_5M3;9nk&O!H*Cq6nsiP?UgqU zK&ikD%SH#L>UvSjRX#vm)H!v|zyBr4>seX&Boh2kfeHBw9a$K%E`)IZ>%@auB6JTt zAH|bB54V4M5GM1PR!g1w$O2Bqf%rky^1uTAEFuQvZujUB&#sH(rsZw~h&K*=vl@gJ z1BebnM*`}4Z9jttkAz!|DjEmUR7x`#>g^A}S&f6;?zd<<-Ml?JV|zNc9{#Yf9dCNM z&!st8zAv52E~@`kQW+^$lH&Qz#`M{?E;~+q^U<0&^~I{Y?PqJTJ$)rLe=ijT91iU= zm+lbXt~e{K?(>XLTLT(Zr55S9E{&7C`jRt~FiO7~E%cvNbX*07fY4)C<{_fp1$@O-xX4feTW?&IBM5Z zg`{#REyEe3;w?iU%T%#U``SasldQ@v9j$_wSkb@z6crw++pS}5V6lvtQ^12^ zsi;=ZGh5oe(b)D>LkYAz6k&@*-WZs|PDETDzREV=Q~9B3MG&ozqQAhrfs03mBdQ_c zn9S_ox%JYWkbEX!`!)_;imBV$>_>5q3vQPJ4u~Rj$`ib9^w_vl$i^k}fn>#7(nt8& z!s_M#lW^;Z7!FDmw+e{Gx!n-~UDNM|76x zBw-KiVTeVvBXFLGSyh-*#4U~l^prh<)U|_8)5RF?N0_~<3yQa8oPXT5CMU!%@SS9Y zB+JJV=x4lk3>UdM_LX*a+s(7)NKQzt`(un8%_eSPEP&SjX`J%OV3C@8><2k5$EZBR z?%C4p1w1?T#)#@5eR#d!vog3EPzWS{IN}{2uJmk7Ue82rNy#?%M;Qnok%YgyQ zblx#xmPia5NwY0-bH8%o-p@L9HzO2ntPhN=hMY+pk$&G;zo~?~0LA#8By5HDV}afn zHJNL$gw3?vq)FnpcM;bQ)C0kIOQ6jqR&jRQconMplf1||^^vyU4}%>%u?}mA4t@JF zD3N{%Y^Fgq*%ndp5B?dekQRuh=xi#NIez~6*f;<>xybncG;YZv467E|H7l~{^+!Buwh1r@d_pF2qhOTC>lnv+C9=&v$Z`~U(75xvI8zVmz zE?Fc0@7k1QVQtN_F^f8|pTi3h0noD{H7=AkD}k2V>gD)5SH}%utxc^YNhk%=tI5fq z{{5rK!Xxr6rDPuw4<6ACyT#8qOb3+_I$r~q9>QM(m$;`DXu;lsX~(!DghGJ}1`zj{ z+v?wCM`ZB1NZ4A)`Dq@A!%*Ioc*2uFfP)Fzee`(Xx0AW#nDNJ|EGmC^?-1<+A>a+L zkL3&=8zmGeeH@L6Ku3%X>x+N8zptLdi1*5F$h;^8HEHEjQZO);^)nM+!oTyAfOy+~OM}wHxu$VH*@Oj1HSZv1j>6+z7r4tb|S44@w!FMCAZF z^gs&=Gy+<+S}yIMC7gqx918uoQ-%vNf_Uh#=C1@$#B8UBa%d8U4RTKh|XV&v4dsz zth#HSyDaM-*|V!(1xpz~$ZiEzhcS!NY89RMLQz%k_$EhEG$G$W})Jnzg2~iWE~L|T_+?$?j`+TutkS;cLj3{o@a^E z4&-vx)SY0+TVZAYr^!o1@xXGFtFJ+f@HK*YjBcO$wUKu&lH{80NtSfV(HOindriKI zn>#c}7L`UYc%d~`%cxt@Jz`z=m_dBwqI3RC1rNJgM`4*+fAya@IFy&o3P*igfZpq=j z$_NGLdESfi{a`9tjePqeDThsilW!f-NLbv^ukyc{pY~g_p2rb7@xP~wDSO-m(~zHwfxE)L)UL$72B%)U#gX^W zrg>ioUz*3ZstR$s5xjKoqPvMz$`PzCVBKZ_o@-O8cr!6EFM!N9o*LDn2R;pZYfBAl zsHyA7pZ?_5#*j={aP$o%a#{tNzt^bTrzsZ3!AgDyHsudL3u1_kT*TzF1ram0{K|Mv zL}kjibQzIQqm2oE?U}mb`mi8jv0f+MD2`2Bn@J`_r$Xgv^omy(h|?iCUK#=d zYJ=M?>#Sy>4*L;CN8~OBv1x`tXsvB?DW&UN+0E|vm)T7&R}-*@i?PcdG?gc21+(P! zQWVf!X@^!RaUzBJTh>*TAp*hFTDcww!x)Gy2SbwEQ|fX6DgRu)q!=xUC>nkT95?{Q z!am$;77-SaqKRcxch)^ruiAEd20-(MI_Z6KZ&ZHI(9j~|Z8(b_g2LF>OM+W;s)8fe zEMvu(%&~CRGkSFQ*vLprqZ77WPpCEh?yw{%?Igl#R@VYUpgeHavM6hR_A}f~t>qRC zaXxMMbt1R@k450VTht9#K~Qqga*p&JCh!a!G+}|{PxQ(N6c}ELGLC=0eW~E2*?Aau z#=7%@l)82UxvZTWKe~DOiq*1=V_eFez)X>Tcx&*}^e&c%@;hM)-QbePmXCT0$$K|f z(=V5n8tU)=e*DrYU^r!=f4(<1%M^qbG?Z3AC_^{OUoZQ#_gJovzt(J0_e|_DIHNbk zXq4Z`W~OA6kO(~zxp93aw|^+KWFtQS8Sj^gv2rkWiNmf9R(P}FB;@0qiX}xJe5p~P zzVo8>znQ;mh>b+I>|f!-9`CafUnSdxTz{7w( zfns0IgwO3{Qv1^^%s)z-YZ`#Bx8e)8K(ChbJ$E&xBZ9c?Vl2;(4x_n(WJfWA__n|} z44Gj#C8c;1S0ekBePBJ!Z`~tY)Q4@hR0$IUXK=0N2g<4R*eaz$K@9V`@N^fgko;r> zc%b6ATVa|jJ#E|d9kUp?ijKBuUb4+dULH&ykEcB0g zXlt*oD%80gDkotoQL<~I%pz&K3{Pq`nHkBTU8RY`JlZE?>OhlzU~8V!;}+O%Vk%6}i$<(?HF&VWCowAxf+ z&6*0sptFc3UcKS?Hd?SV;!ImX;w<#x)ORY7Rgw=3yaIj?mi z#M*(g(YUXhycPu^C)-o#SaZ6PC&h?%l2dWL}yH)j!@ip>9;s+njuJua3K5 z#|K#VKbB9tZ*gBLykDrT?u)He#=w%7o$1Wh|9Lhuw)7)TA7~6uR|`RLtJE~-F>`9Z z$7e13?>m^nV!0eyR>&dP<%m|bOEM|#8y2KdiRbO@IGAI(q7ISqs}2sUCiHU+bl;?+ zv=0Zue;qwgPByenW0`PN|5DI5Eq8XF;KTeMmoOcfE|*x&Gk4CWdtGu3V100u9)9Oo zL;C+?39~F;zso@t6eG%cxhmqJ;e|B~dfG%!C-gMCv5id;)wIAcyf@w{3B6%`r6V@1 zsm%@yjN$6JTb#iyL_VS6pqbq0hq8?6AJKsy&5bLJRdga5D>k!1 zoJ}^aMetrAX-P~`)M?3V$brN;X8JwM3^nB*V=5o6nQn{FZDySbGXu9>9CkD?l(wC^ z1crDX+b6{KtYO>HYT3UD&%dvnA@6R0yt&_)adjyzLRL-{?O$`4j2^VO^#^q{r=}{d zhs=dON)NH!FlY@#xY!ar5{!17nMc8Asv@IRW&bW5qJj`BYANTDx3(o3Nxbf=tS7{QkQx=VKyV!pbNB&4CJ zZ=d|ej9gowH_{J^+wPQok_@EfxHEZO1x;T5?qMzg{GW&U|2)k9=VAUY9%kDUjpoE{ zBl|zu1H4X>91SyyHX=D45mu4YRFn64iJ*^Imz-VaibzVIgbx+CWgr|kcZ@b18y}{M z69kI5W&XkpS1OH5YKDfrD6I|ol3Q-(TL#9qB#YLct`Zu_mP$J=UbN*65B@|yl+fN^ zmP!@9m3IoCn*&~SvT#A3BC&W!KV1L3{#XaBMK!TEc1<$($}rxHN;Zq*kU%`uv!Yzm z_f=y;CY_}(ufTP}vBWgG#Q-(LSiWyeCiz$zqX7^nOBH@SZ|aW%GJ##Cv?G#p4zsX= zqa?3k1j!Cy332&&|HdM-imjt0Jt;m=hfeLsB!lvABDmN#;ss{kt@mCAhSnL2>O<_WMo0x0FeBuTxlj+k`i~(Iz1)^(Jk#&#Y4T@&iOGr;%u;O0g)QYTjLA z^|5|N$#*f_=DX#4U0rIewfl@ra0_ z0~YQMYcKQk$Nsh!tbOnewv%xMnZa%;)57|yjU(&8MTlm22QSw;!hM}5KAS801OEN@ zv;LjF>Kh1?_kvslVD(5*Xg;RWKTIIwly<{{ug6?lb@Z literal 19879 zcmdqoRcs_-nxJd5U1nxxW@eX}nVC7w%yyZXnVFfH*)H2h3vbW_xG$_DCl$ zLaB_9QjsA=DF5es2_m6@{_z3@y6m=g+mo6ne`|kNcYhn?y5`whKZ!Y6-E34+Clw_8 zMLM)RsF;-`Ff;uuSAh6R#Ym@oRsVPCKsbZP5}{jDEoVE5{C{g}SH8DWwfiaxBtW=gR%0TxC9 z0uzMj0|7<*vZEel&Ymnxbbux3i!H&)`lH&!L>!==8*dEKr=Ducec&&}4de6ynwv43 znp+>t0$JO|T=Bjy!y2OoTw9gEV5VGhf8(GyH${WJDRsBtqi|c(>A=Y%4H$?Z^9c!Q zWY&G8rIxdtVVzXQXF=MWnW(Jzk^#O>ZDh&Kyu+gLC`VFfl}LxA@sz~%Iy5b%b~|;B z-k@jD?r-<)FviY@vBXqJ?tBn2HcTb6S8WI7z_+$~&7s+b3}W7d?e;b6&I|BBRQ1_K zfi#!k2wz~%-c$|B;na}Cm4~Mg$_t0M3WM0IFN1z}x0O9C9-`qfpZFyQO+z2EU$I@^ zuMBx(xp*wJ`aKayksO?iKy36~klRF!rxi`*QH4;~ZxT+0mw6`lzah>cw6p zv+{@PkK5*VQb`GK~CJeMpMk7OLhKwv$i0AHPN}njA(;@N4A5jC zM8MjBgMt9ShN)-)O#uoM*n%^HT6NODwF6!H)INdU=vRPafI(q!q2<6M;Ohcm`V=5l zfQ4*L0^S47{m+=QCgl6v$CFMwzzINY2Vp^SfKURlDhGftk&P>U0?DL0fB=Ip1pa`o zM=(Q91T)e|HfKnWrcUTPOb=Sa1GO7Rv~mMKLUtmMz=eWV14@7-j=aP`22SDCE?I!q zgk&0+f%ZU{kAf4jgHi*|1n-8^(6-5cdM8}g1bJ9CC=~`Hc&NO4LZ>JX!o>xZqf0^( zWC5U4Li(fVpu)-uN^Wpt{yk}~B24hQhGg8$$_l7a5C&j2^ESBJfcpM0&@I)`=O~pM z$vjh;?Y(CMX2THPi{Npe^cY2?e3%C|mT@pyHrf$q@LEPuGqWF+eIi)u7z!@fEKbrU zAd@D<>a+_~;5J3>RBQcJ(0wfd6< zeL-B1KIL3*R|b#NuL{yV(JwF3bhksnQ*i)+%JAWHp(!2!foH{2;rE4Pb*n^xJhyu= z9>5q)(xnb!5o()q1DdgpgtYfthQd@&(}Z8wyL>V5huD&o{nR)M6Ia0!S7g!N3Xy0T8h+ROc_TGo~i);#; z5J4oOu^_*A>6-s!LwhL(glg%Z5WO*Fp$=#;qmD2JTN)&tCUmbNc^eGera(;=pE5p4 zXh@S|b0^h_mz=Fcf*Wz^hC0bww5PNo(6pn8Zbl4PP*8Wc5YU4vE}K8`V@1aB431(k zh^1G@s<*WfUwpcEBRy9rm*8C82TPdDkZ-K&61I2S*Sd7#aI6zRZpL4^2sf=~gD_z2 z>nt;P6OmTlbx0nK!n$m4(jR~1^jsTFdQ5G!c06kmTELQoi^kRAGDHimn1V}YoF#9V zX}M8ObHsbc!BHg^Jmz0Zind@jND;@M6@-ZePyL<@XD>P2G7A^=WScpVL|m`pO4jAL z!mUyXfg-@S(b`7ZBUnXrjuN2*s`(n%*ly4mpCLVjLSlEt;0itpm_z+?Os6OU?KMu3Wy~quB zG(v_Dvs9kCeG!S3S5DJ3#$dw1RRRA;R7RULIH>!?bg8fynKL~$EF|!_jFBs6@i_Bl zjP&^KKmqU7E#t=gyF;uSe^`rC)hKXcqmDf zooS?6Q3G_^txW+uUr8xUY(=;Ensm8>a{BZme+|C+Vq4BR2_powfLPO(yWqgC4d(u; zR9~tZs~qnwl0(Gp+{xP*t_9*48!5JWTcx|BdMri;-Xv@8u2#g;Q*mFOlz`?{!H-Vt zwn5l@+l$W= zCK2!iG(1IY3yT^!SWLu*pA?zMQ@rulkx6<};SP~aX@K96nCvJ^0g_IFWpj`CbNvjOgo~2n%VEWH z(-Qy;Bhyqwp+Q9i(?k0lgn*8MvstI3z(UH z)aCr1#_b&V@mg*A% z%^$h7*63E*Y`;Rtm!!`#*J7u<>uGmgzz&pIqp4s`wBr6GWc|E4%~bJb*R8g%!=LB9 zesLHO_8Oynpz#ak)4oMjL`l2!!nMBG(Y$d}iJy)u6#CfBxUta!rCYaTvvm#6YB>YM ztFU z13Ju?H#)8|t_#9blO6wa8zs5dUwYAYW+7~0Ni3T`h%j-SpJ4LDz+C3;6f?S%yu_W} zb5@hZG~V#2{>*Jn+i(SN3?@aiGrzWvHZ?R@n0xpDU8)t(V9N80|3|E_$IpY#U>5dPf*WXe04gsQ`omM{*1S5u+b}R-|HO7@X@)3itTQm(x@fw?$Kk`1#S&G zx=R~3lip+k?U^psRovd@oDH^Q2J^-S-S2^PWv?*qp38L!NKw9yeZ{s|w-0!(QEymL zs+S4d_pp6uh3|>;Kir++*nW!p3|1ea(Y?{BLC+PJ8m$#Va2rGRT)vu zkLAg-C;Jhb?rDC0MdZ6t-l~JW(nxM>lH539tv@75w)m#|qKY{*X>Gz0Z{IJZhX0~` zq6l5ARJydy=`KQBexg2LcFTOR{9*>sNI41IEc=O4=^v+wtEcA2uP*c9e!n`2jCU?eu9*N#_zg#AMZuSVD^ct<`E8e;KWa~faFUxhWeD6h2X zodk#}n=;Y&sj0M>{anghu3*13vV3cs@_+y}f-gxa*jXaW z3WZbZ({cIA;QK=+|L_Fo-PEQM0g@}Jkf7#$R1xt@``s_0`U`JW;CzO8x!%rekd@yX zpAisn8;ypub2K+|y$=)_z`5zx8mD^GeodduQ-I{+0o-vCo!X`XpOhm5C7r&CHchQgu$)X2r5`W0fXQ!w|CQ^Q{1Uc`aOVD;r*!neRCzM|jT!ePO-GVm+6_S<&~NKMV;5Q<-}Uq!>A z7oxc(W5c~d6hb9p5#YrJT`%S0?Gz@hoBHPNjunjKi*_=B0k^(%=7`wBHYsKA$ON^E zz3BUvM$&mFO=SJ@Jl-XKGSf0%|0<3{S=Tc3n3YmU5KEBg$dB5TDi5*4P6fSFb(MCD zoGPqvCq=N(uH^U5L4ndQdCNRJ)Ni~{HD{Mwkr~lURalDT8occ?lsEs4>buU~O;wEd z<;4mzTji_8Hi<9Tsao-Q8qD?~qy)-(U?F^Nk<6msu8{puAfwc%LOckZQmzIp3wXGq zOna$38SdfOt%m#MVbF|#{XsZ?cYtMrFECOo(_V;wk65^MLF>?qEmx0NL~Guj8kO0> z;clc-#jd|DY^1uvpN?Sg+?ogJh-3OMpX}MWZClEYMZYU*K_=w)&FPIPdVNLP2j|9B z?wcIJIDzl|XOEPlc9d^^ao^lhqz_kN94#p>$K}WU?A$@i*gdxeVIud8cy`-(j;`G=d=YZ|Z1&zx&X=z#A*A~hdH=ht9E zkw;OILyQ=8Quk8g0Fs6e;`j)#68c`sCgS|HxS}ZzE#XYx&X)acg3UW2J!~0Y?p9sS zPO-0#;|fTVp_Wbq%)z@rhe zmTfE%vtZfrMeDBPH}9u2jqd(3i~sHk4UkvzyY|X*Pq@z~t^Z==B-4v!%17HBY*(EU zy~DS(ui5++M&R7@z0S)K%v)?OXs$&n|huX>yV-I%aZcpU*X7@0IWm;Vx>xX^c) z@oX1l7YX%FrunpTIO^$LcB-cZ?t2lMjk|JAGg|GHstNjQ!4l1)u#oeoGgM$fonw9& zX&fXng=9tn5xXlC_VN>)tGztq6AZBhhiRNarWsarE{Y z)~hI*m9tu5BX;)UN!9ErZNb7xRcv{q!Ux5IW^T2@f9w!5_ko$+UxwsHewf8Ok=XCu z$S3{u*A+WdB#9zJ7TtD6hiCI-8!Ox`VRDNay< zoB&!YAy}Jr=(26vNRCMLQq>=VLixjUDMhxX-%0Z*4R+it*YhKCMp}o5Ay&`LLl*CY z=VfEV8~x6Yz0SRLvu8j2<|l5hTnF_m^5>8PDVyZjm@pm^qC`uB z5@>q*zNE(`DLrSRL(@Vp{W{va6@Qtj5<2M9v3Fo5-Feq%oM)G$|~O`lUH62Hc@A$MssEI&*Ge`CEm#?Wpv78C+p_i=+hX9CxMEU0IKQt(edEF;bD#el+w<0bHtVsh|v z|5b>->jYC4csbf!kAN&;y^Ax|F<6(RADbc)yS;llmO?__P6W%e@YDkbhh z$Qt031ggufH_p0=U^QbcL;APBo@h%jlpcH5FM@5`oj#WXu0_`mU|LsG8R8a*xRX zjgl`EGByp=YWqW#!pbv;TG)t&WmNLNZLAg(>7&ssQgz(b#-9EjH1?nS-8j^!VGe9O zvofzjz(HgYfCUS8H-PIz!B+}?r*N&9;d>dm+uM^gzp8f$E^eKiipl%Et=OY_S?q)P z#xM=kJ!wl5f(7s+qHdDUM09-SL}1&Yx)hdWF0A^!EUbPX3~;9!)_!>&*ZS$cS6(cC z+aZ&Ek+&u;et*}kn4BVnrEz@LZDTOiLoneC*&sFwO~nSK*}eMrFentjDXW9V9ur{$`N%wGb&<8$$>Pb1xMdf$W!$Z9F-CGAU<6xHp zjTC^ReQE@PF7bQTZ(vusYIxi<;eNf2we3`Ra49p#5g;P-`Rr^#q>@DLB_6JS7MU;; z%@dk{pz^~mV!e38!>Ined)&S1y(sosj%Gic_WS09Ddm_|c&mXab#H|I-Dk4@@OGQ6 z$FU^4dFcs9@CnpUhWWHtS~9u4Lpn*IQ;5WQP0oZXbduN#>*2G)D9P)*bATJ1^S?d>V%r@!>xp!!fV{bHB6v@J4#uRHKxEBKs$Qbtv^HZPL({`(a;hJatMKEdgHuOs|TyvCS!XKum@Mfa#gAxbNt4!1cc zU8#NSL=@A|{i8qwu?^j!BHRdgfhRj;R?SGrLIOIBJWE;4a}u`3Js&SeFc@U~Hn|8r zf$EpHuSf>d81O*6ebh ztT?6o-hDTd$+M2apS2d5p5KE^8Mzj=akWA{xe%{sb+J5j>ovOHGY{F0ZYl@9Ho}M` zJhgirzP5!32g6lqBK55xofdo9eGRXR#SpZatGBTNJZ7O2ztqVQ4u6mmU}GwRv`6@# z3ogVJts#)kk@JwQ3Ej9teqYr|uVvX&e;<$ngw1$DSv2GV}cIiEbtGm}=57PlLaKAYm-$+Zd z^{==uXE1W)qML7@13%3|*1{}FO6)$~AVPABo+f>L_VJ2aG#>y(PZTL@+geYawz&;u zQpn5k6vg)!T*okg97e?;cZL~w8uU@lXK?r$@(v7`U`ExNZ!2GqD8$2kRq?58DHxpbC9OGGK(;d9iLOL(4uK99ds~^MKZzURk$NEO;%7oI4zm$ z{O#QD@$jC96)Jf-m8wPudk@C~XHi2Y7K1f1?;sp;`HKVdhy@;N&1I93mzl@Q*uN`V-V93L9uY$yb8_K7k>GU%H`w~?vTt{(`~*g{BM|Tw0|vQLONXC zl|t&D(3UBJJ4(;)en!Y&EherIlY~;-up?<|co-D6(t9F1En11>jz9hegd>xZhD;Rt z75LqObcbFH4zoQqlcV_2KC_TdNY^SjIhCUB!J=4@L*?`{ z6aoE7b*oD4hz)wz@3lmkWYDI9HU}nayp9f}lSfR4e*Mt_;Tis*qJ;IYw87}rxo7W* zc*wc24EO%dk)L<{Yi%k!|6y$^(xZaLFX>z`L<7VL3Xk^#{ZBl0)Zl<%15duQH*6=` z_(#zI_1C2&#^>2XFEK4Xcam*w7eA(m_dwj)ysxzM1EUhAqwsllxDEF4rX&!G|A*QF zzYp$Lsz*Pnk*5A)o7-P(duqQD3y0F^%Ers%H9DBjiOJU^0a+)d#-c7hrWNNfJ(=Dz zsY}>?8#ZAh_9y(UP5DFu9J7jIiZcK^2>S*^3DAN=+_kJ4#D%CbT=Y0wly;LdvapfO z)rKz1#h30@d<1WPyilMJJjAvnQxw|#;~5yUkg7-BgJhsWSkYH|ZvvSIISg)(tN{I+ z-g1Y5w;A-yl>|o_}z?R;jFzPXH2F3=o7(xKc|M8H6GOL0HHIRrZhX!WU9r*!< zDg#4KImDkPITg~w)ZXJL&vmpZHvAH zn8sa|+8B2jlB?R7SDe{JI5Su`kp{R8`UAUxZGG_Cb&!tvkKg)1+3?MydHTogt(=m0 z2RoSdR{2pt84ittTtHbn1qK!%XF6~HiEqGdUaDxjVY_5`N3gBCY>e!3Nl~0Tx-QE)!KW8n}9C&+mI#IJNmw^gX^~v@D6BF`xcla586V}Oax{Oo<2fn& z7RA2ueJq^TvxIC~o9_>{W!?r$rxf|j&EILQGyes*+-ENI9hT?@gXlIgonVB zUBs7jKN$sR$erI}&9LPkYVGm=V($Tn+W#c_uL74#DLR_HyB~#7IJGoNqoafLYW3rLN+)#C2(-Gw3g@jn<`^nKt6 zV}b_QkICD4s9(BjaYg0$|F6MWb0siSX%W2TsF*|&Rgxl?(??jSjKOwiSr}_w*;;`O z66>Z=2-!3qskm;oIn^m)ou%Fw?BB6Vh%Qyq(O}Kaj}C5J0+Zt44fxbjm{NA4dL8Rp*=&MMQenJaHY5Q}ih!50@f2tD>8 zW?v)z2iX2Ycmat;w96<}jZtEYtNCH8SUFPXT6qSlhcBe6fUSQ`45O&?v}jAswuXx(~rQdct|{?�>m=;T#E zQB`XqQ=65c*z%=3KX7e$67#@ zh~=_yOgWp2Ju34yLvPm7jN?}jy=4pR8gKDD?u7!D)~NSF@Tk7N6>;k(1d23FO{YQt z+4M#GQjBKVioMn5v9#)+hgHTH@sjdsZw9P%SHAOf+d*fOx5QPStmwM2rqK7~2(lJF z41Bg*2f%fg%cj%{d$V?SbO4@f4E9kOPI}N zolO_HFmN|~a<0$8IB;^BjQKJ}CZP+FQ%nqpU{kGj_ky*YRdG6TwG#+T%)6)aTY;5R zPPX{8oSRf*#IhPo$(eEjrjwk#m;iD=xNu27EyyP&t4$ML`&$m1;|cl9WzhlDWPnhS z)bizf)%3LzFB{*2s4?{9a7q8iUx1^P_>UEq#sNL|*g_Msl}WzhW%iW`Ipo{9t6Eo$ zGv1jX@%+?FFX@jtIsDG<(K+L}%J@*E*0@buBk7WPGA~o^EXTPxJL^|9o5Huk9aAlzt6f9a znoIid>@x?lSitCJBfUq{gW`KII=uN(*v0&=>4$DdcAq3#I}3wm^8J!RGk@{YqIYL) z{ke}3UdN)m0_Kw|k-9=&akz{^eajJSQ`tP?jpn@6;{Lz7+@jV9?~H|AuwiF53T3dr zv~{dHBML&SZ^{Bqmepc)k%b^qLZNzLx}?d7{y={oR?YG~g`Vm*FXA%WTe(X)|b!RPC+N z5_k3`F|`qY+`cBlVr=PTf(LSWLNgr<6F&@C0|*^B2V*R)nuN=E>v7 zfMt5Gx|!@Ly zp+2h4;VoqPbfgWMk3&Z49kY(OZ$_`g>k@jPa)d4ZpjMB@HD4ePbB5i57;TQDO+vIiggl-D=Q`S)likryvm-} zHp=`O%IRQ)m3peD2K)*bwtR{jTng4n=deaMNoKnSV@v(Nt8G0YAvq$x8WK}avq-Nx z!sDhAe5zEeQRFS%t-0Y}j~%gECk1_ztZG!=B`CI%sPc}4fe8OjnS|62xj9m3LKy38 zc`R4~AL{f*m?_;ActK8Pk;RmOs`*g=9XYT7Zf0TTc_ld%$YqmRB@Z-|EU3$_kIGlI zJSlZtG$G&~H~TwwRL^2c-3@z=2wFpIl9uNxmcu(boS3Oxw&Ym>4QnL}US!o>YSmu> z3PHxDzUkIijzF_b?rn42;k{&kBhAeO=7j7R2scwvPaPd(r}O>CPToW1@A*z%HK_Nx z&lKvXZSI-k+lWJ1-u$7*=(_l6{l$&&Mjfl%!A!u2^u+3(9@^Jddv740lfK*q^r417 zc&m&*=Jc(3W2^8~ugcz~`y|%dWnO&8Pj&6J$OnAZoFe=xLzU~G`FhcF*6+`Eul7i| z4W8dro1#Bd`gTjpiC})iDyQe0bN;>FxPWR_oU<-nWLW8af!CUzntU0h5P=yiPau_*TK!=Ud3N@JCg-dv01Pf=sUZ z`|n$g%`o!V`!!D|!K;Vie&^-*UC5p~){8#0y5>)*W7}DdFMX>va`wSBRqmA&h3fNX z!V#Bz2?)wp2cP`YMKIgD4KSQ?FDs~k;X-sH$dIDF^l?6iGd`)GX+lnuHCD%0l}@E# z86tI=WYYG0%%HBvi2D>I!NXe7f11`?};&%3c#*j5c!23A8ZZZTRv%vP~i!T-UyPykA zv5sK!ej4kG74ODEc&)>K;LRWVNlj~a-?w%;v$avST=`Km@UUf7yw2oVYsOGZGOh=y z*BwLR#S(SlqO*KX*kGsDSGjr+iP^9X7*6w{54=zYHc@XZw)QZ*h!&V^IDiSgE^o|* zr{5AR{P4WU4W*1hJ_Jrb5zq%6ql)EFChpxJvMZS{*oUiogNz_L^vJRwxGPU8t}@H{ z4%QelyJeUBhI-0T@@D!N@F2?az}1sGvwoU`JOCtbe_qyftn}8X(V1s{Ek4{WD1ULz zMD$#~FtyI8<);7X#fs6M!4*J?qA@v*9Dk;p^T6ZtA`!V4n>tGmC#3YlJun@!lGOYM zi3;sljK9e`_^#D^IkB8S=6S2BF9#IW5|x~~d1^4%JHq@s?OUkAozBpFNI2A}DD%3)v zw-m@{A66exqS_Tws5d@y4gD=8{eIWNP3+4jxTf{!M9sf%a!WSHYL@{gvn891Z)jge zwtIz^t+i?c+X|gN4$?m0>(Q>(P3x_{0V0K)U;EALy@>e%A0}&XMAAc|r%#~^=K7Vc zz1jTlD(l|hh0ze!tSha*4P?Q@3;TQc1A|UJbg-)yh-!CIsh52fz+NrhE)Xc!>`MB- z1lPTh{`2@>;wZa9e%%n(-zM0wKGE}-5#CMiv6(hTq<1-Q7n9m&G%J86?ZV9-QRlJc z+FYNrP`ou%7h~L9-w9xv)-z#z;!J$>-U=(4}8Ot^CsDq z3Ct5r-;8p9;|~%rg!0UnR1(;d@E};=oaS-DkwKEkLymN%?rwjWX&b+AgnY-^y?oEs z_fFssSb*x|*AA0Qrwd=%eZ65iHgJ3$_4-12)kYU? z4{r@(_gMqg0(TMM-(=aMdf|$6-(BvG_VGsK^@8r@!|kz!Efa1b#JZR3Q>Gsaq#-_t zJ$2~7eBRWBs+jZa4r!m`a^3}4ln3C-_N(NmRzx4 z+^D+NH$J~&x4*q11t_X~EIoVVzPEe7U!VQhaV#?6ShvkU;hrA74sLU8LE*&y@&tq1 zrQGp8p32hKVNFoa`M620O*DgH^(y6pP0}j>$5i2LyFn2`RH2KP4_&;cx3PGYYC(nu zyY_x|&n1Rso#ygAwkKqZi2HE25bWy{xNent0muN3JhWYX;F8;Gg$4a&~+O_N=|xQ3)?;0sqfEWawh&rWu1`b~K^0QNPcX-x~Nd zOGFh^4w)lTB=ynVDA-1_3cf)h1fXyN5v231U!>vJX2}M!GVUayvNA97=qNp(2KhU( zynD0Au`iFknGx{<>b%54NCZrn^+uI%i>+>vq8}pC-oVv>Q{2^5lsERjbYP%n27pSe#!<(efUtWZSS-Iaul zN{g%uopB0Czovace-Zyz2NIWZsoe^Um;kwqkc!5<5w z!X3+i*=)g!xga*lsu6LszWLAL{TlG)mKpA0Df6#g(R_V8zl|3g?A+=17n>tQ`WEYN3dcZo zhZ)`8KxOeY!25*O+m;PPSW?NH@F2 zMaoh!E+5dtKVQ$6GQ98e+b-tBYqZAsXv^Rs(O6U(8J>`xqV1Lm#gY$~4wpEBYF zpZ!fmh z(jBYf_q)c-BPhGWP&XGF{M(F#0-m~DEi$OLr82yU9aw}aV#%yM8TCwG&s~ z5)8+v4}nS!Y?c2mFg7-qFl#0PY3>KKqo$;rjS#W?Q8ea3i4GzeSn9Q;#%=NhN$v2!+?U(kaSJa0MckdpJ*AU9UtdB&2qXWeZg$$f2|)?@GR)}GIpwx zOcJ?G0|M$py%DHc)Y9rx#sYJGbn0vaMgc(wMoA72CNxYMdK$R}d$R7pMJ37Vv4y8x0q|E)j~ugj3REASb< z&=(D=#`;od<{eJFGUQPp-3z>@l#T3RKt&a9RFhm%!%FzYkcPXiDS7IgghtyP^1eZx z@lbyNguezPlIj0dZtqYrg3;@eRwPHog()chc5J1*T#Ek~XgcI3oInd6p7OU$y@3PZ z^3^BiQpsIcnd*|*5OlN=BWNi6O|l?p>zC0F-CTO{$^`ioL15~D?a4s|VTqip8=Pg; zIU5f^%zwhL9H1O(fam}@Cr^@P;@Ihc5>b4b;CdD??k#qM0byoQvibHpB4-@#l8llf zainxw4QZ6bgTu-KA^LO614V{*0ZppJ(8tP+Fm(kG$Z~L&S6Ud0fS6BU{6`)+2Au4S z4rZsY_2Ql4XtO)j?^VKDQ97D#fm_YYI79tv!j8{RFGbDRJEq*4@bSP}W>xr)IC3(r z6|Hven{#c6`WQvdp7e8bjo|M3LuV%uk-D&rxRf&Nif_cIW`;zuC zc@@|Qry2gMmaGIO=8bkq8}w(Z;+F6BJe`q*jjld*k_LWUFfQk)=*erz!8-#h(;=xQ zr7p|VlK<`SHZMKQ0TQJ(!ualza{=^_*9Ig9A_crgcBx_N>0g=;ZD1xycs5@hI6jeQ z&S7Bxn~v~LBXJx%VwWc97?uP~OA6;dn@N1ZZEzl-=*R$Yc<@pXQAiD7)w925D~pE6 zD~{gA;0{qnKUn?&7L_W~x?`MYnLyyKpWi^5%xnj z$O=v&>o?e#I22N1xHmQ2e=l&LJP4if8+tEf;t!SsLk8I^EHos!B;CB)bV+iH3PYw6 z8uwr-W)^7;aPo<^4*!aUwizctxQs!%%t3?{Fb;!Wge#*pizZP?MYl0~UduV+q9!9x z_ay376qij0$s!ID=JumM1M@*cCxmb1Oe2Ylq>Egg0f|fGPTyUM3B+znGAR{&fq09& zgiB68v8gyt`hVL^X2`qtoZeccG{6*@`!AAY-gblW0#>892K(Q}&@(#@iw%SuD7RY| z=?up6dVzVuyN}-B9-#?DOkS zQs!fk_8)Gp7!m-64eQOTk?E&VA-5v0O~gyr(mv@CcRD=X-kab!uual?VoLJA z1(eK5x)I%ybqxsN%lCc8(XfgkPYF9rTvg)TKecESd(Ps=u;-xjXX!ZPoWrbitJ=6m z3uOGP4r3vCFG81G2b665E&=OE-4})CJSOGM9iQlmlqhB_C*|O-2h9OFmqEc2a7Xj` zt#KI5({*dO3V=LV#B)vo_;8^}anDhQsxh!Yhc}Tqq^6OM{Up^Rs1jUr$iw))vhYq7 zM*q?cZ*CKnpT`AjH8}7f3F$)@_EL#F^kkthk#}vQgZ9#`?}^@gMGdae_d4-0BH4Nl zqr(!3+)H3E;{0u?oXnVffnI@y*5^jQZmVtPB2?6XS#o6mbz;h{CJBjXPqrE1Y@{i9)vJT)Oti z=NyQbfx$%~+gd1!OjK37VrzoglaBQ$t%?n2^VNMZwbh#q<|xU+AG4m+D4EWj$Eg-W zZsyPN3;r1~4U@IOPpM(#^&FVkj2F&Ux2+BUnN8+e^~^{gJdZe>lk|`TMN%vK(or^F z`~9%mrWAc9hT#PGl}U4K>E?j$QK%}nfetF)-;DAX6a<$dey8V=CWMqAeYa}NrW@aa zH4T^mTEROE1u=IpBXFi|Sh=)z_%gn;wt zQs>15bp@(fqG>rNhn?+)#glZMDyB=owf1Y@_}0KDI$DQt0&J0yFI8%Pvy>G_! z6E8-*IF(?7!$5__NXytW$9g6F;Wp~EVbowbxK@PWkPeHH zfkKA=#VHT5bQue9j(BaL{e!dPh6%?0`#kdHHq!I~YB8X@Z2KqwA(aR2wA^Vh>*FlU z6ryI7SwD}fN~G9;UY}WMVfKXoGZ|xn#T92X3x#{0RZD0+aZuN#-Wa9^Z6>5$eoa^T z^nxt&h!YMilor%2GrQIfF*V(EhgKj- zF7+9IN0qpd-fDx>QQ&vDNcI5vqT_ADZ?>n4&T9KKgudb2<%mac>Lw>!u#KEsm+41& zHvl!WH*cghJk4KSD{~F8SY$9sMMv{2mHW>#&}lsvk@KyrwS{baq@qp|CISj93qxb+ zw_k--dU$xi<%X2Lhp;FF{g7Hgz1}8Or3U#Vij{x2RvMfT6 zrNA6l#HfjATx?+>K-8A8=({9!XgM-n1|f=`j;TWi$KU@Sl-t;QWFBuNqaUsydJW_j zlM%WtjCT`X*QT$F`EM!&MrWmMX}yZ_0qk^o_c=Am`CGbnn2mYagS%@%g8-1)Q&?}; zYP*-0Uz8+#2Mwc|M1V2d-a*SN;rKn!`B%0@eX#pXAL+Ka-@9oey&JND#^jYqZgj$> z3p|7$6!{JD8Eh4mRH?IVKAYV-uj%g4`LOZn`ld}^&&bSAsRO}=j~#V?OjgLjXWUA3 zWXMw$^R?R(cjI#d%Up6Ay#Ec0W9!S)GP%`!+aSj=+FNiR`FFRqV>YUY8pl+Mr(QRD zBw4Y)d}%%I%al!Ba;s7Ir6^#znt8X^%jlhr4+>WL!&9e9St%|ns3EbnTZfr90mCb=J#-Mn(@Z$wOaGr$KHAG-XUCP z%ZxufK$Idihzn`$RB1#6zPQ2AbB|Q+!29a zkXiCu`wfR9{sI=RWKv&ybfNAK^q!I`9MBj9c8qIwzi&@3i+ctiS{4@%mfrR~0b&?m z91c6Antg0*?;bdw-}h_auHq7%1rE-7z}M*aEM9#AZP9eH=uQ-{FdaNEdNXNp5?U?# zv7v?cs$Y=Wcu%JL<)qD;IKlW5^$+1IE5(X2e$R!!(mu^kV)47t1A4x>`yL%jdLRA~ z{K&QFSpn4r7c--t%Mn$k(D22};L?AIHYBHO(%r{lXcAu<77Tsn!7kR)dt)GzPFl$) z373c;cAnOEUQn^D)0C0Rkr__7n^v9@dg_8o$6@6RHxeeT- zpJZrOGP1t?2`;t%Uei@vYa7b2loDilyTQa-$nXEB-D*RxY7J@)pN!uhAjPV$VQRi) z4Kltvf(%~d+hXCn;2I7N{cT;1Q2rZjucn?{LhtYPI!&r9LXJ&W1tHX116u-*9YP>fXxu3@64ca$|pO8H>#d6J;qp-fbR8^P4) zqP#F0(TqO)Ui(fEeY$G*U>@pViCebQLROR3Qq-tZ=DDuwj!Gsc;pJ7?NMo5&ec%Fr zr98*ys=Z_Yyv6<+@K=RZNBPv%Goi5uEc5plexmgQwkDQz)ptaMNI+*oDRu0nJONeC z5{_<3w>kq?8t41ZVjLZ<7$-Dta2qQuM{mYjpWW3vAK?E#O@p78apf?beN%B*%!sGz`~e$4gX diff --git a/graphics/Horizontal Update.svg b/graphics/Horizontal Update.svg new file mode 100644 index 0000000..27d4795 --- /dev/null +++ b/graphics/Horizontal Update.svg @@ -0,0 +1,327 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Produced by OmniGraffle 7.10.2 + 2019-05-29 10:21:17 +0000 + + + Horizontal Update + + + Layer 1 + + + + *Repo.Update() + + + + + + *Repo.tryUpdate() + + + + + + if updateErr != errUpdateRejected + + + + + + + + + *Repo.tryToRestoreCurrent() + + + + + + + + + + + + + *Repo.history.lock() + + + + + + return errUpdateRejected + + + + + + + + + + + + queue full + + + + + + queue free + + + + + + return updateResponse + + + + + + + + + *Repo.updateRoutine() + + + + + + + + + resChan <- *Repo.updateInProgressChannel: + + + + + + *Repo.update() + + + + + + *Repo.get() + + + + + + *Repo.loadNodesFromJSON() + + + + + + *Repo.loadNodes() + + + + + + + + + for dimension, newNode := range nodes { + *Repo.updateDimension(dimension, newNode) + } + + + + + + *Repo.dimensionUpdateRoutine() + + + + + + *Repo._updateDimension() + + + + + + + + + newNode.WireParents() + + + + + + buildDirectory() + + + + + + wireAliases() + + + + + + dimensionUpdateDoneChan <- err + + + + + + *Repo.history.add(jsonBytes) + + + + + + + + + resultChan <- updateResponse + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + update error + + + + + + + return errUpdateRejected + + + + + + + + + lockfile exists + + + + + + *Repo.updateInProgressChan <- make(chan updateResponse) + + + + + + + + + + *Repo.history.unlock() + + + + + + + + + + *Repo.history. + broadcastUpdate() + + + + + + + + Contentserver Horizontal Scaling: Update Flow + + + + + + Remove lockfile + + + + + + Broadcast update via NATS + + + + + + Create lockfile + + + + + diff --git a/graphics/Horizontal.svg b/graphics/Horizontal.svg new file mode 100644 index 0000000..818bf78 --- /dev/null +++ b/graphics/Horizontal.svg @@ -0,0 +1,391 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + Produced by OmniGraffle 7.10.2 + 2019-05-29 10:21:17 +0000 + + + Horizontal + + + Layer 1 + + + + + + + + + + + + + + + + + + + + contentserver + + + + + + + Web server + + + + + + + TCP Socket server + + + + + + CONTENT.JSON + + + + + + Content Source Webservice + + + + + + Repo + + + + + + RepoNode + + + + + + Dimension + + + + + + RepoNode + + + + + + Dimension + + + + + + + + + + GetContent + + + + + + + GetURIs + + + + + + + GetNodes + + + + + + + Update + + + + + + + GetRepo + + + + + + contentserver-repo-current.json + + + + + + Var Directory (aka History) + + + + + + Logfile (JSON or TXT) + + + + + + + API + + + + + + … + + + + + + … + + + + + + contentserver-repo-[Timestamp].json + + + + + + Log entry + + + + + + Log entry + + + + + + Log entry + + + + + + Log entry + + + + + + + + + + + + + + + + + + + + + + + + + contentserver + + + + + + + Web server + + + + + + + TCP Socket server + + + + + + Repo + + + + + + RepoNode + + + + + + Dimension + + + + + + RepoNode + + + + + + Dimension + + + + + + + GetContent + + + + + + + GetURIs + + + + + + + GetNodes + + + + + + + Update + + + + + + + GetRepo + + + + + + + API + + + + + + … + + + + + + … + + + + + + + + + + + + + + + + + + Logfile (JSON or TXT) + + + + + + Log entry + + + + + + Log entry + + + + + + Log entry + + + + + + Log entry + + + + + + + updateInProgress.lock + + + + + diff --git a/graphics/Overview.svg b/graphics/Overview.svg index 76e7e97..6cd547d 100644 --- a/graphics/Overview.svg +++ b/graphics/Overview.svg @@ -1,6 +1,6 @@ - + @@ -24,9 +24,9 @@ Produced by OmniGraffle 7.10.2 - 2019-05-27 08:17:48 +0000 + 2019-05-29 10:21:17 +0000 - + Overview @@ -35,7 +35,7 @@ - + @@ -148,8 +148,8 @@ - - + + contentserver-repo-current.json @@ -185,20 +185,8 @@ - - - contentserver-repo-[Timestamp].json - - - - - - contentserver-repo-[Timestamp].json - - - - - + + contentserver-repo-[Timestamp].json diff --git a/graphics/Update-Flow.svg b/graphics/Update-Flow.svg index ecbefa5..c5952f2 100644 --- a/graphics/Update-Flow.svg +++ b/graphics/Update-Flow.svg @@ -1,6 +1,6 @@ - + @@ -22,14 +22,19 @@ + + + + + Produced by OmniGraffle 7.10.2 - 2019-05-27 08:17:48 +0000 + 2019-05-29 10:21:17 +0000 - - Update Flow - - + + Update-Flow + + Layer 1 @@ -43,12 +48,6 @@ *Repo.tryUpdate() - - - - *Repo.tryUpdate() - - @@ -58,21 +57,12 @@ - - - failure - - - - success - - @@ -98,26 +88,26 @@ - - - failure + + + update error - + - + - - + + queue full - - + + queue free @@ -216,7 +206,7 @@ - <- dimensionUpdateDoneChan + dimensionUpdateDoneChan <- err @@ -231,7 +221,7 @@ - return updateResponse + resultChan <- updateResponse @@ -258,6 +248,13 @@ + + + Contentserver + : + Update Flow + + From 1c814a450cb153035febd7134fe7e69692e77627 Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Wed, 29 May 2019 13:58:14 +0200 Subject: [PATCH 67/79] testclient: log number of dimensions for GetRepo --- testing/client/client.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/testing/client/client.go b/testing/client/client.go index 99ea991..6a21cbb 100644 --- a/testing/client/client.go +++ b/testing/client/client.go @@ -42,13 +42,13 @@ func main() { if *flagGetRepo { go func(num int) { - log.Println("get repo", num) - _, err := c.GetRepo() + log.Println("GetRepo", num) + resp, err := c.GetRepo() if err != nil { // spew.Dump(resp) log.Fatal("failed to get repo") } - log.Println(num, "get repo done") + log.Println(num, "GetRepo done, got", len(resp), "dimensions") }(i) } From 581e68599c778395d0f6b0afa5b7281ac5f076a3 Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Wed, 29 May 2019 13:59:18 +0200 Subject: [PATCH 68/79] added flag to set the maximum number of history versions, set default to 1 --- contentserver.go | 2 +- repo/history.go | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/contentserver.go b/contentserver.go index 3058761..23c4403 100644 --- a/contentserver.go +++ b/contentserver.go @@ -35,9 +35,9 @@ var ( flagWebserverAddress = flag.String("webserver-address", "", "address to bind web server host:port, when empty no webserver will be spawned") flagWebserverPath = flag.String("webserver-path", "/contentserver", "path to export the webserver on - useful when behind a proxy") flagVarDir = flag.String("var-dir", "/var/lib/contentserver", "where to put my data") - flagDebug = flag.Bool("debug", false, "toggle debug mode") // debugging / profiling + flagDebug = flag.Bool("debug", false, "toggle debug mode") flagFreeOSMem = flag.Int("free-os-mem", 0, "free OS mem every X minutes") flagHeapDump = flag.Int("heap-dump", 0, "dump heap every X minutes") ) diff --git a/repo/history.go b/repo/history.go index fb14d25..a822571 100644 --- a/repo/history.go +++ b/repo/history.go @@ -3,6 +3,7 @@ package repo import ( "bytes" "errors" + "flag" "fmt" "io" "io/ioutil" @@ -16,9 +17,10 @@ import ( const ( historyRepoJSONPrefix = "contentserver-repo-" historyRepoJSONSuffix = ".json" - maxHistoryVersions = 20 ) +var flagMaxHistoryVersions = flag.Int("max-history", 1, "set the maximum number of content backup files") + type history struct { varDir string } @@ -63,7 +65,7 @@ func (h *history) getHistory() (files []string, err error) { } func (h *history) cleanup() error { - files, err := h.getFilesForCleanup(maxHistoryVersions) + files, err := h.getFilesForCleanup(*flagMaxHistoryVersions) if err != nil { return err } From 6381c7c0c2a0e242254d3a08107e7b46a2b52717 Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Wed, 29 May 2019 14:14:15 +0200 Subject: [PATCH 69/79] added metric for failed attempts to persist the content history --- contentserver.go | 2 -- repo/loader.go | 1 + repo/repo.go | 5 ++++- status/metrics.go | 22 ++++++++++++++-------- 4 files changed, 19 insertions(+), 11 deletions(-) diff --git a/contentserver.go b/contentserver.go index 23c4403..b4a54fe 100644 --- a/contentserver.go +++ b/contentserver.go @@ -58,13 +58,11 @@ func main() { }() if *flagFreeOSMem > 0 { - Log.Info("dumping heap every $interval minutes", zap.Int("interval", *flagHeapDump)) Log.Info("freeing OS memory every $interval minutes", zap.Int("interval", *flagFreeOSMem)) go func() { for { select { case <-time.After(time.Duration(*flagFreeOSMem) * time.Minute): - Log.Info("dumping heap every $interval minutes", zap.Int("interval", *flagHeapDump)) log.Info("FreeOSMemory") debug.FreeOSMemory() } diff --git a/repo/loader.go b/repo/loader.go index fa1cc4b..4b4618f 100644 --- a/repo/loader.go +++ b/repo/loader.go @@ -249,6 +249,7 @@ func (repo *Repo) loadJSONBytes() error { historyErr := repo.history.add(repo.jsonBuf.Bytes()) if historyErr != nil { Log.Error("could not add valid json to history", zap.Error(historyErr)) + status.M.HistoryPersistFailedCounter.WithLabelValues(historyErr.Error()).Inc() } else { Log.Info("added valid json to history") } diff --git a/repo/repo.go b/repo/repo.go index 2a7c22f..23405de 100644 --- a/repo/repo.go +++ b/repo/repo.go @@ -10,6 +10,8 @@ import ( "strings" "time" + "github.com/foomo/contentserver/status" + "github.com/mgutz/ansi" "github.com/foomo/contentserver/content" @@ -275,7 +277,8 @@ func (repo *Repo) Update() (updateResponse *responses.Update) { // persist the currently loaded one historyErr := repo.history.add(repo.jsonBuf.Bytes()) if historyErr != nil { - Log.Warn("could not persist current repo in history", zap.Error(historyErr)) + Log.Error("could not persist current repo in history", zap.Error(historyErr)) + status.M.HistoryPersistFailedCounter.WithLabelValues(historyErr.Error()).Inc() } // add some stats for dimension := range repo.Directory { diff --git a/status/metrics.go b/status/metrics.go index 36d69c8..61cc44b 100644 --- a/status/metrics.go +++ b/status/metrics.go @@ -19,14 +19,15 @@ const ( // Metrics is the structure that holds all prometheus metrics type Metrics struct { - ServiceRequestCounter *prometheus.CounterVec // count the number of requests for each service function - ServiceRequestDuration *prometheus.SummaryVec // observe the duration of requests for each service function - UpdatesRejectedCounter *prometheus.CounterVec // count the number of completed updates - UpdatesCompletedCounter *prometheus.CounterVec // count the number of rejected updates - UpdatesFailedCounter *prometheus.CounterVec // count the number of updates that had an error - UpdateDuration *prometheus.SummaryVec // observe the duration of each repo.update() call - ContentRequestCounter *prometheus.CounterVec // count the total number of content requests - NumSocketsGauge *prometheus.GaugeVec // keep track of the total number of open sockets + ServiceRequestCounter *prometheus.CounterVec // count the number of requests for each service function + ServiceRequestDuration *prometheus.SummaryVec // observe the duration of requests for each service function + UpdatesRejectedCounter *prometheus.CounterVec // count the number of completed updates + UpdatesCompletedCounter *prometheus.CounterVec // count the number of rejected updates + UpdatesFailedCounter *prometheus.CounterVec // count the number of updates that had an error + UpdateDuration *prometheus.SummaryVec // observe the duration of each repo.update() call + ContentRequestCounter *prometheus.CounterVec // count the total number of content requests + NumSocketsGauge *prometheus.GaugeVec // keep track of the total number of open sockets + HistoryPersistFailedCounter *prometheus.CounterVec // count the number of failed attempts to persist the content history } // newMetrics can be used to instantiate a metrics instance @@ -72,6 +73,11 @@ func newMetrics() *Metrics { "Total number of currently open socket connections", metricLabelRemote, ), + HistoryPersistFailedCounter: newCounterVec( + "history_persist_failed_count", + "Number of failures to store the content history on the filesystem", + metricLabelError, + ), } } From f5d1117c67dbacd43b56bdb5a3d4f8c65f27da43 Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Wed, 29 May 2019 14:19:07 +0200 Subject: [PATCH 70/79] set explicit go version in Dockerfile, current latest: 1.12.5 --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 5ac6ae8..ec05fc1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,7 @@ ############################## ###### STAGE: BUILD ###### ############################## -FROM golang:latest AS build-env +FROM golang:1.12.5 AS build-env WORKDIR /src From a2b0eabb4113e70fdd34d46416e7da4822a8d4a7 Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Wed, 29 May 2019 15:17:47 +0200 Subject: [PATCH 71/79] logging: set console encoding explicitely if LOG_JSON env var is not set --- logger/log.go | 26 +++++++++++++++----------- 1 file changed, 15 insertions(+), 11 deletions(-) diff --git a/logger/log.go b/logger/log.go index df0d920..e5a7b3f 100644 --- a/logger/log.go +++ b/logger/log.go @@ -17,22 +17,26 @@ var ( // SetupLogging configures the logger func SetupLogging(debug bool, outputPath string) { - var err error + var ( + zc zap.Config + err error + ) + if debug { - zc := zap.NewDevelopmentConfig() - if os.Getenv("LOG_JSON") == "1" { - zc.Encoding = "json" - } + zc = zap.NewDevelopmentConfig() zc.OutputPaths = append(zc.OutputPaths, outputPath) - Log, err = zc.Build() } else { - zc := zap.NewProductionConfig() - if os.Getenv("LOG_JSON") == "1" { - zc.Encoding = "json" - } + zc = zap.NewProductionConfig() zc.OutputPaths = append(zc.OutputPaths, outputPath) - Log, err = zc.Build() } + + if os.Getenv("LOG_JSON") == "1" { + zc.Encoding = "json" + } else { + zc.Encoding = "console" + } + + Log, err = zc.Build() if err != nil { log.Fatalf("can't initialize zap logger: %v", err) } From e6e95db5864c536a06da951fea80950a08858978 Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Wed, 29 May 2019 15:23:42 +0200 Subject: [PATCH 72/79] fixed log rotation and updated tests accordingly --- repo/history.go | 34 ++++++++++++++++++++++++++++++---- repo/history_test.go | 9 +++++---- repo/loader.go | 16 ++++------------ repo/repo.go | 7 ++----- 4 files changed, 41 insertions(+), 25 deletions(-) diff --git a/repo/history.go b/repo/history.go index a822571..81f9422 100644 --- a/repo/history.go +++ b/repo/history.go @@ -12,6 +12,9 @@ import ( "sort" "strings" "time" + + . "github.com/foomo/contentserver/logger" + "go.uber.org/zap" ) const ( @@ -19,7 +22,7 @@ const ( historyRepoJSONSuffix = ".json" ) -var flagMaxHistoryVersions = flag.Int("max-history", 1, "set the maximum number of content backup files") +var flagMaxHistoryVersions = flag.Int("max-history", 2, "set the maximum number of content backup files") type history struct { varDir string @@ -42,8 +45,21 @@ func (h *history) add(jsonBytes []byte) error { return err } + Log.Info("adding content backup", zap.String("file", filename)) + // current filename - return ioutil.WriteFile(h.getCurrentFilename(), jsonBytes, 0644) + err = ioutil.WriteFile(h.getCurrentFilename(), jsonBytes, 0644) + if err != nil { + return err + } + + err = h.cleanup() + if err != nil { + Log.Error("an error occured while cleaning up my history", zap.Error(err)) + return err + } + + return nil } func (h *history) getHistory() (files []string, err error) { @@ -69,7 +85,9 @@ func (h *history) cleanup() error { if err != nil { return err } + for _, f := range files { + Log.Info("removing outdated backup", zap.String("file", f)) err := os.Remove(f) if err != nil { return fmt.Errorf("could not remove file %s : %s", f, err.Error()) @@ -84,8 +102,16 @@ func (h *history) getFilesForCleanup(historyVersions int) (files []string, err e if err != nil { return nil, errors.New("could not generate file cleanup list: " + err.Error()) } - if len(contentFiles) > historyVersions { - for i := historyVersions; i < len(contentFiles); i++ { + + // fmt.Println("contentFiles:") + // for _, f := range contentFiles { + // fmt.Println(f) + // } + + // -1 to remove the current backup file from the number of items + // so that only files with a timestamp are compared + if len(contentFiles)-1 > historyVersions { + for i := historyVersions + 1; i < len(contentFiles); i++ { // ignore current repository file to fall back on if contentFiles[i] == h.getCurrentFilename() { continue diff --git a/repo/history_test.go b/repo/history_test.go index 914fdc5..cbc99b0 100644 --- a/repo/history_test.go +++ b/repo/history_test.go @@ -53,8 +53,10 @@ func TestHistoryCleanup(t *testing.T) { if err != nil { t.Fatal(err) } - if len(files) != maxHistoryVersions { - t.Fatal("history too long", len(files), "instead of", maxHistoryVersions) + + // -1 for ignoring the current content backup file + if len(files)-1 != *flagMaxHistoryVersions { + t.Fatal("history too long", len(files), "instead of", *flagMaxHistoryVersions) } } @@ -80,8 +82,7 @@ func TestGetFilesForCleanup(t *testing.T) { if err != nil { t.Fatal("error not expected") } - assertStringEqual(t, "testdata/order/contentserver-repo-2017-10-22.json", files[0]) - assertStringEqual(t, "testdata/order/contentserver-repo-2017-10-21.json", files[1]) + assertStringEqual(t, "testdata/order/contentserver-repo-2017-10-21.json", files[0]) } func assertStringEqual(t *testing.T, expected, actual string) { diff --git a/repo/loader.go b/repo/loader.go index 4b4618f..8371ab1 100644 --- a/repo/loader.go +++ b/repo/loader.go @@ -7,8 +7,6 @@ import ( "net/http" "time" - "github.com/mgutz/ansi" - "github.com/foomo/contentserver/content" . "github.com/foomo/contentserver/logger" "github.com/foomo/contentserver/status" @@ -53,10 +51,10 @@ func (repo *Repo) updateRoutine() { func (repo *Repo) dimensionUpdateRoutine() { for newDimension := range repo.dimensionUpdateChannel { - Log.Info("update routine received a new dimension", zap.String("dimension", newDimension.Dimension)) + Log.Info("dimensionUpdateRoutine received a new dimension", zap.String("dimension", newDimension.Dimension)) err := repo._updateDimension(newDimension.Dimension, newDimension.Node) - Log.Info("update routine received result") + Log.Info("dimensionUpdateRoutine received result") if err != nil { Log.Debug("update dimension failed", zap.Error(err)) } @@ -184,10 +182,10 @@ func (repo *Repo) get(URL string) (err error) { return fmt.Errorf("Bad HTTP Response: %q", response.Status) } - Log.Info(ansi.Red + "RESETTING BUFFER" + ansi.Reset) + // Log.Info(ansi.Red + "RESETTING BUFFER" + ansi.Reset) repo.jsonBuf.Reset() - Log.Info(ansi.Green + "LOADING DATA INTO BUFFER" + ansi.Reset) + // Log.Info(ansi.Green + "LOADING DATA INTO BUFFER" + ansi.Reset) _, err = io.Copy(&repo.jsonBuf, response.Body) return err } @@ -253,12 +251,6 @@ func (repo *Repo) loadJSONBytes() error { } else { Log.Info("added valid json to history") } - cleanUpErr := repo.history.cleanup() - if cleanUpErr != nil { - Log.Error("an error occured while cleaning up my history", zap.Error(cleanUpErr)) - } else { - Log.Info("cleaned up history") - } } return err } diff --git a/repo/repo.go b/repo/repo.go index 23405de..8326237 100644 --- a/repo/repo.go +++ b/repo/repo.go @@ -6,14 +6,11 @@ import ( "fmt" "io" "os" - "strconv" "strings" "time" "github.com/foomo/contentserver/status" - "github.com/mgutz/ansi" - "github.com/foomo/contentserver/content" . "github.com/foomo/contentserver/logger" "github.com/foomo/contentserver/requests" @@ -247,7 +244,7 @@ func (repo *Repo) Update() (updateResponse *responses.Update) { } Log.Info("Update triggered") - Log.Info(ansi.Yellow + "BUFFER LENGTH BEFORE tryUpdate(): " + strconv.Itoa(len(repo.jsonBuf.Bytes())) + ansi.Reset) + // Log.Info(ansi.Yellow + "BUFFER LENGTH BEFORE tryUpdate(): " + strconv.Itoa(len(repo.jsonBuf.Bytes())) + ansi.Reset) startTime := time.Now().UnixNano() updateRepotime, updateErr := repo.tryUpdate() @@ -260,7 +257,7 @@ func (repo *Repo) Update() (updateResponse *responses.Update) { updateResponse.Stats.NumberOfURIs = -1 // let us try to restore the world from a file Log.Error("could not update repository:", zap.Error(updateErr)) - Log.Info(ansi.Yellow + "BUFFER LENGTH AFTER ERROR: " + strconv.Itoa(len(repo.jsonBuf.Bytes())) + ansi.Reset) + // Log.Info(ansi.Yellow + "BUFFER LENGTH AFTER ERROR: " + strconv.Itoa(len(repo.jsonBuf.Bytes())) + ansi.Reset) updateResponse.ErrorMessage = updateErr.Error() // only try to restore if the update failed during processing From 0661a69601ec3ccf4c8b7d4551cf60dad6171905 Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Mon, 3 Jun 2019 12:16:48 +0200 Subject: [PATCH 73/79] removed version flag --- contentserver.go | 1 - 1 file changed, 1 deletion(-) diff --git a/contentserver.go b/contentserver.go index b4a54fe..3d45927 100644 --- a/contentserver.go +++ b/contentserver.go @@ -30,7 +30,6 @@ const ( ) var ( - flagShowVersionFlag = flag.Bool("version", false, "version info") flagAddress = flag.String("address", "", "address to bind socket server host:port") flagWebserverAddress = flag.String("webserver-address", "", "address to bind web server host:port, when empty no webserver will be spawned") flagWebserverPath = flag.String("webserver-path", "/contentserver", "path to export the webserver on - useful when behind a proxy") From fd4f87da9553b631177007104d6baffea87e336a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Frederik=20L=C3=B6ffert?= <967268+loeffert@users.noreply.github.com> Date: Mon, 3 Jun 2019 19:06:20 +0200 Subject: [PATCH 74/79] update go version (1.12) in travis CI --- .travis.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1a0bbea..9147ceb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,2 @@ language: go -go: - - tip +go: "1.12" From 2ab4c0364c307538260fc2e2d0bc8d1ad742bccb Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Mon, 3 Jun 2019 19:10:42 +0200 Subject: [PATCH 75/79] updated go modules --- go.mod | 6 ++++++ go.sum | 12 ++++++++++++ 2 files changed, 18 insertions(+) diff --git a/go.mod b/go.mod index e57b7a4..3311864 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,14 @@ module github.com/foomo/contentserver require ( + github.com/apex/log v1.1.0 + github.com/davecgh/go-spew v1.1.1 github.com/json-iterator/go v1.1.6 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.1 // indirect + github.com/pkg/errors v0.8.1 // indirect github.com/prometheus/client_golang v0.9.2 + go.uber.org/atomic v1.4.0 // indirect + go.uber.org/multierr v1.1.0 // indirect + go.uber.org/zap v1.10.0 ) diff --git a/go.sum b/go.sum index ae534c7..9ffb36c 100644 --- a/go.sum +++ b/go.sum @@ -1,5 +1,9 @@ +github.com/apex/log v1.1.0 h1:J5rld6WVFi6NxA6m8GJ1LJqu3+GiTFIt3mYv27gdQWI= +github.com/apex/log v1.1.0/go.mod h1:yA770aXIDQrhVOIGurT/pVdfCpSq1GQV/auzMN5fzvY= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs= @@ -10,6 +14,8 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740= github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8= @@ -18,5 +24,11 @@ github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jO github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= From 09192334724f2af9f57f2509ae66c249a9ebed4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Frederik=20L=C3=B6ffert?= <967268+loeffert@users.noreply.github.com> Date: Mon, 3 Jun 2019 19:19:32 +0200 Subject: [PATCH 76/79] enabled gomods for travis CI --- .travis.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.travis.yml b/.travis.yml index 9147ceb..f206d8f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,2 +1,10 @@ language: go go: "1.12" +os: + - linux +dist: trusty +sudo: false +install: true +script: + - env GO111MODULE=on go build + - env GO111MODULE=on go test From d15c524be837d2fcf824fcad994b22fa1afa1f82 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Frederik=20L=C3=B6ffert?= <967268+loeffert@users.noreply.github.com> Date: Mon, 3 Jun 2019 19:24:06 +0200 Subject: [PATCH 77/79] use makefile in travis CI --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index f206d8f..0bdb094 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,5 +6,5 @@ dist: trusty sudo: false install: true script: - - env GO111MODULE=on go build - - env GO111MODULE=on go test + - make dep + - make test From 1911f68de6b8aac80b70661af05187c1ac0e2b27 Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Mon, 3 Jun 2019 19:27:06 +0200 Subject: [PATCH 78/79] set go modules var in make dep --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 500b18a..bf7e312 100644 --- a/Makefile +++ b/Makefile @@ -9,7 +9,7 @@ all: build test tag: echo $(TAG) dep: - go mod download && go mod vendor && go install -i ./vendor/... + env GO111MODULE=on go mod download && env GO111MODULE=on go mod vendor && go install -i ./vendor/... clean: rm -fv bin/contentserve* From 25afa0523ded1fd4eb4bf38a19605f7adc6e1ed7 Mon Sep 17 00:00:00 2001 From: Philipp Mieden Date: Tue, 4 Jun 2019 08:55:07 +0200 Subject: [PATCH 79/79] added delays to tests for travis --- repo/repo_test.go | 35 ++++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/repo/repo_test.go b/repo/repo_test.go index 4c0d96e..43adf05 100644 --- a/repo/repo_test.go +++ b/repo/repo_test.go @@ -3,6 +3,7 @@ package repo import ( "strings" "testing" + "time" . "github.com/foomo/contentserver/logger" _ "github.com/foomo/contentserver/logger" @@ -31,8 +32,9 @@ func TestLoad404(t *testing.T) { mockServer, varDir = mock.GetMockData(t) server = mockServer.URL + "/repo-no-have" r = NewRepo(server, varDir) - response = r.Update() ) + time.Sleep(500 * time.Millisecond) + response := r.Update() if response.Success { t.Fatal("can not get a repo, if the server responds with a 404") } @@ -43,8 +45,9 @@ func TestLoadBrokenRepo(t *testing.T) { mockServer, varDir = mock.GetMockData(t) server = mockServer.URL + "/repo-broken-json.json" r = NewRepo(server, varDir) - response = r.Update() ) + time.Sleep(500 * time.Millisecond) + response := r.Update() if response.Success { t.Fatal("how could we load a broken json") } @@ -104,22 +107,34 @@ func BenchmarkLoadRepo(b *testing.B) { } func TestLoadRepoDuplicateUris(t *testing.T) { - mockServer, varDir := mock.GetMockData(t) - server := mockServer.URL + "/repo-duplicate-uris.json" - r := NewRepo(server, varDir) + + var ( + mockServer, varDir = mock.GetMockData(t) + server = mockServer.URL + "/repo-duplicate-uris.json" + r = NewRepo(server, varDir) + ) + + time.Sleep(500 * time.Millisecond) + response := r.Update() if response.Success { t.Fatal("there are duplicates, this repo update should have failed") } if !strings.Contains(response.ErrorMessage, "update dimension") { - t.Fatal("error message not as expected") + t.Fatal("error message not as expected: " + response.ErrorMessage) } } func TestDimensionHygiene(t *testing.T) { - mockServer, varDir := mock.GetMockData(t) - server := mockServer.URL + "/repo-two-dimensions.json" - r := NewRepo(server, varDir) + + var ( + mockServer, varDir = mock.GetMockData(t) + server = mockServer.URL + "/repo-two-dimensions.json" + r = NewRepo(server, varDir) + ) + + time.Sleep(500 * time.Millisecond) + response := r.Update() if !response.Success { t.Fatal("well those two dimension should be fine") @@ -138,6 +153,7 @@ func getTestRepo(path string, t *testing.T) *Repo { mockServer, varDir := mock.GetMockData(t) server := mockServer.URL + path r := NewRepo(server, varDir) + time.Sleep(500 * time.Millisecond) response := r.Update() if !response.Success { t.Fatal("well those two dimension should be fine") @@ -178,6 +194,7 @@ func TestLinkIds(t *testing.T) { mockServer, varDir := mock.GetMockData(t) server := mockServer.URL + "/repo-link-ok.json" r := NewRepo(server, varDir) + time.Sleep(500 * time.Millisecond) response := r.Update() if !response.Success { t.Fatal("those links should have been fine")