feat: upgrade

.editorconfig

@@ -0,0 +1,12 @@
+root = true
+
+[*]
+charset = utf-8
+end_of_line = lf
+indent_size = 2
+indent_style = tab
+insert_final_newline = true
+trim_trailing_whitespace = true
+
+[*.{yaml,yml,md,mdx}]
+indent_style = space

.gitignore

@@ -1,11 +1,14 @@
-data
-*.log
-*.test
-cprof-*
-var
 .*
-*~
+*.log
+!.github/
+!.husky/
+!.editorconfig
+!.gitignore
+!.golangci.yml
+!.goreleaser.yml
+!.husky.yaml
+!.yamllint.yaml
 /bin/
-/pkg/tmp/
+/coverage.out
-/vendor
+/coverage.html
-!.git*
+/tmp/

.golangci.yml

@@ -0,0 +1,58 @@
+run:
+  timeout: 5m
+
+linters-settings:
+  gocritic:
+    disabled-checks:
+      - ifElseChain
+      - commentFormatting
+
+linters:
+  enable:
+    # Enabled by default linters:
+    - errcheck
+    - gosimple
+    - govet
+    - ineffassign
+    - staticcheck
+
+    # Disabled by default linters:
+    - asciicheck # Simple linter to check that your code does not contain non-ASCII identifiers [fast: true, auto-fix: false]
+    - bidichk # Checks for dangerous unicode character sequences [fast: true, auto-fix: false]
+    #- dupl # Tool for code clone detection [fast: true, auto-fix: false]
+    - forcetypeassert # finds forced type assertions [fast: true, auto-fix: false]
+    #- gochecknoinits # Checks that no init functions are present in Go code [fast: true, auto-fix: false]
+    #- goconst # Finds repeated strings that could be replaced by a constant [fast: true, auto-fix: false]
+    - gocritic # Provides diagnostics that check for bugs, performance and style issues. [fast: false, auto-fix: false]
+    - goimports # In addition to fixing imports, goimports also formats your code in the same style as gofmt. [fast: true, auto-fix: true]
+    #- gomoddirectives # Manage the use of 'replace', 'retract', and 'excludes' directives in go.mod. [fast: true, auto-fix: false]
+    - gomodguard # Allow and block list linter for direct Go module dependencies. This is different from depguard where there are different block types for example version constraints and module recommendations. [fast: true, auto-fix: false]
+    - goprintffuncname # Checks that printf-like functions are named with `f` at the end [fast: true, auto-fix: false]
+    - gosec # (gas): Inspects source code for security problems [fast: false, auto-fix: false]
+    - grouper # An analyzer to analyze expression groups. [fast: true, auto-fix: false]
+    - importas # Enforces consistent import aliases [fast: false, auto-fix: false]
+    #- maintidx # maintidx measures the maintainability index of each function. [fast: true, auto-fix: false]
+    #- makezero # Finds slice declarations with non-zero initial length [fast: false, auto-fix: false]
+    - misspell # Finds commonly misspelled English words in comments [fast: true, auto-fix: true]
+    - nakedret # Finds naked returns in functions greater than a specified function length [fast: true, auto-fix: false]
+    #- nestif # Reports deeply nested if statements [fast: true, auto-fix: false]
+    - nilerr # Finds the code that returns nil even if it checks that the error is not nil. [fast: false, auto-fix: false]
+    - nilnil # Checks that there is no simultaneous return of `nil` error and an invalid value. [fast: false, auto-fix: false]
+    - noctx # noctx finds sending http request without context.Context [fast: false, auto-fix: false]
+    - nolintlint # Reports ill-formed or insufficient nolint directives [fast: true, auto-fix: false]
+    #- nonamedreturns # Reports all named returns [fast: false, auto-fix: false]
+    - nosprintfhostport # Checks for misuse of Sprintf to construct a host with port in a URL. [fast: true, auto-fix: false]
+    - prealloc # Finds slice declarations that could potentially be pre-allocated [fast: true, auto-fix: false]
+    - predeclared # find code that shadows one of Go's predeclared identifiers [fast: true, auto-fix: false]
+    - promlinter # Check Prometheus metrics naming via promlint [fast: true, auto-fix: false]
+    #- revive # Fast, configurable, extensible, flexible, and beautiful linter for Go. Drop-in replacement of golint. [fast: false, auto-fix: false]
+    #- tagliatelle # Checks the struct tags. [fast: true, auto-fix: false]
+    #- testpackage # linter that makes you use a separate _test package [fast: true, auto-fix: false]
+    - thelper # thelper detects golang test helpers without t.Helper() call and checks the consistency of test helpers [fast: false, auto-fix: false]
+    - unconvert # Remove unnecessary type conversions [fast: false, auto-fix: false]
+    - unparam # Reports unused function parameters [fast: false, auto-fix: false]
+    - usestdlibvars # A linter that detect the possibility to use variables/constants from the Go standard library. [fast: true, auto-fix: false]
+    - wastedassign # wastedassign finds wasted assignment statements. [fast: false, auto-fix: false]
+    - whitespace # Tool for detection of leading and trailing whitespace [fast: true, auto-fix: true]
+  disable:
+    - unused

.goreleaser.yml

@@ -0,0 +1,165 @@
+project_name: contentserver
+
+release:
+  github:
+    owner: foomo
+    name: contentserver
+  prerelease: auto
+
+builds:
+  - binary: contentserver
+    goos:
+      - windows
+      - darwin
+      - linux
+    goarch:
+      - amd64
+      - arm64
+    goarm:
+      - 7
+    env:
+      - CGO_ENABLED=0
+    main: ./main.go
+    flags:
+      - -trimpath
+    ldflags: -s -w -X github.com/foomo/contentserver/cmd.version={{.Version}}
+
+archives:
+  - format: tar.gz
+    format_overrides:
+      - goos: windows
+        format: zip
+    files:
+      - LICENSE
+      - README.md
+
+changelog:
+  use: github-native
+
+brews:
+  # Repository to push the tap to.
+  - repository:
+      owner: foomo
+      name: homebrew-tap
+    caveats: "sesamy --help"
+    homepage: "https://github.com/foomo/contentserver"
+    description: "Serves content tree structures very quickly"
+    test: |
+      system "#{bin}/contentserver --version"
+
+docker_manifests:
+  # basic
+  - name_template: 'foomo/contentserver:latest'
+    image_templates:
+      - 'foomo/contentserver:{{ .Tag }}-amd64'
+      - 'foomo/contentserver:{{ .Tag }}-arm64'
+
+  - name_template: 'foomo/contentserver:v{{ .Major }}.{{ .Minor }}'
+    image_templates:
+      - 'foomo/contentserver:v{{ .Major }}.{{ .Minor }}-amd64'
+      - 'foomo/contentserver:v{{ .Major }}.{{ .Minor }}-arm64'
+
+  - name_template: 'foomo/contentserver:{{ .Tag }}'
+    image_templates:
+      - 'foomo/contentserver:{{ .Tag }}-amd64'
+      - 'foomo/contentserver:{{ .Tag }}-arm64'
+
+  # alpine
+  - name_template: 'foomo/contentserver:latest-alpine'
+    image_templates:
+      - 'foomo/contentserver:{{ .Tag }}-alpine-amd64'
+      - 'foomo/contentserver:{{ .Tag }}-alpine-arm64'
+
+  - name_template: 'foomo/contentserver:v{{ .Major }}.{{ .Minor }}-alpine'
+    image_templates:
+      - 'foomo/contentserver:v{{ .Major }}.{{ .Minor }}-alpine-amd64'
+      - 'foomo/contentserver:v{{ .Major }}.{{ .Minor }}-alpine-arm64'
+
+  - name_template: 'foomo/contentserver:{{ .Tag }}-alpine'
+    image_templates:
+      - 'foomo/contentserver:{{ .Tag }}-alpine-amd64'
+      - 'foomo/contentserver:{{ .Tag }}-alpine-arm64'
+dockers:
+  - use: buildx
+    goos: linux
+    goarch: amd64
+    dockerfile: build/buildx.Dockerfile
+    image_templates:
+      - 'foomo/contentserver:latest-amd64'
+      - 'foomo/contentserver:{{ .Tag }}-amd64'
+      - 'foomo/contentserver:v{{ .Major }}.{{ .Minor }}-amd64'
+    build_flag_templates:
+      - '--pull'
+      # https://github.com/opencontainers/image-spec/blob/main/annotations.md#pre-defined-annotation-keys
+      - '--label=org.opencontainers.image.title={{.ProjectName}}'
+      - '--label=org.opencontainers.image.description=Serves content tree structures very quickly'
+      - '--label=org.opencontainers.image.source={{.GitURL}}'
+      - '--label=org.opencontainers.image.url={{.GitURL}}'
+      - '--label=org.opencontainers.image.documentation={{.GitURL}}'
+      - '--label=org.opencontainers.image.created={{.Date}}'
+      - '--label=org.opencontainers.image.revision={{.FullCommit}}'
+      - '--label=org.opencontainers.image.version={{.Version}}'
+      - '--platform=linux/amd64'
+
+  - use: buildx
+    goos: linux
+    goarch: arm64
+    dockerfile: build/buildx.Dockerfile
+    image_templates:
+      - 'foomo/contentserver:latest-arm64'
+      - 'foomo/contentserver:{{ .Tag }}-arm64'
+      - 'foomo/contentserver:v{{ .Major }}.{{ .Minor }}-arm64'
+    build_flag_templates:
+      - '--pull'
+      # https://github.com/opencontainers/image-spec/blob/main/annotations.md#pre-defined-annotation-keys
+      - '--label=org.opencontainers.image.title={{.ProjectName}}'
+      - '--label=org.opencontainers.image.description=Serves content tree structures very quickly'
+      - '--label=org.opencontainers.image.source={{.GitURL}}'
+      - '--label=org.opencontainers.image.url={{.GitURL}}'
+      - '--label=org.opencontainers.image.documentation={{.GitURL}}'
+      - '--label=org.opencontainers.image.created={{.Date}}'
+      - '--label=org.opencontainers.image.revision={{.FullCommit}}'
+      - '--label=org.opencontainers.image.version={{.Version}}'
+      - '--platform=linux/arm64'
+
+  - use: buildx
+    goos: linux
+    goarch: amd64
+    dockerfile: build/buildx-alpine.Dockerfile
+    image_templates:
+      - 'foomo/contentserver:latest-alpine-amd64'
+      - 'foomo/contentserver:{{ .Tag }}-alpine-amd64'
+      - 'foomo/contentserver:v{{ .Major }}.{{ .Minor }}-alpine-amd64'
+    build_flag_templates:
+      - '--pull'
+      # https://github.com/opencontainers/image-spec/blob/main/annotations.md#pre-defined-annotation-keys
+      - '--label=org.opencontainers.image.title={{.ProjectName}}'
+      - '--label=org.opencontainers.image.description=Serves content tree structures very quickly'
+      - '--label=org.opencontainers.image.source={{.GitURL}}'
+      - '--label=org.opencontainers.image.url={{.GitURL}}'
+      - '--label=org.opencontainers.image.documentation={{.GitURL}}'
+      - '--label=org.opencontainers.image.created={{.Date}}'
+      - '--label=org.opencontainers.image.revision={{.FullCommit}}'
+      - '--label=org.opencontainers.image.version={{.Version}}'
+      - '--platform=linux/amd64'
+
+  - use: buildx
+    goos: linux
+    goarch: arm64
+    dockerfile: build/buildx-alpine.Dockerfile
+    image_templates:
+      - 'foomo/contentserver:latest-alpine-arm64'
+      - 'foomo/contentserver:{{ .Tag }}-alpine-arm64'
+      - 'foomo/contentserver:v{{ .Major }}.{{ .Minor }}-alpine-arm64'
+    build_flag_templates:
+      - '--pull'
+      # https://github.com/opencontainers/image-spec/blob/main/annotations.md#pre-defined-annotation-keys
+      - '--label=org.opencontainers.image.title={{.ProjectName}}'
+      - '--label=org.opencontainers.image.description=Serves content tree structures very quickly'
+      - '--label=org.opencontainers.image.source={{.GitURL}}'
+      - '--label=org.opencontainers.image.url={{.GitURL}}'
+      - '--label=org.opencontainers.image.documentation={{.GitURL}}'
+      - '--label=org.opencontainers.image.created={{.Date}}'
+      - '--label=org.opencontainers.image.revision={{.FullCommit}}'
+      - '--label=org.opencontainers.image.version={{.Version}}'
+      - '--platform=linux/arm64'

.travis.yml

@@ -1,9 +0,0 @@
-language: go
-go: "1.18"
-os:
-  - linux
-dist: trusty
-sudo: false
-install: true
-script:
-  - make test

Dockerfile

@@ -1,39 +0,0 @@
-##############################
-###### STAGE: BUILD     ######
-##############################
-FROM golang:1.18 AS build-env
-
-WORKDIR /src
-
-COPY ./go.mod ./go.sum ./
-
-RUN --mount=type=cache,target=/root/.cache/go-build \
-    --mount=type=cache,target=/go/pkg/mod \
-     go mod download -x
-
-COPY ./ ./
-
-RUN GOARCH=amd64 GOOS=linux CGO_ENABLED=0  go build -trimpath -o /contentserver
-
-##############################
-###### STAGE: PACKAGE   ######
-##############################
-FROM alpine
-
-ENV CONTENT_SERVER_ADDR=0.0.0.0:80
-ENV CONTENT_SERVER_VAR_DIR=/var/lib/contentserver
-ENV LOG_JSON=1
-
-RUN apk add --update --no-cache ca-certificates curl bash && rm -rf /var/cache/apk/*
-
-COPY --from=build-env /contentserver /usr/sbin/contentserver
-
-
-VOLUME $CONTENT_SERVER_VAR_DIR
-
-ENTRYPOINT ["/usr/sbin/contentserver"]
-
-CMD ["-address=$CONTENT_SERVER_ADDR", "-var-dir=$CONTENT_SERVER_VAR_DIR"]
-
-EXPOSE 80
-EXPOSE 9200

Makefile

@@ -1,92 +1,86 @@
-SHELL := /bin/bash
+-include .makerc
+.DEFAULT_GOAL:=help
 
-TAG?=latest
+# --- Targets -----------------------------------------------------------------
-IMAGE=foomo/contentserver
 
-# Utils
+## === Tasks ===
 
-all: build test
+.PHONY: doc
-tag:
+## Open go docs
-	echo $(TAG)
+doc:
-clean:
+	@open "http://localhost:6060/pkg/github.com/foomo/contentserver/"
-	rm -fv bin/contentserve*
+	@godoc -http=localhost:6060 -play
-
-# Build
-
-build: clean
-	go build -o bin/contentserver
-
-build-arch: clean
-	GOOS=linux GOARCH=amd64 go build -o bin/contentserver-linux-amd64
-	GOOS=darwin GOARCH=amd64 go build -o bin/contentserver-darwin-amd64
-build-docker: clean build-arch
-	curl https://curl.haxx.se/ca/cacert.pem > .cacert.pem
-	docker build -q . > .image_id
-	docker tag `cat .image_id` $(IMAGE):$(TAG)
-	echo "# tagged container `cat .image_id` as $(IMAGE):$(TAG)"
-	rm -vf .image_id .cacert.pem
-
-build-testclient:
-	go build -o bin/testclient -i github.com/foomo/contentserver/testing/client
-
-build-testserver:
-	go build -o bin/testserver -i github.com/foomo/contentserver/testing/server
-
-package: build
-	pkg/build.sh
-
-# Docker
-
-docker-build:
-	DOCKER_BUILDKIT=1 docker build -t $(IMAGE):$(TAG) --platform linux/amd64 --progress=plain .
-
-docker-push:
-	docker push $(IMAGE):$(TAG)
-
-# Testing / benchmarks
 
+.PHONY: test
+## Run tests
 test:
-	go test -v ./...
+	@go test -coverprofile=coverage.out -race -json ./... | gotestfmt
 
-bench:
+.PHONY: test.update
-	go test -run=none -bench=. ./...
+## Run tests and update snapshots
+test.update:
+	@go test -update -coverprofile=coverage.out -race -json ./... | gotestfmt
 
-run-testserver:
+.PHONY: lint
-	bin/testserver -json-file var/cse-globus-stage-b-with-main-section.json
+## Run linter
+lint:
+	@golangci-lint run
 
-run-contentserver:
+.PHONY: lint.fix
-	contentserver -var-dir var -webserver-address :9191 -address :9999 http://127.0.0.1:1234
+## Fix lint violations
+lint.fix:
+	@golangci-lint run --fix
 
-run-contentserver-freeosmem:
+.PHONY: tidy
-	contentserver -var-dir var -webserver-address :9191 -address :9999 -free-os-mem 1 http://127.0.0.1:1234
+## Run go mod tidy
+tidy:
+	@go mod tidy
 
-run-prometheus:
+.PHONY: outdated
-	prometheus --config.file=prometheus/prometheus.yml
+## Show outdated direct dependencies
+outdated:
+	@go list -u -m -json all | go-mod-outdated -update -direct
 
-clean-var:
+## Install binary
-	rm var/contentserver-repo-2019*
+install:
+	@go build -o ${GOPATH}/bin/contentserver main.go
 
-# Profiling
+## Build binary
+build:
+	@mkdir -p bin
+	@go build -o bin/sesamy main.go
 
-test-cpu-profile:
+## === Utils ===
-	go test -cpuprofile=cprof-client github.com/foomo/contentserver/client
-	go tool pprof --text client.test cprof-client
 
-	go test -cpuprofile=cprof-repo github.com/foomo/contentserver/repo
+.PHONY: help
-	go tool pprof --text repo.test cprof-repo
+## Show help text
-
+help:
-test-gctrace:
+	@awk '{ \
-	GODEBUG=gctrace=1 go test ./...
+		if ($$0 ~ /^.PHONY: [a-zA-Z\-\_0-9]+$$/) { \
-
+			helpCommand = substr($$0, index($$0, ":") + 2); \
-test-malloctrace:
+			if (helpMessage) { \
-	GODEBUG=allocfreetrace=1 go test ./...
+				printf "\033[36m%-23s\033[0m %s\n", \
-
+					helpCommand, helpMessage; \
-trace:
+				helpMessage = ""; \
-	curl http://localhost:6060/debug/pprof/trace?seconds=60 > cs-trace
+			} \
-	go tool trace cs-trace
+		} else if ($$0 ~ /^[a-zA-Z\-\_0-9.]+:/) { \
-
+			helpCommand = substr($$0, 0, index($$0, ":")); \
-pprof-heap-web:
+			if (helpMessage) { \
-	go tool pprof -http=":8081" http://localhost:6060/debug/pprof/heap
+				printf "\033[36m%-23s\033[0m %s\n", \
-
+					helpCommand, helpMessage"\n"; \
-pprof-cpu-web:
+				helpMessage = ""; \
-	go tool pprof -http=":8081" http://localhost:6060/debug/pprof/profile
+			} \
+		} else if ($$0 ~ /^##/) { \
+			if (helpMessage) { \
+				helpMessage = helpMessage"\n                        "substr($$0, 3); \
+			} else { \
+				helpMessage = substr($$0, 3); \
+			} \
+		} else { \
+			if (helpMessage) { \
+				print "\n                        "helpMessage"\n" \
+			} \
+			helpMessage = ""; \
+		} \
+	}' \
+	$(MAKEFILE_LIST)

README.md

@@ -1,25 +1,31 @@
-[![Travis CI](https://travis-ci.org/foomo/contentserver.svg?branch=master)](https://travis-ci.org/foomo/contentserver)
-
 # Content Server
 
-Serves content tree structures very quickly through a json socket api
+[![Build Status](https://github.com/foomo/contentserver/actions/workflows/test.yml/badge.svg?branch=main&event=push)](https://github.com/foomo/contentserver/actions/workflows/test.yml)
+[![Go Report Card](https://goreportcard.com/badge/github.com/foomo/contentserver)](https://goreportcard.com/report/github.com/foomo/contentserver)
+[![godoc](https://godoc.org/github.com/foomo/contentserver?status.svg)](https://godoc.org/github.com/foomo/contentserver)
+[![goreleaser](https://github.com/foomo/contentserver/actions/workflows/release.yml/badge.svg)](https://github.com/foomo/contentserver/actions)
+
+Serves content tree structures very quickly.
 
 ## Concept
 
-A Server written in GoLang to mix and resolve content from different content sources, e.g. CMS, Blog, Shop and many other more. The server provides a simple to use API for non blocking content repository updates, to resolve site content by an URI or to get deep-linking multilingual URIs for a given contentID.
+A Server written in GoLang to mix and resolve content from different content sources, e.g. CMS, Blog, Shop and many
+other more. The server provides a simple to use API for non blocking content repository updates, to resolve site content
+by an URI or to get deep-linking multilingual URIs for a given contentID.
 
-It's up to you how you use it and which data you want to export to the server. Our intention was to write a fast and cache hazzle-free content server to mix different content sources.
+It's up to you how you use it and which data you want to export to the server. Our intention was to write a fast and
+cache hazzle-free content server to mix different content sources.
 
 ### Overview
 
-<img src="graphics/Overview.svg" width="100%" height="500">
+<img src="docs/assets/Overview.svg" width="100%" height="500">
 
 ## Export Data
 
 All you have to do is to provide a tree of content nodes as a JSON encoded RepoNode.
 
 | Attribute     |          Type          |                                                 Usage |
-|---------------|:----------------------:|----------------------------------------------------------------------------------:|
+|---------------|:----------------------:|------------------------------------------------------:|
 | Id            |         string         |                        unique id to identify the node |
 | MimeType      |         string         | mime-type of the node, e.g. text/html, image/png, ... |
 | LinkId        |         string         |                 (symbolic) link/alias to another node |
@@ -35,17 +41,20 @@ All you have to do is to provide a tree of content nodes as a JSON encoded RepoN
 ### Tips
 
 - If you do not want to build a multi-market website define a generic market, e.g. call it *universe*
--	keep it lean and do not export content which should not be accessible at all, e.g. you are working on a super secret fancy new category of your website
+- keep it lean and do not export content which should not be accessible at all, e.g. you are working on a super secret
+  fancy new category of your website
 - Hidden nodes can be resolved by their uri, but are hidden on nodes
 - To avoid duplicate content provide a DestinationId ( = ContentId of the node you want to reference) instead of URIs
 
 ## Request Data
 
-There is a PHP Proxy implementation for foomo in [Foomo.ContentServer](https://github.com/foomo/Foomo.ContentServer). Feel free to use it or to implement your own proxy in the language you love. The API should be easily to implement in every other framework and language, too.
+There is a PHP Proxy implementation for foomo in [Foomo.ContentServer](https://github.com/foomo/Foomo.ContentServer).
+Feel free to use it or to implement your own proxy in the language you love. The API should be easily to implement in
+every other framework and language, too.
 
 ## Update Flowchart
 
-<img src="graphics/Update-Flow.svg" width="100%" height="700">
+<img src="docs/assets/Update-Flow.svg" width="100%" height="700">
 
 ### Usage
 

build/buildx-alpine.Dockerfile

@@ -0,0 +1,30 @@
+# syntax=docker/dockerfile:1.4
+FROM golang:1.21-alpine
+
+# related to https://github.com/golangci/golangci-lint/issues/3107
+ENV GOROOT /usr/local/go
+
+# Allow to download a more recent version of Go.
+# https://go.dev/doc/toolchain
+# GOTOOLCHAIN=auto is shorthand for GOTOOLCHAIN=local+auto
+ENV GOTOOLCHAIN auto
+
+# gcc is required to support cgo;
+# git and mercurial are needed most times for go get`, etc.
+# See https://github.com/docker-library/golang/issues/80
+RUN apk --no-cache add gcc musl-dev git mercurial
+
+# Set all directories as safe
+RUN git config --global --add safe.directory '*'
+
+COPY contentserver /usr/bin/
+ENTRYPOINT ["/usr/bin/contentserver"]
+
+ENV CONTENT_SERVER_ADDRESS=0.0.0.0:8080
+ENV CONTENT_SERVER_VAR_DIR=/var/lib/contentserver
+ENV LOG_JSON=1
+
+EXPOSE 8080
+EXPOSE 9200
+
+CMD ["-address=$CONTENT_SERVER_ADDRESS", "-var-dir=$CONTENT_SERVER_VAR_DIR"]

build/buildx.Dockerfile

@@ -0,0 +1,25 @@
+# syntax=docker/dockerfile:1.4
+FROM golang:1.21
+
+# related to https://github.com/golangci/golangci-lint/issues/3107
+ENV GOROOT /usr/local/go
+
+# Allow to download a more recent version of Go.
+# https://go.dev/doc/toolchain
+# GOTOOLCHAIN=auto is shorthand for GOTOOLCHAIN=local+auto
+ENV GOTOOLCHAIN auto
+
+# Set all directories as safe
+RUN git config --global --add safe.directory '*'
+
+COPY contentserver /usr/bin/
+ENTRYPOINT ["/usr/bin/contentserver"]
+
+ENV CONTENT_SERVER_ADDRESS=0.0.0.0:8080
+ENV CONTENT_SERVER_VAR_DIR=/var/lib/contentserver
+ENV LOG_JSON=1
+
+EXPOSE 8080
+EXPOSE 9200
+
+CMD ["-address=$CONTENT_SERVER_ADDRESS", "-var-dir=$CONTENT_SERVER_VAR_DIR"]

client/client.go

@@ -1,96 +1,58 @@
 package client
 
 import (
+	"context"
 	"errors"
-	"net/http"
-	"net/url"
-	"time"
 
 	"github.com/foomo/contentserver/content"
+	"github.com/foomo/contentserver/pkg/handler"
 	"github.com/foomo/contentserver/requests"
 	"github.com/foomo/contentserver/responses"
-	"github.com/foomo/contentserver/server"
 )
 
-// Client a content server client
-type Client struct {
-	t transport
-}
-
-func NewClient(
-	server string,
-	connectionPoolSize int,
-	waitTimeout time.Duration,
-) (c *Client, err error) {
-	return NewClientWithTransport(NewSocketTransport(server, connectionPoolSize, waitTimeout))
-}
-
-func NewClientWithTransport(
-	transport transport,
-) (c *Client, err error) {
-	c = &Client{
-		t: transport,
-	}
-	return
-}
-
 var (
 	ErrEmptyServerURL   = errors.New("empty contentserver url provided")
 	ErrInvalidServerURL = errors.New("invalid contentserver url provided")
 )
 
-func isValidUrl(str string) bool {
+// Client a content server client
-    u, err := url.Parse(str)
+type Client struct {
-
+	t Transport
-	if u.Scheme != "http" && u.Scheme != "https" {
-		return false
 }
 
-    return err == nil && u.Scheme != "" && u.Host != ""
+// ------------------------------------------------------------------------------------------------
-}
+// ~ Constructor
+// ------------------------------------------------------------------------------------------------
 
-// NewHTTPClient constructs a new client to talk to the contentserver.
+func New(transport Transport) *Client {
-// It returns an error if the provided url is empty or invalid.
+	return &Client{
-func NewHTTPClient(server string) (c *Client, err error) {
-
-	if server == "" {
-		return nil, ErrEmptyServerURL
-	}
-
-	// validate url
-	if !isValidUrl(server) {
-		return nil, ErrInvalidServerURL
-	}
-
-	return NewHTTPClientWithTransport(NewHTTPTransport(server, http.DefaultClient))
-}
-
-func NewHTTPClientWithTransport(transport transport) (c *Client, err error) {
-	c = &Client{
 		t: transport,
 	}
-	return
 }
 
+// ------------------------------------------------------------------------------------------------
+// ~ Public methods
+// ------------------------------------------------------------------------------------------------
+
 // Update tell the server to update itself
-func (c *Client) Update() (*responses.Update, error) {
+func (c *Client) Update(ctx context.Context) (*responses.Update, error) {
 	type serverResponse struct {
 		Reply *responses.Update
 	}
 	resp := serverResponse{}
-	if err := c.t.call(server.HandlerUpdate, &requests.Update{}, &resp); err != nil {
+	if err := c.t.Call(ctx, handler.RouteUpdate, &requests.Update{}, &resp); err != nil {
 		return nil, err
 	}
 	return resp.Reply, nil
 }
 
 // GetContent request site content
-func (c *Client) GetContent(request *requests.Content) (*content.SiteContent, error) {
+func (c *Client) GetContent(ctx context.Context, request *requests.Content) (*content.SiteContent, error) {
 	type serverResponse struct {
 		Reply *content.SiteContent
 	}
 	resp := serverResponse{}
-	if err := c.t.call(server.HandlerGetContent, request, &resp); err != nil {
+	if err := c.t.Call(ctx, handler.RouteGetContent, request, &resp); err != nil {
 		return nil, err
 	}
 
@@ -98,20 +60,20 @@ func (c *Client) GetContent(request *requests.Content) (*content.SiteContent, er
 }
 
 // GetURIs resolve uris for ids in a dimension
-func (c *Client) GetURIs(dimension string, IDs []string) (map[string]string, error) {
+func (c *Client) GetURIs(ctx context.Context, dimension string, ids []string) (map[string]string, error) {
 	type serverResponse struct {
 		Reply map[string]string
 	}
 
 	resp := serverResponse{}
-	if err := c.t.call(server.HandlerGetURIs, &requests.URIs{Dimension: dimension, IDs: IDs}, &resp); err != nil {
+	if err := c.t.Call(ctx, handler.RouteGetURIs, &requests.URIs{Dimension: dimension, IDs: ids}, &resp); err != nil {
 		return nil, err
 	}
 	return resp.Reply, nil
 }
 
 // GetNodes request nodes
-func (c *Client) GetNodes(env *requests.Env, nodes map[string]*requests.Node) (map[string]*content.Node, error) {
+func (c *Client) GetNodes(ctx context.Context, env *requests.Env, nodes map[string]*requests.Node) (map[string]*content.Node, error) {
 	r := &requests.Nodes{
 		Env:   env,
 		Nodes: nodes,
@@ -120,24 +82,24 @@ func (c *Client) GetNodes(env *requests.Env, nodes map[string]*requests.Node) (m
 		Reply map[string]*content.Node
 	}
 	resp := serverResponse{}
-	if err := c.t.call(server.HandlerGetNodes, r, &resp); err != nil {
+	if err := c.t.Call(ctx, handler.RouteGetNodes, r, &resp); err != nil {
 		return nil, err
 	}
 	return resp.Reply, nil
 }
 
 // GetRepo get the whole repo
-func (c *Client) GetRepo() (map[string]*content.RepoNode, error) {
+func (c *Client) GetRepo(ctx context.Context) (map[string]*content.RepoNode, error) {
 	type serverResponse struct {
 		Reply map[string]*content.RepoNode
 	}
 	resp := serverResponse{}
-	if err := c.t.call(server.HandlerGetRepo, &requests.Repo{}, &resp); err != nil {
+	if err := c.t.Call(ctx, handler.RouteGetRepo, &requests.Repo{}, &resp); err != nil {
 		return nil, err
 	}
 	return resp.Reply, nil
 }
 
-func (c *Client) ShutDown() {
+func (c *Client) Close() {
-	c.t.shutdown()
+	c.t.Close()
 }

client/client_test.go

@@ -1,210 +1,56 @@
-package client
+package client_test
 
 import (
-	"net"
+	"context"
-	"strconv"
+	"encoding/json"
 	"sync"
 	"testing"
 	"time"
 
+	"github.com/foomo/contentserver/client"
 	"github.com/foomo/contentserver/content"
-	. "github.com/foomo/contentserver/logger"
+	"github.com/foomo/contentserver/pkg/repo"
-	"github.com/foomo/contentserver/repo/mock"
+	"github.com/foomo/contentserver/pkg/repo/mock"
-	"github.com/foomo/contentserver/requests"
-	"github.com/foomo/contentserver/server"
 	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.uber.org/zap"
 )
 
-const pathContentserver = "/contentserver"
-
-var (
-	testServerSocketAddr    string
-	testServerWebserverAddr string
-)
-
-func init() {
-	SetupLogging(true, "contentserver_client_test.log")
-}
-
-func TestInvalidHTTPClientInit(t *testing.T) {
-	c, err := NewHTTPClient("")
-	assert.Nil(t, c)
-	assert.Error(t, err)
-
-	c, err = NewHTTPClient("bogus")
-	assert.Nil(t, c)
-	assert.Error(t, err)
-
-	c, err = NewHTTPClient("htt:/notaurl")
-	assert.Nil(t, c)
-	assert.Error(t, err)
-
-	c, err = NewHTTPClient("htts://notaurl")
-	assert.Nil(t, c)
-	assert.Error(t, err)
-
-	c, err = NewHTTPClient("/path/segment/only")
-	assert.Nil(t, c)
-	assert.Error(t, err)
-}
-
-func dump(t *testing.T, v interface{}) {
-	jsonBytes, err := json.MarshalIndent(v, "", "	")
-	if err != nil {
-		t.Fatal("could not dump v", v, "err", err)
-		return
-	}
-	t.Log(string(jsonBytes))
-}
-
-func getFreePort() int {
-	addr, err := net.ResolveTCPAddr("tcp", "localhost:0")
-	if err != nil {
-		panic(err)
-	}
-	l, err := net.ListenTCP("tcp", addr)
-	if err != nil {
-		panic(err)
-	}
-	defer l.Close()
-	return l.Addr().(*net.TCPAddr).Port
-}
-
-func getAvailableAddr() string {
-	return "127.0.0.1:" + strconv.Itoa(getFreePort())
-}
-
-func initTestServer(t testing.TB) (socketAddr, webserverAddr string) {
-	socketAddr = getAvailableAddr()
-	webserverAddr = getAvailableAddr()
-	testServer, varDir := mock.GetMockData(t)
-
-	go func() {
-		err := server.RunServerSocketAndWebServer(
-			testServer.URL+"/repo-two-dimensions.json",
-			socketAddr,
-			webserverAddr,
-			pathContentserver,
-			varDir,
-			server.DefaultRepositoryTimeout,
-			false,
-		)
-		if err != nil {
-			t.Fatal("test server crashed: ", err)
-		}
-	}()
-
-	socketClient, errClient := NewClient(socketAddr, 1, time.Duration(time.Millisecond*100))
-	if errClient != nil {
-		panic(errClient)
-	}
-	i := 0
-	for {
-		time.Sleep(time.Millisecond * 100)
-		r, err := socketClient.GetRepo()
-		if err != nil {
-			continue
-		}
-
-		if len(r) == 0 {
-			t.Fatal("received empty JSON from GetRepo")
-		}
-
-		if r["dimension_foo"].Nodes["id-a"].Data["baz"].(float64) == float64(1) {
-			break
-		}
-		if i > 100 {
-			panic("this is taking too long")
-		}
-		i++
-	}
-	return
-}
-
-func getTestClients(t testing.TB) (socketClient *Client, httpClient *Client) {
-	if testServerSocketAddr == "" {
-		socketAddr, webserverAddr := initTestServer(t)
-		testServerSocketAddr = socketAddr
-		testServerWebserverAddr = webserverAddr
-	}
-	socketClient, errClient := NewClient(testServerSocketAddr, 25, time.Duration(time.Millisecond*100))
-	if errClient != nil {
-		t.Log(errClient)
-		t.Fail()
-	}
-	httpClient, errHTTPClient := NewHTTPClient("http://" + testServerWebserverAddr + pathContentserver)
-	if errHTTPClient != nil {
-		t.Log(errHTTPClient)
-		t.Fail()
-	}
-	return
-}
-
-func testWithClients(t *testing.T, testFunc func(c *Client)) {
-	socketClient, httpClient := getTestClients(t)
-	defer socketClient.ShutDown()
-	defer httpClient.ShutDown()
-	testFunc(socketClient)
-	testFunc(httpClient)
-}
-
 func TestUpdate(t *testing.T) {
-	testWithClients(t, func(c *Client) {
+	testWithClients(t, func(c *client.Client) {
-		response, err := c.Update()
+		response, err := c.Update(context.TODO())
-		if err != nil {
+		require.NoError(t, err)
-			t.Fatal("unexpected err", err)
+		require.True(t, response.Success, "update has to return .Sucesss true")
-		}
+		assert.Greater(t, response.Stats.OwnRuntime, 0.0)
-		if !response.Success {
+		assert.Greater(t, response.Stats.RepoRuntime, 0.0)
-			t.Fatal("update has to return .Sucesss true", response)
-		}
-		stats := response.Stats
-		if !(stats.RepoRuntime > float64(0.0)) || !(stats.OwnRuntime > float64(0.0)) {
-			t.Fatal("stats invalid")
-		}
 	})
 }
 
 func TestGetURIs(t *testing.T) {
-	testWithClients(t, func(c *Client) {
+	testWithClients(t, func(c *client.Client) {
-		defer c.ShutDown()
 		request := mock.MakeValidURIsRequest()
-		uriMap, err := c.GetURIs(request.Dimension, request.IDs)
+		uriMap, err := c.GetURIs(context.TODO(), request.Dimension, request.IDs)
-		if err != nil {
+		time.Sleep(100 * time.Millisecond)
-			t.Fatal(err)
+		require.NoError(t, err)
-		}
+		assert.Equal(t, "/a", uriMap[request.IDs[0]])
-		if uriMap[request.IDs[0]] != "/a" {
-			t.Fatal(uriMap)
-		}
 	})
 }
 
 func TestGetRepo(t *testing.T) {
-	testWithClients(t, func(c *Client) {
+	testWithClients(t, func(c *client.Client) {
-
+		r, err := c.GetRepo(context.TODO())
-		time.Sleep(time.Millisecond * 100)
+		require.NoError(t, err)
-
+		if assert.NotEmpty(t, r, "received empty JSON from GetRepo") {
-		r, err := c.GetRepo()
+			assert.Equal(t, 1.0, r["dimension_foo"].Nodes["id-a"].Data["baz"].(float64), "failed to drill deep for data") //nolint:forcetypeassert
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		if len(r) == 0 {
-			t.Fatal("received empty JSON from GetRepo")
-		}
-
-		if r["dimension_foo"].Nodes["id-a"].Data["baz"].(float64) != float64(1) {
-			t.Fatal("failed to drill deep for data")
 		}
 	})
 }
 
 func TestGetNodes(t *testing.T) {
-	testWithClients(t, func(c *Client) {
+	testWithClients(t, func(c *client.Client) {
 		nodesRequest := mock.MakeNodesRequest()
-		nodes, err := c.GetNodes(nodesRequest.Env, nodesRequest.Nodes)
+		nodes, err := c.GetNodes(context.TODO(), nodesRequest.Env, nodesRequest.Nodes)
-		if err != nil {
+		require.NoError(t, err)
-			t.Fatal(err)
-		}
 		testNode, ok := nodes["test"]
 		if !ok {
 			t.Fatal("that should be a node")
@@ -220,9 +66,9 @@ func TestGetNodes(t *testing.T) {
 }
 
 func TestGetContent(t *testing.T) {
-	testWithClients(t, func(c *Client) {
+	testWithClients(t, func(c *client.Client) {
 		request := mock.MakeValidContentRequest()
-		response, err := c.GetContent(request)
+		response, err := c.GetContent(context.TODO(), request)
 		if err != nil {
 			t.Fatal("unexpected err", err)
 		}
@@ -237,17 +83,8 @@ func TestGetContent(t *testing.T) {
 	})
 }
 
-func BenchmarkSocketClientAndServerGetContent(b *testing.B) {
-	socketClient, _ := getTestClients(b)
-	benchmarkServerAndClientGetContent(b, 30, 100, socketClient)
-
-}
-func BenchmarkWebClientAndServerGetContent(b *testing.B) {
-	_, httpClient := getTestClients(b)
-	benchmarkServerAndClientGetContent(b, 30, 100, httpClient)
-}
-
 func benchmarkServerAndClientGetContent(b *testing.B, numGroups, numCalls int, client GetContentClient) {
+	b.Helper()
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		start := time.Now()
@@ -258,30 +95,55 @@ func benchmarkServerAndClientGetContent(b *testing.B, numGroups, numCalls int, c
 	}
 }
 
-type GetContentClient interface {
+func benchmarkClientAndServerGetContent(tb testing.TB, numGroups, numCalls int, client GetContentClient) {
-	GetContent(request *requests.Content) (response *content.SiteContent, err error)
+	tb.Helper()
-}
-
-func benchmarkClientAndServerGetContent(b testing.TB, numGroups, numCalls int, client GetContentClient) {
 	var wg sync.WaitGroup
 	wg.Add(numGroups)
 	for group := 0; group < numGroups; group++ {
-		go func(g int) {
+		go func() {
 			defer wg.Done()
 			request := mock.MakeValidContentRequest()
 			for i := 0; i < numCalls; i++ {
-				response, err := client.GetContent(request)
+				response, err := client.GetContent(context.TODO(), request)
 				if err == nil {
 					if request.URI != response.URI {
-						b.Fatal("uri mismatch")
+						tb.Fatal("uri mismatch")
 					}
 					if response.Status != content.StatusOk {
-						b.Fatal("unexpected status")
+						tb.Fatal("unexpected status")
 					}
 				}
 			}
-		}(group)
+		}()
 	}
 	// Wait for all HTTP fetches to complete.
 	wg.Wait()
 }
+
+func initRepo(tb testing.TB, l *zap.Logger) *repo.Repo {
+	tb.Helper()
+	testRepoServer, varDir := mock.GetMockData(tb)
+	r := repo.New(l,
+		testRepoServer.URL+"/repo-two-dimensions.json",
+		repo.NewHistory(l,
+			repo.HistoryWithVarDir(varDir),
+		),
+	)
+	up := make(chan bool, 1)
+	r.OnStart(func() {
+		up <- true
+	})
+	go r.Start(context.TODO()) //nolint:errcheck
+	<-up
+	return r
+}
+
+func dump(t *testing.T, v interface{}) {
+	t.Helper()
+	jsonBytes, err := json.MarshalIndent(v, "", "	")
+	if err != nil {
+		t.Fatal("could not dump v", v, "err", err)
+		return
+	}
+	t.Log(string(jsonBytes))
+}

client/connectionpool.go

@@ -6,16 +6,16 @@ import (
 )
 
 type connectionPool struct {
-	server string
+	url string
 	// conn           net.Conn
 	chanConnGet    chan chan net.Conn
 	chanConnReturn chan connReturn
 	chanDrainPool  chan int
 }
 
-func newConnectionPool(server string, connectionPoolSize int, waitTimeout time.Duration) *connectionPool {
+func newConnectionPool(url string, connectionPoolSize int, waitTimeout time.Duration) *connectionPool {
 	connPool := &connectionPool{
-		server:         server,
+		url:            url,
 		chanConnGet:    make(chan chan net.Conn),
 		chanConnReturn: make(chan connReturn),
 		chanDrainPool:  make(chan int),
@@ -92,7 +92,7 @@ RunLoop:
 		// refill connection pool
 		for _, poolEntry := range connectionPool {
 			if poolEntry.conn == nil {
-				newConn, errDial := net.Dial("tcp", c.server)
+				newConn, errDial := net.Dial("tcp", c.url)
 				poolEntry.err = errDial
 				poolEntry.conn = newConn
 			}

client/httptransport.go

@@ -2,45 +2,86 @@ package client
 
 import (
 	"bytes"
+	"context"
 	"errors"
-	"io/ioutil"
+	"io"
 	"net/http"
 
-	"github.com/foomo/contentserver/server"
+	"github.com/foomo/contentserver/pkg/handler"
+	"github.com/foomo/contentserver/pkg/utils"
 )
 
-type httpTransport struct {
+type (
-	client   *http.Client
+	HTTPTransport struct {
+		httpClient *http.Client
 		endpoint   string
 	}
+	HTTPTransportOption func(*HTTPTransport)
+)
+
+// ------------------------------------------------------------------------------------------------
+// ~ Constructor
+// ------------------------------------------------------------------------------------------------
 
 // NewHTTPTransport will create a new http transport for the given server and client.
 // Caution: the provided server url is not validated!
-func NewHTTPTransport(server string, client *http.Client) transport {
+func NewHTTPTransport(server string, opts ...HTTPTransportOption) *HTTPTransport {
-	return &httpTransport{
+	inst := &HTTPTransport{
 		endpoint:   server,
-		client:   client,
+		httpClient: http.DefaultClient,
+	}
+
+	for _, opt := range opts {
+		opt(inst)
+	}
+
+	return inst
+}
+
+// NewHTTPClient constructs a new client to talk to the contentserver.
+// It returns an error if the provided url is empty or invalid.
+func NewHTTPClient(url string) (c *Client, err error) {
+	if url == "" {
+		return nil, ErrEmptyServerURL
+	}
+
+	// validate url
+	if !utils.IsValidUrl(url) {
+		return nil, ErrInvalidServerURL
+	}
+
+	return New(NewHTTPTransport(url)), nil
+}
+
+// ------------------------------------------------------------------------------------------------
+// ~ Options
+// ------------------------------------------------------------------------------------------------
+
+func HTTPTransportWithHTTPClient(v *http.Client) HTTPTransportOption {
+	return func(o *HTTPTransport) {
+		o.httpClient = v
 	}
 }
 
-func (ht *httpTransport) shutdown() {
+// ------------------------------------------------------------------------------------------------
-	// nothing to do here
+// ~ Public methods
-}
+// ------------------------------------------------------------------------------------------------
 
-func (ht *httpTransport) call(handler server.Handler, request interface{}, response interface{}) error {
+func (t *HTTPTransport) Call(ctx context.Context, route handler.Route, request interface{}, response interface{}) error {
 	requestBytes, errMarshal := json.Marshal(request)
 	if errMarshal != nil {
 		return errMarshal
 	}
-	req, errNewRequest := http.NewRequest(
+	req, errNewRequest := http.NewRequestWithContext(
+		ctx,
 		http.MethodPost,
-		ht.endpoint+"/"+string(handler),
+		t.endpoint+"/"+string(route),
 		bytes.NewBuffer(requestBytes),
 	)
 	if errNewRequest != nil {
 		return errNewRequest
 	}
-	httpResponse, errDo := ht.client.Do(req)
+	httpResponse, errDo := t.httpClient.Do(req)
 	if errDo != nil {
 		return errDo
 	}
@@ -52,9 +93,13 @@ func (ht *httpTransport) call(handler server.Handler, request interface{}, respo
 	if httpResponse.Body == nil {
 		return errors.New("empty response body")
 	}
-	responseBytes, errRead := ioutil.ReadAll(httpResponse.Body)
+	responseBytes, errRead := io.ReadAll(httpResponse.Body)
 	if errRead != nil {
 		return errRead
 	}
 	return json.Unmarshal(responseBytes, response)
 }
+
+func (t *HTTPTransport) Close() {
+	// nothing to do here
+}

client/httptransport_test.go

@@ -0,0 +1,77 @@
+package client_test
+
+import (
+	"context"
+	"net/http/httptest"
+	"testing"
+
+	"github.com/foomo/contentserver/client"
+	"github.com/foomo/contentserver/content"
+	"github.com/foomo/contentserver/pkg/handler"
+	"github.com/foomo/contentserver/requests"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.uber.org/zap"
+	"go.uber.org/zap/zaptest"
+)
+
+const pathContentserver = "/contentserver"
+
+func TestInvalidHTTPClientInit(t *testing.T) {
+	c, err := client.NewHTTPClient("")
+	assert.Nil(t, c)
+	assert.Error(t, err)
+
+	c, err = client.NewHTTPClient("bogus")
+	assert.Nil(t, c)
+	assert.Error(t, err)
+
+	c, err = client.NewHTTPClient("htt:/notaurl")
+	assert.Nil(t, c)
+	assert.Error(t, err)
+
+	c, err = client.NewHTTPClient("htts://notaurl")
+	assert.Nil(t, c)
+	assert.Error(t, err)
+
+	c, err = client.NewHTTPClient("/path/segment/only")
+	assert.Nil(t, c)
+	assert.Error(t, err)
+}
+
+func BenchmarkWebClientAndServerGetContent(b *testing.B) {
+	l := zaptest.NewLogger(b)
+	server := initHTTPRepoServer(b, l)
+	httpClient := newHTTPClient(b, server)
+	benchmarkServerAndClientGetContent(b, 30, 100, httpClient)
+}
+
+type GetContentClient interface {
+	GetContent(ctx context.Context, request *requests.Content) (response *content.SiteContent, err error)
+}
+
+func newHTTPClient(tb testing.TB, server *httptest.Server) *client.Client {
+	tb.Helper()
+	c, err := client.NewHTTPClient(server.URL + pathContentserver)
+	require.NoError(tb, err)
+	return c
+}
+
+func testWithClients(t *testing.T, testFunc func(c *client.Client)) {
+	t.Helper()
+	l := zaptest.NewLogger(t)
+	httpRepoServer := initHTTPRepoServer(t, l)
+	socketRepoServer := initSocketRepoServer(t, l)
+	httpClient := newHTTPClient(t, httpRepoServer)
+	socketClient := newSocketClient(t, socketRepoServer)
+	defer httpClient.Close()
+	defer socketClient.Close()
+	testFunc(httpClient)
+	testFunc(socketClient)
+}
+
+func initHTTPRepoServer(tb testing.TB, l *zap.Logger) *httptest.Server {
+	tb.Helper()
+	r := initRepo(tb, l)
+	return httptest.NewServer(handler.NewHTTP(l, r))
+}

client/json.go

@@ -0,0 +1,7 @@
+package client
+
+import (
+	jsoniter "github.com/json-iterator/go"
+)
+
+var json = jsoniter.ConfigCompatibleWithStandardLibrary

client/sockettransport.go

@@ -1,6 +1,7 @@
 package client
 
 import (
+	"context"
 	"errors"
 	"fmt"
 	"io"
@@ -8,36 +9,35 @@ import (
 	"strconv"
 	"time"
 
+	"github.com/foomo/contentserver/pkg/handler"
 	"github.com/foomo/contentserver/responses"
-	"github.com/foomo/contentserver/server"
-	jsoniter "github.com/json-iterator/go"
 )
 
-var json = jsoniter.ConfigCompatibleWithStandardLibrary
-
 type connReturn struct {
 	conn net.Conn
 	err  error
 }
 
-type socketTransport struct {
+type SocketTransport struct {
 	connPool *connectionPool
 }
 
-func NewSocketTransport(server string, connectionPoolSize int, waitTimeout time.Duration) transport {
+// ------------------------------------------------------------------------------------------------
-	return &socketTransport{
+// ~ Constructor
-		connPool: newConnectionPool(server, connectionPoolSize, waitTimeout),
+// ------------------------------------------------------------------------------------------------
+
+func NewSocketTransport(url string, connectionPoolSize int, waitTimeout time.Duration) *SocketTransport {
+	return &SocketTransport{
+		connPool: newConnectionPool(url, connectionPoolSize, waitTimeout),
 	}
 }
 
-func (st *socketTransport) shutdown() {
+// ------------------------------------------------------------------------------------------------
-	if st.connPool.chanDrainPool != nil {
+// ~ Public methods
-		st.connPool.chanDrainPool <- 1
+// ------------------------------------------------------------------------------------------------
-	}
-}
 
-func (c *socketTransport) call(handler server.Handler, request interface{}, response interface{}) error {
+func (t *SocketTransport) Call(ctx context.Context, route handler.Route, request interface{}, response interface{}) error {
-	if c.connPool.chanDrainPool == nil {
+	if t.connPool.chanDrainPool == nil {
 		return errors.New("connection pool has been drained, client is dead")
 	}
 	jsonBytes, err := json.Marshal(request)
@@ -45,19 +45,19 @@ func (c *socketTransport) call(handler server.Handler, request interface{}, resp
 		return fmt.Errorf("could not marshal request : %q", err)
 	}
 	netChan := make(chan net.Conn)
-	c.connPool.chanConnGet <- netChan
+	t.connPool.chanConnGet <- netChan
 	conn := <-netChan
 	if conn == nil {
 		return errors.New("could not get a connection")
 	}
 	returnConn := func(err error) {
-		c.connPool.chanConnReturn <- connReturn{
+		t.connPool.chanConnReturn <- connReturn{
 			conn: conn,
 			err:  err,
 		}
 	}
 	// write header result will be like handler:2{}
-	jsonBytes = append([]byte(fmt.Sprintf("%s:%d", handler, len(jsonBytes))), jsonBytes...)
+	jsonBytes = append([]byte(fmt.Sprintf("%s:%d", route, len(jsonBytes))), jsonBytes...)
 
 	// send request
 	var (
@@ -125,3 +125,9 @@ func (c *socketTransport) call(handler server.Handler, request interface{}, resp
 	returnConn(nil)
 	return nil
 }
+
+func (t *SocketTransport) Close() {
+	if t.connPool.chanDrainPool != nil {
+		t.connPool.chanDrainPool <- 1
+	}
+}

client/sockettransport_test.go

@@ -0,0 +1,56 @@
+package client_test
+
+import (
+	"testing"
+	"time"
+
+	"github.com/foomo/contentserver/client"
+	"github.com/foomo/contentserver/pkg/handler"
+	"github.com/stretchr/testify/require"
+	"go.uber.org/zap"
+	"go.uber.org/zap/zaptest"
+	"golang.org/x/net/nettest"
+)
+
+func BenchmarkSocketClientAndServerGetContent(b *testing.B) {
+	l := zaptest.NewLogger(b)
+	server := initSocketRepoServer(b, l)
+	socketClient := newSocketClient(b, server)
+	benchmarkServerAndClientGetContent(b, 30, 100, socketClient)
+}
+
+func newSocketClient(tb testing.TB, address string) *client.Client {
+	tb.Helper()
+	return client.New(client.NewSocketTransport(address, 25, 100*time.Millisecond))
+}
+
+func initSocketRepoServer(tb testing.TB, l *zap.Logger) string {
+	tb.Helper()
+	r := initRepo(tb, l)
+	h := handler.NewSocket(l, r)
+
+	// listen on socket
+	ln, err := nettest.NewLocalListener("tcp")
+
+	require.NoError(tb, err)
+
+	go func() {
+		for {
+			// this blocks until connection or error
+			conn, err := ln.Accept()
+			if err != nil {
+				tb.Error("runSocketServer: could not accept connection", err.Error())
+				continue
+			}
+
+			// a goroutine handles conn so that the loop can accept other connections
+			go func() {
+				l.Debug("accepted connection", zap.String("source", conn.RemoteAddr().String()))
+				h.Serve(conn)
+				require.NoError(tb, conn.Close())
+			}()
+		}
+	}()
+
+	return ln.Addr().String()
+}

client/transport.go

@@ -1,8 +1,12 @@
 package client
 
-import "github.com/foomo/contentserver/server"
+import (
+	"context"
 
-type transport interface {
+	"github.com/foomo/contentserver/pkg/handler"
-	call(handler server.Handler, request interface{}, response interface{}) error
+)
-	shutdown()
+
+type Transport interface {
+	Call(ctx context.Context, route handler.Route, request interface{}, response interface{}) error
+	Close()
 }

cmd/flags.go

@@ -0,0 +1,57 @@
+package cmd
+
+import (
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+func addAddressFlag(cmd *cobra.Command, v *viper.Viper) {
+	cmd.Flags().String("address", "localhost:8080", "Address to bind to (host:port)")
+	_ = v.BindPFlag("address", cmd.Flags().Lookup("address"))
+}
+
+func addBasePathFlag(cmd *cobra.Command, v *viper.Viper) {
+	cmd.Flags().String("base-path", "/contentserver", "Base path to export the webserver on")
+	_ = v.BindPFlag("base_path", cmd.Flags().Lookup("base_path"))
+}
+
+func addPollFlag(cmd *cobra.Command, v *viper.Viper) {
+	cmd.Flags().Bool("poll", false, "If true, the address arg will be used to periodically poll the content url")
+	_ = v.BindPFlag("poll", cmd.Flags().Lookup("poll"))
+}
+
+func addHistoryDirFlag(cmd *cobra.Command, v *viper.Viper) {
+	cmd.Flags().String("history-dir", "/var/lib/contentserver", "Where to put my data")
+	_ = v.BindPFlag("history.dir", cmd.Flags().Lookup("history-dir"))
+	_ = v.BindEnv("history.dir", "CONTENT_SERVER_HISTORY_DIR")
+}
+
+func addHistoryLimitFlag(cmd *cobra.Command, v *viper.Viper) {
+	cmd.Flags().Int("history-limit", 2, "Number of history records to keep")
+	_ = v.BindPFlag("history.limit", cmd.Flags().Lookup("history-limit"))
+}
+
+func addGracefulTimeoutFlag(cmd *cobra.Command, v *viper.Viper) {
+	cmd.Flags().Duration("graceful-timeout", 0, "Timeout duration for graceful shutdown")
+	_ = v.BindPFlag("graceful.timeout", cmd.Flags().Lookup("graceful-timeout"))
+}
+
+func addShutdownTimeoutFlag(cmd *cobra.Command, v *viper.Viper) {
+	cmd.Flags().Duration("shutdown-timeout", 0, "Timeout duration for shutdown")
+	_ = v.BindPFlag("shutdown.timeout", cmd.Flags().Lookup("shutdown-timeout"))
+}
+
+func addOtelEnabledFlag(cmd *cobra.Command, v *viper.Viper) {
+	cmd.Flags().Bool("otel-enabled", false, "Enable otel service")
+	_ = v.BindPFlag("otel.enabled", cmd.Flags().Lookup("otel-enabled"))
+}
+
+func addHealthzEnabledFlag(cmd *cobra.Command, v *viper.Viper) {
+	cmd.Flags().Bool("healthz-enabled", false, "Enable healthz service")
+	_ = v.BindPFlag("healthz.enabled", cmd.Flags().Lookup("healthz-enabled"))
+}
+
+func addPrometheusEnabledFlag(cmd *cobra.Command, v *viper.Viper) {
+	cmd.Flags().Bool("prometheus-enabled", false, "Enable prometheus service")
+	_ = v.BindPFlag("prometheus.enabled", cmd.Flags().Lookup("prometheus-enabled"))
+}

cmd/http.go

@@ -0,0 +1,104 @@
+package cmd
+
+import (
+	"context"
+	"errors"
+
+	"github.com/foomo/contentserver/pkg/handler"
+	"github.com/foomo/contentserver/pkg/repo"
+	"github.com/foomo/keel"
+	"github.com/foomo/keel/healthz"
+	keelhttp "github.com/foomo/keel/net/http"
+	"github.com/foomo/keel/net/http/middleware"
+	"github.com/foomo/keel/service"
+	"github.com/spf13/cobra"
+	"go.uber.org/zap"
+)
+
+func NewHTTPCommand() *cobra.Command {
+	v := NewViper()
+	cmd := &cobra.Command{
+		Use:   "http <url>",
+		Short: "Start http server",
+		Args:  cobra.ExactArgs(1),
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			var comps []string
+			if len(args) == 0 {
+				comps = cobra.AppendActiveHelp(comps, "You must specify the URL for the repository you are adding")
+			} else {
+				comps = cobra.AppendActiveHelp(comps, "This command does not take any more arguments")
+			}
+			return comps, cobra.ShellCompDirectiveNoFileComp
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			svr := keel.NewServer(
+				keel.WithLogger(logger),
+				keel.WithHTTPReadmeService(true),
+				keel.WithHTTPPrometheusService(v.GetBool("prometheus.enabled")),
+				keel.WithHTTPHealthzService(v.GetBool("healthz.enabled")),
+				keel.WithPrometheusMeter(v.GetBool("prometheus.enabled")),
+				keel.WithOTLPGRPCTracer(v.GetBool("otel.enabled")),
+				keel.WithGracefulTimeout(v.GetDuration("graceful.timeout")),
+				keel.WithShutdownTimeout(v.GetDuration("shutdown.timeout")),
+			)
+
+			l := svr.Logger()
+
+			r := repo.New(l,
+				args[0],
+				repo.NewHistory(l,
+					repo.HistoryWithVarDir(v.GetString("history.dir")),
+					repo.HistoryWithMax(v.GetInt("history.limit")),
+				),
+				repo.WithHTTPClient(
+					keelhttp.NewHTTPClient(
+						keelhttp.HTTPClientWithTelemetry(),
+					),
+				),
+				repo.WithPollForUpdates(v.GetBool("poll")),
+			)
+
+			// start initial update and handle error
+			svr.AddReadinessHealthzers(healthz.NewHealthzerFn(func(ctx context.Context) error {
+				if !r.Loaded() {
+					return errors.New("repo not ready yet")
+				}
+				return nil
+			}))
+
+			svr.AddServices(
+				service.NewHTTP(l, "http", v.GetString("address"),
+					handler.NewHTTP(l, r, handler.WithPath(v.GetString("path"))),
+					middleware.Telemetry(),
+					middleware.Logger(),
+					middleware.Recover(),
+				),
+				service.NewGoRoutine(l, "repo", func(ctx context.Context, l *zap.Logger) error {
+					return r.Start(ctx)
+				}),
+			)
+
+			svr.Run()
+			return nil
+		},
+	}
+
+	addAddressFlag(cmd, v)
+	addBasePathFlag(cmd, v)
+	addPollFlag(cmd, v)
+	addHistoryDirFlag(cmd, v)
+	addHistoryLimitFlag(cmd, v)
+	addGracefulTimeoutFlag(cmd, v)
+	addShutdownTimeoutFlag(cmd, v)
+	addOtelEnabledFlag(cmd, v)
+	addHealthzEnabledFlag(cmd, v)
+	addPrometheusEnabledFlag(cmd, v)
+	// cmd.Flags().SetNormalizeFunc(normalizeFlag)
+
+	return cmd
+}
+
+func init() {
+	rootCmd.AddCommand(NewHTTPCommand())
+
+}

cmd/init.go

@@ -0,0 +1,27 @@
+package cmd
+
+import (
+	"strings"
+
+	"github.com/spf13/viper"
+	"go.uber.org/zap"
+)
+
+// initConfig reads in config file and ENV variables if set.
+func initConfig() {
+	viper.EnvKeyReplacer(strings.NewReplacer(".", "_"))
+}
+
+// initConfig reads in config file and ENV variables if set.
+func initLogger() {
+	var err error
+	c := zap.NewProductionConfig()
+	c.Level, err = zap.ParseAtomicLevel(logLevel)
+	if err != nil {
+		panic(err)
+	}
+	logger, err = c.Build()
+	if err != nil {
+		panic(err)
+	}
+}

cmd/root.go

@@ -0,0 +1,49 @@
+package cmd
+
+import (
+	"os"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	"go.uber.org/zap"
+)
+
+var (
+	logger   *zap.Logger
+	logLevel string
+)
+
+// rootCmd represents the base command when called without any subcommands
+var rootCmd = &cobra.Command{
+	Use:   "contentserver",
+	Short: "Serves content tree structures very quickly",
+}
+
+// Execute adds all child commands to the root command and sets flags appropriately.
+// This is called by main.main(). It only needs to happen once to the rootCmd.
+func Execute() {
+	err := rootCmd.Execute()
+	if err != nil {
+		os.Exit(1)
+	}
+}
+
+func init() {
+	cobra.OnInitialize(initConfig, initLogger)
+
+	// Here you will define your flags and configuration settings.
+	// Cobra supports persistent flags, which, if defined here,
+	// will be global for your application.
+
+	rootCmd.PersistentFlags().StringVar(&logLevel, "log-level", "info", "log level")
+
+	// Cobra also supports local flags, which will only run
+	// when this action is called directly.
+	// rootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
+}
+
+func NewViper() *viper.Viper {
+	v := viper.New()
+	v.AutomaticEnv()
+	return v
+}

cmd/socket.go

@@ -0,0 +1,96 @@
+package cmd
+
+import (
+	"context"
+	"net"
+
+	"github.com/foomo/contentserver/pkg/handler"
+	"github.com/foomo/contentserver/pkg/repo"
+	keelhttp "github.com/foomo/keel/net/http"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+	"go.uber.org/zap"
+)
+
+func NewSocketCommand() *cobra.Command {
+	v := viper.New()
+	cmd := &cobra.Command{
+		Use:   "socket <url>",
+		Short: "Start socket server",
+		Args:  cobra.ExactArgs(1),
+		ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+			var comps []string
+			if len(args) == 0 {
+				comps = cobra.AppendActiveHelp(comps, "You must specify the URL for the repository you are adding")
+			} else {
+				comps = cobra.AppendActiveHelp(comps, "This command does not take any more arguments")
+			}
+			return comps, cobra.ShellCompDirectiveNoFileComp
+		},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			l := logger
+
+			r := repo.New(l,
+				args[0],
+				repo.NewHistory(l,
+					repo.HistoryWithVarDir(v.GetString("history.dir")),
+					repo.HistoryWithMax(v.GetInt("history.limit")),
+				),
+				repo.WithHTTPClient(
+					keelhttp.NewHTTPClient(
+						keelhttp.HTTPClientWithTelemetry(),
+					),
+				),
+				repo.WithPollForUpdates(v.GetBool("poll")),
+			)
+
+			// create socket server
+			handle := handler.NewSocket(l, r)
+
+			// listen on socket
+			ln, err := net.Listen("tcp", v.GetString("address"))
+			if err != nil {
+				return err
+			}
+
+			// start repo
+			up := make(chan bool, 1)
+			r.OnStart(func() {
+				up <- true
+			})
+			go r.Start(context.Background()) //nolint:errcheck
+			<-up
+
+			l.Info("started listening", zap.String("address", v.GetString("address")))
+
+			for {
+				// this blocks until connection or error
+				conn, err := ln.Accept()
+				if err != nil {
+					l.Error("runSocketServer: could not accept connection", zap.Error(err))
+					continue
+				}
+
+				// a goroutine handles conn so that the loop can accept other connections
+				go func() {
+					l.Debug("accepted connection", zap.String("source", conn.RemoteAddr().String()))
+					handle.Serve(conn)
+					if err := conn.Close(); err != nil {
+						l.Warn("failed to close connection", zap.Error(err))
+					}
+				}()
+			}
+		},
+	}
+
+	addAddressFlag(cmd, v)
+	addPollFlag(cmd, v)
+	addHistoryDirFlag(cmd, v)
+	addHistoryLimitFlag(cmd, v)
+
+	return cmd
+}
+
+func init() {
+	rootCmd.AddCommand(NewSocketCommand())
+}

cmd/version.go

@@ -0,0 +1,22 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+)
+
+// Populated by goreleaser during build
+var version = "latest"
+
+var versionCmd = &cobra.Command{
+	Use:   "version",
+	Short: "Print version information",
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println(version)
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(versionCmd)
+}

content/reponode.go

@@ -25,23 +25,23 @@ type RepoNode struct {
 // // NewRepoNode constructor
 // func NewRepoNode() *RepoNode {
 // 	return &RepoNode{
-// 		Data:  make(map[string]interface{}, 0), // set initial size to zero explicitely?
+// 		Data:  make(map[string]interface{}, 0), // set initial size to zero explicitly?
 // 		Nodes: make(map[string]*RepoNode, 0),
 // 	}
 // }
 
 // WireParents helper method to reference from child to parent in a tree
 // recursively
-func (node *RepoNode) WireParents() {
+func (n *RepoNode) WireParents() {
-	for _, childNode := range node.Nodes {
+	for _, childNode := range n.Nodes {
-		childNode.parent = node
+		childNode.parent = n
 		childNode.WireParents()
 	}
 }
 
 // InPath is the given node in a path
-func (node *RepoNode) InPath(path []*Item) bool {
+func (n *RepoNode) InPath(path []*Item) bool {
-	myParentID := node.parent.ID
+	myParentID := n.parent.ID
 	for _, pathItem := range path {
 		if pathItem.ID == myParentID {
 			return true
@@ -51,17 +51,16 @@ func (node *RepoNode) InPath(path []*Item) bool {
 }
 
 // GetPath get a path for a repo node
-func (node *RepoNode) GetPath(dataFields []string) []*Item {
+func (n *RepoNode) GetPath(dataFields []string) []*Item {
-
 	var (
-		parentNode = node.parent
+		parentNode = n.parent
 		pathLength = 0
 	)
 	for parentNode != nil {
 		parentNode = parentNode.parent
 		pathLength++
 	}
-	parentNode = node.parent
+	parentNode = n.parent
 
 	var (
 		i    = 0
@@ -81,19 +80,19 @@ func (node *RepoNode) GetPath(dataFields []string) []*Item {
 }
 
 // ToItem convert a repo node to a simple repo item
-func (node *RepoNode) ToItem(dataFields []string) *Item {
+func (n *RepoNode) ToItem(dataFields []string) *Item {
 	item := NewItem()
-	item.ID = node.ID
+	item.ID = n.ID
-	item.Name = node.Name
+	item.Name = n.Name
-	item.MimeType = node.MimeType
+	item.MimeType = n.MimeType
-	item.Hidden = node.Hidden
+	item.Hidden = n.Hidden
-	item.URI = node.URI
+	item.URI = n.URI
-	item.Groups = node.Groups
+	item.Groups = n.Groups
 	if dataFields == nil {
-		item.Data = node.Data
+		item.Data = n.Data
 	} else {
 		for _, dataField := range dataFields {
-			if data, ok := node.Data[dataField]; ok {
+			if data, ok := n.Data[dataField]; ok {
 				item.Data[dataField] = data
 			}
 		}
@@ -102,23 +101,23 @@ func (node *RepoNode) ToItem(dataFields []string) *Item {
 }
 
 // GetParent get the parent node of a node
-func (node *RepoNode) GetParent() *RepoNode {
+func (n *RepoNode) GetParent() *RepoNode {
-	return node.parent
+	return n.parent
 }
 
 // AddNode adds a named child node
-func (node *RepoNode) AddNode(name string, childNode *RepoNode) *RepoNode {
+func (n *RepoNode) AddNode(name string, childNode *RepoNode) *RepoNode {
-	node.Nodes[name] = childNode
+	n.Nodes[name] = childNode
-	return node
+	return n
 }
 
 // IsOneOfTheseMimeTypes is the node one of the given mime types
-func (node *RepoNode) IsOneOfTheseMimeTypes(mimeTypes []string) bool {
+func (n *RepoNode) IsOneOfTheseMimeTypes(mimeTypes []string) bool {
 	if len(mimeTypes) == 0 {
 		return true
 	}
 	for _, mimeType := range mimeTypes {
-		if mimeType == node.MimeType {
+		if mimeType == n.MimeType {
 			return true
 		}
 	}
@@ -127,14 +126,14 @@ func (node *RepoNode) IsOneOfTheseMimeTypes(mimeTypes []string) bool {
 
 // CanBeAccessedByGroups can this node be accessed by at least one the given
 // groups
-func (node *RepoNode) CanBeAccessedByGroups(groups []string) bool {
+func (n *RepoNode) CanBeAccessedByGroups(groups []string) bool {
 	// no groups set on node => anybody can access it
-	if len(node.Groups) == 0 {
+	if len(n.Groups) == 0 {
 		return true
 	}
 
 	for _, group := range groups {
-		for _, myGroup := range node.Groups {
+		for _, myGroup := range n.Groups {
 			if group == myGroup {
 				return true
 			}
@@ -144,10 +143,10 @@ func (node *RepoNode) CanBeAccessedByGroups(groups []string) bool {
 }
 
 // PrintNode essentially a recursive dump
-func (node *RepoNode) PrintNode(id string, level int) {
+func (n *RepoNode) PrintNode(id string, level int) {
 	prefix := strings.Repeat(Indent, level)
-	fmt.Printf("%s %s %s:\n", prefix, id, node.Name)
+	fmt.Printf("%s %s %s:\n", prefix, id, n.Name)
-	for key, childNode := range node.Nodes {
+	for key, childNode := range n.Nodes {
 		childNode.PrintNode(key, level+1)
 	}
 }

content/sitecontent.go

@@ -1,17 +1,5 @@
 package content
 
-// Status status type SiteContent respnses
-type Status int
-
-const (
-	// StatusOk we found content
-	StatusOk Status = 200
-	// StatusForbidden we found content but you mst not access it
-	StatusForbidden = 403
-	// StatusNotFound we did not find content
-	StatusNotFound = 404
-)
-
 // SiteContent resolved content for a site
 type SiteContent struct {
 	Status    Status            `json:"status"`

content/status.go

@@ -0,0 +1,13 @@
+package content
+
+// Status status type SiteContent respnses
+type Status int
+
+const (
+	// StatusOk we found content
+	StatusOk Status = 200
+	// StatusForbidden we found content but you mst not access it
+	StatusForbidden = 403
+	// StatusNotFound we did not find content
+	StatusNotFound = 404
+)

contentserver.go

@@ -1,111 +0,0 @@
-package main
-
-import (
-	"flag"
-	"fmt"
-	_ "net/http/pprof"
-	"os"
-	"runtime/debug"
-	"time"
-
-	. "github.com/foomo/contentserver/logger"
-	"github.com/foomo/contentserver/metrics"
-	"github.com/foomo/contentserver/server"
-	"github.com/foomo/contentserver/status"
-	"go.uber.org/zap"
-)
-
-const (
-	ServiceName                  = "Content Server"
-	DefaultHealthzHandlerAddress = ":8080"
-	DefaultPrometheusListener    = ":9200"
-)
-
-var (
-	flagAddress                   = flag.String("address", "", "address to bind socket server host:port")
-	flagWebserverAddress          = flag.String("webserver-address", "", "address to bind web server host:port, when empty no webserver will be spawned")
-	flagWebserverPath             = flag.String("webserver-path", "/contentserver", "path to export the webserver on - useful when behind a proxy")
-	flagVarDir                    = flag.String("var-dir", "/var/lib/contentserver", "where to put my data")
-	flagPrometheusListener        = flag.String("prometheus-listener", getenv("PROMETHEUS_LISTENER", DefaultPrometheusListener), "address for the prometheus listener")
-	flagRepositoryTimeoutDuration = flag.Duration("repository-timeout-duration", server.DefaultRepositoryTimeout, "timeout duration for the contentserver")
-
-	flagPoll = flag.Bool("poll", false, "if true, the address arg will be used to periodically poll the content url")
-
-	// debugging / profiling
-	flagDebug     = flag.Bool("debug", false, "toggle debug mode")
-	flagFreeOSMem = flag.Int("free-os-mem", 0, "free OS mem every X minutes")
-	flagHeapDump  = flag.Int("heap-dump", 0, "dump heap every X minutes")
-)
-
-func getenv(env, fallback string) string {
-	if value, ok := os.LookupEnv(env); ok {
-		return value
-	}
-	return fallback
-}
-
-func exitUsage(code int) {
-	fmt.Println("Usage:", os.Args[0], "http(s)://your-content-server/path/to/content.json")
-	flag.PrintDefaults()
-	os.Exit(code)
-}
-
-func main() {
-	flag.Parse()
-
-	SetupLogging(*flagDebug, "contentserver.log")
-
-	if *flagFreeOSMem > 0 {
-		Log.Info("freeing OS memory every $interval minutes", zap.Int("interval", *flagFreeOSMem))
-		go func() {
-			for {
-				time.Sleep(time.Duration(*flagFreeOSMem) * time.Minute)
-				Log.Info("FreeOSMemory")
-				debug.FreeOSMemory()
-			}
-		}()
-	}
-
-	if *flagHeapDump > 0 {
-		Log.Info("dumping heap every $interval minutes", zap.Int("interval", *flagHeapDump))
-		go func() {
-			for {
-				time.Sleep(time.Duration(*flagFreeOSMem) * time.Minute)
-				Log.Info("HeapDump")
-				f, err := os.Create("heapdump")
-				if err != nil {
-					panic("failed to create heap dump file")
-				}
-				debug.WriteHeapDump(f.Fd())
-				err = f.Close()
-				if err != nil {
-					panic("failed to create heap dump file")
-				}
-			}
-		}()
-	}
-
-	if len(flag.Args()) == 1 {
-		fmt.Println(*flagAddress, flag.Arg(0))
-
-		// kickoff metric handlers
-		go metrics.RunPrometheusHandler(*flagPrometheusListener)
-		go status.RunHealthzHandlerListener(DefaultHealthzHandlerAddress, ServiceName)
-
-		err := server.RunServerSocketAndWebServer(
-			flag.Arg(0),
-			*flagAddress,
-			*flagWebserverAddress,
-			*flagWebserverPath,
-			*flagVarDir,
-			*flagRepositoryTimeoutDuration,
-			*flagPoll,
-		)
-		if err != nil {
-			fmt.Println("exiting with error", err)
-			os.Exit(1)
-		}
-	} else {
-		exitUsage(1)
-	}
-}

go.mod

@@ -3,6 +3,7 @@ module github.com/foomo/contentserver
 go 1.21
 
 require (
+	github.com/foomo/keel v0.17.4-0.20240315155218-1be83011f59e // #190 (1be8301) update otel
 	github.com/google/uuid v1.6.0
 	github.com/json-iterator/go v1.1.12
 	github.com/pkg/errors v0.9.1
@@ -13,16 +14,117 @@ require (
 )
 
 require (
+	github.com/spf13/cobra v1.8.0
+	github.com/spf13/viper v1.18.2
+	golang.org/x/sync v0.6.0
+)
+
+require (
+	cloud.google.com/go v0.111.0 // indirect
+	cloud.google.com/go/compute v1.23.3 // indirect
+	cloud.google.com/go/compute/metadata v0.2.3 // indirect
+	cloud.google.com/go/firestore v1.14.0 // indirect
+	cloud.google.com/go/longrunning v0.5.4 // indirect
+	github.com/armon/go-metrics v0.4.1 // indirect
+	github.com/avast/retry-go v3.0.0+incompatible // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
-	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/coreos/go-semver v0.3.0 // indirect
+	github.com/coreos/go-systemd/v22 v22.3.2 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
+	github.com/fatih/color v1.14.1 // indirect
+	github.com/fbiville/markdown-table-formatter v0.3.0 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/go-logr/logr v1.4.1 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-ole/go-ole v1.2.6 // indirect
+	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
+	github.com/google/s2a-go v0.1.7 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
+	github.com/googleapis/gax-go/v2 v2.12.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0 // indirect
+	github.com/hashicorp/consul/api v1.25.1 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+	github.com/hashicorp/go-hclog v1.5.0 // indirect
+	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
+	github.com/hashicorp/go-rootcerts v1.0.2 // indirect
+	github.com/hashicorp/golang-lru v0.5.4 // indirect
+	github.com/hashicorp/hcl v1.0.0 // indirect
+	github.com/hashicorp/serf v0.10.1 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/klauspost/compress v1.17.2 // indirect
+	github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect
+	github.com/magiconair/properties v1.8.7 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.17 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
-	github.com/pmezard/go-difflib v1.0.0 // indirect
+	github.com/nats-io/nats.go v1.33.1 // indirect
-	github.com/prometheus/client_model v0.5.0 // indirect
+	github.com/nats-io/nkeys v0.4.7 // indirect
+	github.com/nats-io/nuid v1.0.1 // indirect
+	github.com/pelletier/go-toml/v2 v2.1.0 // indirect
+	github.com/philhofer/fwd v1.1.2 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
+	github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect
+	github.com/prometheus/client_model v0.6.0 // indirect
 	github.com/prometheus/common v0.48.0 // indirect
 	github.com/prometheus/procfs v0.12.0 // indirect
-	golang.org/x/sys v0.16.0 // indirect
+	github.com/sagikazarmark/crypt v0.17.0 // indirect
+	github.com/sagikazarmark/locafero v0.4.0 // indirect
+	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
+	github.com/shirou/gopsutil/v3 v3.24.1 // indirect
+	github.com/shoenig/go-m1cpu v0.1.6 // indirect
+	github.com/sony/gobreaker v0.5.0 // indirect
+	github.com/sourcegraph/conc v0.3.0 // indirect
+	github.com/spf13/afero v1.11.0 // indirect
+	github.com/spf13/cast v1.6.0 // indirect
+	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/subosito/gotenv v1.6.0 // indirect
+	github.com/tinylib/msgp v1.1.9 // indirect
+	github.com/tklauser/go-sysconf v0.3.12 // indirect
+	github.com/tklauser/numcpus v0.6.1 // indirect
+	github.com/yusufpapurcu/wmi v1.2.3 // indirect
+	go.etcd.io/etcd/api/v3 v3.5.10 // indirect
+	go.etcd.io/etcd/client/pkg/v3 v3.5.10 // indirect
+	go.etcd.io/etcd/client/v2 v2.305.10 // indirect
+	go.etcd.io/etcd/client/v3 v3.5.10 // indirect
+	go.opencensus.io v0.24.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/host v0.49.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/runtime v0.49.0 // indirect
+	go.opentelemetry.io/otel v1.24.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.24.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.24.0 // indirect
+	go.opentelemetry.io/otel/exporters/prometheus v0.46.0 // indirect
+	go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.24.0 // indirect
+	go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.24.0 // indirect
+	go.opentelemetry.io/otel/metric v1.24.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.24.0 // indirect
+	go.opentelemetry.io/otel/sdk/metric v1.24.0 // indirect
+	go.opentelemetry.io/otel/trace v1.24.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.1.0 // indirect
+	golang.org/x/crypto v0.21.0 // indirect
+	golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
+	golang.org/x/net v0.21.0 // indirect
+	golang.org/x/oauth2 v0.16.0 // indirect
+	golang.org/x/sys v0.18.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
+	golang.org/x/time v0.5.0 // indirect
+	google.golang.org/api v0.153.0 // indirect
+	google.golang.org/appengine v1.6.8 // indirect
+	google.golang.org/genproto v0.0.0-20231212172506-995d672761c0 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20240102182953-50ed04b92917 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20240102182953-50ed04b92917 // indirect
+	google.golang.org/grpc v1.61.1 // indirect
 	google.golang.org/protobuf v1.32.0 // indirect
+	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -1,56 +1,549 @@
+cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.111.0 h1:YHLKNupSD1KqjDbQ3+LVdQ81h/UJbJyZG203cEfnQgM=
+cloud.google.com/go v0.111.0/go.mod h1:0mibmpKP1TyOOFYQY5izo0LnT+ecvOQ0Sg3OdmMiNRU=
+cloud.google.com/go/compute v1.23.3 h1:6sVlXXBmbd7jNX0Ipq0trII3e4n1/MsADLK6a+aiVlk=
+cloud.google.com/go/compute v1.23.3/go.mod h1:VCgBUoMnIVIR0CscqQiPJLAG25E3ZRZMzcFZeQ+h8CI=
+cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY=
+cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
+cloud.google.com/go/firestore v1.14.0 h1:8aLcKnMPoldYU3YHgu4t2exrKhLQkqaXAGqT0ljrFVw=
+cloud.google.com/go/firestore v1.14.0/go.mod h1:96MVaHLsEhbvkBEdZgfN+AS/GIkco1LRpH9Xp9YZfzQ=
+cloud.google.com/go/longrunning v0.5.4 h1:w8xEcbZodnA2BbW6sVirkkoC+1gP8wS57EUUgGS0GVg=
+cloud.google.com/go/longrunning v0.5.4/go.mod h1:zqNVncI0BOP8ST6XQD1+VcvuShMmq7+xFSzOL++V0dI=
+github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
+github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc=
+github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
+github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
+github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
+github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA=
+github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4=
+github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
+github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
+github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0=
+github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY=
+github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
+github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
+github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
+github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM=
+github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE=
+github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
 github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag=
+github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I=
+github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM=
+github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI=
+github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
+github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
+github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
+github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
+github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU=
+github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk=
+github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w=
+github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg=
+github.com/fbiville/markdown-table-formatter v0.3.0 h1:PIm1UNgJrFs8q1htGTw+wnnNYvwXQMMMIKNZop2SSho=
+github.com/fbiville/markdown-table-formatter v0.3.0/go.mod h1:q89TDtSEVDdTaufgSbfHpNVdPU/bmfvqNkrC5HagmLY=
+github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
+github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
+github.com/foomo/keel v0.17.4-0.20240315155218-1be83011f59e h1:cAUtpoQqHHhuXPMY930rG3zzBx2Dp86/3l9gp04yPHU=
+github.com/foomo/keel v0.17.4-0.20240315155218-1be83011f59e/go.mod h1:+90rU3I9pErPp3n28ZaSB708n+nfPEf5cVP1lqn5Sf8=
+github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
+github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
+github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA=
+github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
+github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
+github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
+github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ=
+github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY=
+github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0=
+github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
+github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
+github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
+github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
+github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY=
+github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
+github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE=
+github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
+github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
+github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
+github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
+github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
+github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
+github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
+github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
+github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ=
+github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4=
+github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA=
+github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
+github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o=
+github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs=
+github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0=
+github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas=
+github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0 h1:Wqo399gCIufwto+VfwCSvsnfGpF/w5E9CNxSwbpD6No=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0/go.mod h1:qmOFXW2epJhM0qSnUUYpldc7gVz2KMQwJ/QYCDIa7XU=
+github.com/hashicorp/consul/api v1.25.1 h1:CqrdhYzc8XZuPnhIYZWH45toM0LB9ZeYr/gvpLVI3PE=
+github.com/hashicorp/consul/api v1.25.1/go.mod h1:iiLVwR/htV7mas/sy0O+XSuEnrdBUUydemjxcUrAt4g=
+github.com/hashicorp/consul/sdk v0.14.1 h1:ZiwE2bKb+zro68sWzZ1SgHF3kRMBZ94TwOCFRF4ylPs=
+github.com/hashicorp/consul/sdk v0.14.1/go.mod h1:vFt03juSzocLRFo59NkeQHHmQa6+g7oU0pfzdI1mUhg=
+github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
+github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
+github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ=
+github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48=
+github.com/hashicorp/go-hclog v1.5.0 h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+13c=
+github.com/hashicorp/go-hclog v1.5.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M=
+github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
+github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc=
+github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
+github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
+github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI=
+github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
+github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
+github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA=
+github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
+github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
+github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs=
+github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc=
+github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8=
+github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
+github.com/hashicorp/go-sockaddr v1.0.2 h1:ztczhD1jLxIRjVejw8gFomI1BQZOe2WoVOu0SyteCQc=
+github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A=
+github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
+github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8=
+github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/hashicorp/go-version v1.2.1 h1:zEfKbn2+PDgroKdiOzqiE8rsmLqU2uwi5PB5pBJ3TkI=
+github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
+github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
+github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc=
+github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
+github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
+github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
+github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
+github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc=
+github.com/hashicorp/memberlist v0.5.0 h1:EtYPN8DpAURiapus508I4n9CzHs2W+8NZGbmmR/prTM=
+github.com/hashicorp/memberlist v0.5.0/go.mod h1:yvyXLpo0QaGE59Y7hDTsTzDD25JYBZ4mHgHUZ8lrOI0=
+github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY=
+github.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4=
+github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
+github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
+github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
+github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
 github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM=
 github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo=
+github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
+github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
+github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
+github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4=
+github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
+github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
+github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
 github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
 github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
+github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4=
+github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I=
+github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY=
+github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0=
+github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
+github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE=
+github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
+github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc=
+github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4=
+github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
+github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
+github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
+github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE=
+github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU=
+github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94=
+github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
+github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
+github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso=
+github.com/miekg/dns v1.1.41 h1:WMszZWJG0XmzbK9FEmzH2TVcqYzFesusSIB41b8KHxY=
+github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI=
+github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI=
+github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
+github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
+github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
+github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
+github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
+github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
+github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
 github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M=
 github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk=
+github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
+github.com/nats-io/nats.go v1.33.1 h1:8TxLZZ/seeEfR97qV0/Bl939tpDnt2Z2fK3HkPypj70=
+github.com/nats-io/nats.go v1.33.1/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8=
+github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI=
+github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc=
+github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw=
+github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
+github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
+github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY=
+github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
+github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4=
+github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
+github.com/philhofer/fwd v1.1.2 h1:bnDivRJ1EWPjUIRXV5KfORO897HTbpFAQddBdE8t7Gw=
+github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0=
+github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
+github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
+github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s=
+github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw=
+github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE=
+github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
+github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
+github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU=
 github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU=
 github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k=
-github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw=
+github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
-github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI=
+github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
+github.com/prometheus/client_model v0.6.0 h1:k1v3CzpSRUTrKMppY35TLwPvxHqBu0bYgxZzqGIgaos=
+github.com/prometheus/client_model v0.6.0/go.mod h1:NTQHnmxFpouOD0DpvP4XujX3CdOAGQPoaGhyTchlyt8=
+github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
+github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4=
 github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE=
 github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc=
+github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
+github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
+github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
 github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo=
 github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo=
 github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
 github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
+github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
+github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
+github.com/sagikazarmark/crypt v0.17.0 h1:ZA/7pXyjkHoK4bW4mIdnCLvL8hd+Nrbiw7Dqk7D4qUk=
+github.com/sagikazarmark/crypt v0.17.0/go.mod h1:SMtHTvdmsZMuY/bpZoqokSoChIrcJ/epOxZN58PbZDg=
+github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ=
+github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4=
+github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE=
+github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ=
+github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I=
+github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
+github.com/shirou/gopsutil/v3 v3.24.1 h1:R3t6ondCEvmARp3wxODhXMTLC/klMa87h2PHUw5m7QI=
+github.com/shirou/gopsutil/v3 v3.24.1/go.mod h1:UU7a2MSBQa+kW1uuDq8DeEBS8kmrnQwsv2b5O513rwU=
+github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM=
+github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ=
+github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU=
+github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k=
+github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
+github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
+github.com/sony/gobreaker v0.5.0 h1:dRCvqm0P490vZPmy7ppEk2qCnCieBooFJ+YoXGYB+yg=
+github.com/sony/gobreaker v0.5.0/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY=
+github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
+github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
+github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8=
+github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY=
+github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0=
+github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=
+github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
+github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
+github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
+github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/viper v1.18.2 h1:LUXCnvUvSM6FXAsj6nnfc8Q2tp1dIgUfY9Kc8GsSOiQ=
+github.com/spf13/viper v1.18.2/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMVB+yk=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
+github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
+github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
+github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
+github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
+github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
+github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8=
+github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU=
+github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4=
+github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU=
+github.com/tinylib/msgp v1.1.9 h1:SHf3yoO2sGA0veCJeCBYLHuttAVFHGm2RHgNodW7wQU=
+github.com/tinylib/msgp v1.1.9/go.mod h1:BCXGB54lDD8qUEPmiG0cQQUANC4IUQyB2ItS2UDlO/k=
+github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU=
+github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI=
+github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk=
+github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY=
+github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM=
+github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
+github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw=
+github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
+go.etcd.io/etcd/api/v3 v3.5.10 h1:szRajuUUbLyppkhs9K6BRtjY37l66XQQmw7oZRANE4k=
+go.etcd.io/etcd/api/v3 v3.5.10/go.mod h1:TidfmT4Uycad3NM/o25fG3J07odo4GBB9hoxaodFCtI=
+go.etcd.io/etcd/client/pkg/v3 v3.5.10 h1:kfYIdQftBnbAq8pUWFXfpuuxFSKzlmM5cSn76JByiT0=
+go.etcd.io/etcd/client/pkg/v3 v3.5.10/go.mod h1:DYivfIviIuQ8+/lCq4vcxuseg2P2XbHygkKwFo9fc8U=
+go.etcd.io/etcd/client/v2 v2.305.10 h1:MrmRktzv/XF8CvtQt+P6wLUlURaNpSDJHFZhe//2QE4=
+go.etcd.io/etcd/client/v2 v2.305.10/go.mod h1:m3CKZi69HzilhVqtPDcjhSGp+kA1OmbNn0qamH80xjA=
+go.etcd.io/etcd/client/v3 v3.5.10 h1:W9TXNZ+oB3MCd/8UjxHTWK5J9Nquw9fQBLJd5ne5/Ao=
+go.etcd.io/etcd/client/v3 v3.5.10/go.mod h1:RVeBnDz2PUEZqTpgqwAtUd8nAPf5kjyFyND7P1VkOKc=
+go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0=
+go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo=
+go.opentelemetry.io/contrib/instrumentation/host v0.49.0 h1:PHK4Cnis16iENFfqnzvuak5vfRl5L0UaTG2Z03vr3iI=
+go.opentelemetry.io/contrib/instrumentation/host v0.49.0/go.mod h1:0XQuDAhohvWG6+cdmjX6aFbC4mGMjYf1xILFh5OUcEg=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw=
+go.opentelemetry.io/contrib/instrumentation/runtime v0.49.0 h1:dg9y+7ArpumB6zwImJv47RHfdgOGQ1EMkzP5vLkEnTU=
+go.opentelemetry.io/contrib/instrumentation/runtime v0.49.0/go.mod h1:Ul4MtXqu/hJBM+v7a6dCF0nHwckPMLpIpLeCi4+zfdw=
+go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo=
+go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0 h1:t6wl9SPayj+c7lEIFgm4ooDBZVb01IhLB4InpomhRw8=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0/go.mod h1:iSDOcsnSA5INXzZtwaBPrKp/lWu/V14Dd+llD0oI2EA=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.24.0 h1:Mw5xcxMwlqoJd97vwPxA8isEaIoxsta9/Q51+TTJLGE=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.24.0/go.mod h1:CQNu9bj7o7mC6U7+CA/schKEYakYXWr79ucDHTMGhCM=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.24.0 h1:Xw8U6u2f8DK2XAkGRFV7BBLENgnTGX9i4rQRxJf+/vs=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.24.0/go.mod h1:6KW1Fm6R/s6Z3PGXwSJN2K4eT6wQB3vXX6CVnYX9NmM=
+go.opentelemetry.io/otel/exporters/prometheus v0.46.0 h1:I8WIFXR351FoLJYuloU4EgXbtNX2URfU/85pUPheIEQ=
+go.opentelemetry.io/otel/exporters/prometheus v0.46.0/go.mod h1:ztwVUHe5DTR/1v7PeuGRnU5Bbd4QKYwApWmuutKsJSs=
+go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.24.0 h1:JYE2HM7pZbOt5Jhk8ndWZTUWYOVift2cHjXVMkPdmdc=
+go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.24.0/go.mod h1:yMb/8c6hVsnma0RpsBMNo0fEiQKeclawtgaIaOp2MLY=
+go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.24.0 h1:s0PHtIkN+3xrbDOpt2M8OTG92cWqUESvzh2MxiR5xY8=
+go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.24.0/go.mod h1:hZlFbDbRt++MMPCCfSJfmhkGIWnX1h3XjkfxZUjLrIA=
+go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI=
+go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco=
+go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw=
+go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg=
+go.opentelemetry.io/otel/sdk/metric v1.24.0 h1:yyMQrPzF+k88/DbH7o4FMAs80puqd+9osbiBrJrz/w8=
+go.opentelemetry.io/otel/sdk/metric v1.24.0/go.mod h1:I6Y5FjH6rvEnTTAYQz3Mmv2kl6Ek5IIrmwTLqMrrOE0=
+go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI=
+go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU=
+go.opentelemetry.io/proto/otlp v1.1.0 h1:2Di21piLrCqJ3U3eXGCTPHE9R8Nh+0uglSnOyxikMeI=
+go.opentelemetry.io/proto/otlp v1.1.0/go.mod h1:GpBHCBWiqvVLDqmHZsoMM3C5ySeKTC7ej/RNTae6MdY=
 go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto=
 go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE=
 go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0=
 go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y=
 go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8=
 go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E=
-golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
+golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
+golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
+golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY=
+golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA=
+golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
+golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
+golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g=
+golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k=
+golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
+golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
+golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
+golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
+golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
+golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
+golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
+golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8=
+golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
+golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
+golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
+golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ=
+golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o=
+golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ=
+golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
+golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4=
+golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
+golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
+golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
+golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
+golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
+golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
+golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
+golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
+golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
+golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
+golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
+golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
+golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
+golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE=
+golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA=
+golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
+golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+google.golang.org/api v0.153.0 h1:N1AwGhielyKFaUqH07/ZSIQR3uNPcV7NVw0vj+j4iR4=
+google.golang.org/api v0.153.0/go.mod h1:3qNJX5eOmhiWYc67jRA/3GsDw97UFb5ivv7Y2PrriAY=
+google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
+google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM=
+google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds=
+google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
+google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/genproto v0.0.0-20231212172506-995d672761c0 h1:YJ5pD9rF8o9Qtta0Cmy9rdBwkSjrTCT6XTiUQVOtIos=
+google.golang.org/genproto v0.0.0-20231212172506-995d672761c0/go.mod h1:l/k7rMz0vFTBPy+tFSGvXEd3z+BcoG1k7EHbqm+YBsY=
+google.golang.org/genproto/googleapis/api v0.0.0-20240102182953-50ed04b92917 h1:rcS6EyEaoCO52hQDupoSfrxI3R6C2Tq741is7X8OvnM=
+google.golang.org/genproto/googleapis/api v0.0.0-20240102182953-50ed04b92917/go.mod h1:CmlNWB9lSezaYELKS5Ym1r44VrrbPUa7JTvw+6MbpJ0=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20240102182953-50ed04b92917 h1:6G8oQ016D88m1xAKljMlBOOGWDZkes4kMhgGFlf8WcQ=
+google.golang.org/genproto/googleapis/rpc v0.0.0-20240102182953-50ed04b92917/go.mod h1:xtjpI3tXFPP051KaWnhvxkiubL/6dJ18vLVf7q2pTOU=
+google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
+google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
+google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc=
+google.golang.org/grpc v1.61.1 h1:kLAiWrZs7YeDM6MumDe7m3y4aM6wacLzM1Y/wiLP9XY=
+google.golang.org/grpc v1.61.1/go.mod h1:VUbo7IFqmF1QtCAstipjG0GIoq49KvMe9+h1jFLBNJs=
+google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
+google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
+google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
+google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
+google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
+google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
+google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I=
 google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
+gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=
+gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA=
+gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
+gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY=
+gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
+honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

logger/log.go

@@ -1,43 +0,0 @@
-package logger
-
-import (
-	"log"
-	"os"
-
-	"go.uber.org/zap"
-)
-
-var (
-	// Log is the logger instance exposed by this package
-	// call Setup() prior to using it
-	// want JSON output? Set LOG_JSON env var to 1!
-	Log *zap.Logger
-)
-
-// SetupLogging configures the logger
-func SetupLogging(debug bool, outputPath string) {
-
-	var (
-		zc  zap.Config
-		err error
-	)
-
-	if debug {
-		zc = zap.NewDevelopmentConfig()
-		zc.OutputPaths = append(zc.OutputPaths, outputPath)
-	} else {
-		zc = zap.NewProductionConfig()
-		zc.OutputPaths = append(zc.OutputPaths, outputPath)
-	}
-
-	if os.Getenv("LOG_JSON") == "1" {
-		zc.Encoding = "json"
-	} else {
-		zc.Encoding = "console"
-	}
-
-	Log, err = zc.Build()
-	if err != nil {
-		log.Fatalf("can't initialize zap logger: %v", err)
-	}
-}

main.go

@@ -0,0 +1,9 @@
+package main
+
+import (
+	"github.com/foomo/contentserver/cmd"
+)
+
+func main() {
+	cmd.Execute()
+}

metrics/prometheus.go

@@ -1,21 +0,0 @@
-package metrics
-
-import (
-	"net/http"
-
-	. "github.com/foomo/contentserver/logger"
-	"github.com/prometheus/client_golang/prometheus/promhttp"
-	"go.uber.org/zap"
-)
-
-const metricsRoute = "/metrics"
-
-func RunPrometheusHandler(listener string) {
-	Log.Info("starting prometheus handler on",
-		zap.String("address", listener),
-		zap.String("route", metricsRoute),
-	)
-	Log.Error("prometheus listener failed",
-		zap.Error(http.ListenAndServe(listener, promhttp.Handler())),
-	)
-}

pkg/README.md

@@ -1,44 +0,0 @@
-Packaging & Deployment
-----------------------
-
-In order to build packages and upload to Package Cloud, please install the following requirements and run the make task.
-
-[Package Cloud Command Line Client](https://packagecloud.io/docs#cli_install)
-
-```
-$ gem install package_cloud
-```
-
-[FPM](https://github.com/jordansissel/fpm)
-
-```
-$ gem install fpm
-```
-
-Building package
-
-```
-$ make package
-```
-
-*NOTE: you will be prompted for Package Cloud credentials.*
-
-Testing
--------
-
-```
-$ git clone https://github.com/foomo/contentserver.git
-$ cd contentserver
-$ make test
-```
-
-Contributing
-------------
-
-In lieu of a formal styleguide, take care to maintain the existing coding style. Add unit tests and examples for any new or changed functionality.
-
-1.	Fork it
-2.	Create your feature branch (`git checkout -b my-new-feature`\)
-3.	Commit your changes (`git commit -am 'Add some feature'`\)
-4.	Push to the branch (`git push origin my-new-feature`\)
-5.	Create new Pull Request

pkg/build.sh

@@ -1,61 +0,0 @@
-#!/bin/bash
-
-USER="foomo"
-NAME="content-server"
-URL="http://www.foomo.org"
-DESCRIPTION="Serves content tree structures very quickly through a json socket api."
-LICENSE="LGPL-3.0"
-
-# get version
-VERSION=`bin/content-server --version | sed 's/content-server //'`
-
-# create temp dir
-TEMP=`pwd`/pkg/tmp
-mkdir -p $TEMP
-
-package()
-{
-	OS=$1
-	ARCH=$2
-	TYPE=$3
-	TARGET=$4
-
-	# copy license file
-	cp LICENSE $LICENSE
-
-	# define source dir
-	SOURCE=`pwd`/pkg/${TYPE}
-
-	# create build folder
-	BUILD=${TEMP}/${NAME}-${VERSION}
-	#rsync -rv --exclude **/.git* --exclude /*.sh $SOURCE/ $BUILD/
-
-	# build binary
-	GOOS=$OS GOARCH=$ARCH go build -o $BUILD/usr/local/bin/${NAME}
-
-	# create package
-	fpm -s dir \
-		-t $TYPE \
-		--name $NAME \
-		--maintainer $USER \
-		--version $VERSION \
-		--license $LICENSE \
-		--description "${DESCRIPTION}" \
-		--architecture $ARCH \
-		--package $TEMP \
-		--url "${URL}" \
-		-C $BUILD \
-		.
-
-	# push
-	package_cloud push $TARGET $TEMP/${NAME}_${VERSION}_${ARCH}.${TYPE}
-
-	# cleanup
-	rm -rf $TEMP
-	rm $LICENSE
-}
-
-package linux amd64 deb foomo/content-server/ubuntu/precise
-package linux amd64 deb foomo/content-server/ubuntu/trusty
-
-#package linux amd64 rpm

pkg/handler/http.go

@@ -0,0 +1,175 @@
+package handler
+
+import (
+	"io"
+	"net/http"
+	"strings"
+	"time"
+
+	"github.com/foomo/contentserver/pkg/metrics"
+	"github.com/foomo/contentserver/pkg/repo"
+	"github.com/foomo/contentserver/requests"
+	"github.com/foomo/contentserver/responses"
+	httputils "github.com/foomo/keel/utils/net/http"
+	"github.com/pkg/errors"
+	"go.uber.org/zap"
+)
+
+type (
+	HTTP struct {
+		l    *zap.Logger
+		path string
+		repo *repo.Repo
+	}
+	HTTPOption func(*HTTP)
+)
+
+// ------------------------------------------------------------------------------------------------
+// ~ Constructor
+// ------------------------------------------------------------------------------------------------
+
+// NewHTTP returns a shiny new web server
+func NewHTTP(l *zap.Logger, repo *repo.Repo, opts ...HTTPOption) http.Handler {
+	inst := &HTTP{
+		l:    l.Named("http"),
+		path: "/contentserver",
+		repo: repo,
+	}
+
+	for _, opt := range opts {
+		opt(inst)
+	}
+
+	return inst
+}
+
+// ------------------------------------------------------------------------------------------------
+// ~ Options
+// ------------------------------------------------------------------------------------------------
+
+func WithPath(v string) HTTPOption {
+	return func(o *HTTP) {
+		o.path = v
+	}
+}
+
+// ------------------------------------------------------------------------------------------------
+// ~ Public methods
+// ------------------------------------------------------------------------------------------------
+
+func (h *HTTP) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	if r.Method != http.MethodPost {
+		httputils.ServerError(h.l, w, r, http.StatusMethodNotAllowed, errors.New("method not allowed"))
+		return
+	}
+	if r.Body == nil {
+		httputils.BadRequestServerError(h.l, w, r, errors.New("empty request body"))
+		return
+	}
+
+	bytes, err := io.ReadAll(r.Body)
+	if err != nil {
+		httputils.BadRequestServerError(h.l, w, r, errors.Wrap(err, "failed to read incoming request"))
+		return
+	}
+
+	route := Route(strings.TrimPrefix(r.URL.Path, h.path+"/"))
+	if route == RouteGetRepo {
+		h.repo.WriteRepoBytes(w)
+		w.Header().Set("Content-Type", "application/json")
+		return
+	}
+
+	reply, errReply := h.handleRequest(h.repo, route, bytes, "webserver")
+	if errReply != nil {
+		http.Error(w, errReply.Error(), http.StatusInternalServerError)
+		return
+	}
+	_, _ = w.Write(reply)
+}
+
+// ------------------------------------------------------------------------------------------------
+// ~ Private methods
+// ------------------------------------------------------------------------------------------------
+
+func (h *HTTP) handleRequest(r *repo.Repo, handler Route, jsonBytes []byte, source string) ([]byte, error) {
+	start := time.Now()
+
+	reply, err := h.executeRequest(r, handler, jsonBytes, source)
+	result := "success"
+	if err != nil {
+		result = "error"
+	}
+
+	metrics.ServiceRequestCounter.WithLabelValues(string(handler), result, source).Inc()
+	metrics.ServiceRequestDuration.WithLabelValues(string(handler), result, source).Observe(time.Since(start).Seconds())
+
+	return reply, err
+}
+
+func (h *HTTP) executeRequest(r *repo.Repo, handler Route, jsonBytes []byte, source string) (replyBytes []byte, err error) {
+	var (
+		reply             interface{}
+		apiErr            error
+		jsonErr           error
+		processIfJSONIsOk = func(err error, processingFunc func()) {
+			if err != nil {
+				jsonErr = err
+				return
+			}
+			processingFunc()
+		}
+	)
+	metrics.ContentRequestCounter.WithLabelValues(source).Inc()
+
+	// handle and process
+	switch handler {
+	// case HandlerGetRepo: // This case is handled prior to handleRequest being called.
+	// since the resulting bytes are written directly in to the http.ResponseWriter / net.Connection
+	case RouteGetURIs:
+		getURIRequest := &requests.URIs{}
+		processIfJSONIsOk(json.Unmarshal(jsonBytes, &getURIRequest), func() {
+			reply = r.GetURIs(getURIRequest.Dimension, getURIRequest.IDs)
+		})
+	case RouteGetContent:
+		contentRequest := &requests.Content{}
+		processIfJSONIsOk(json.Unmarshal(jsonBytes, &contentRequest), func() {
+			reply, apiErr = r.GetContent(contentRequest)
+		})
+	case RouteGetNodes:
+		nodesRequest := &requests.Nodes{}
+		processIfJSONIsOk(json.Unmarshal(jsonBytes, &nodesRequest), func() {
+			reply = r.GetNodes(nodesRequest)
+		})
+	case RouteUpdate:
+		updateRequest := &requests.Update{}
+		processIfJSONIsOk(json.Unmarshal(jsonBytes, &updateRequest), func() {
+			reply = r.Update()
+		})
+	default:
+		reply = responses.NewError(1, "unknown handler: "+string(handler))
+	}
+
+	// error handling
+	if jsonErr != nil {
+		h.l.Error("could not read incoming json", zap.Error(jsonErr))
+		reply = responses.NewError(2, "could not read incoming json "+jsonErr.Error())
+	} else if apiErr != nil {
+		h.l.Error("an API error occurred", zap.Error(apiErr))
+		reply = responses.NewError(3, "internal error "+apiErr.Error())
+	}
+
+	return h.encodeReply(reply)
+}
+
+// encodeReply takes an interface and encodes it as JSON
+// it returns the resulting JSON and a marshalling error
+func (h *HTTP) encodeReply(reply interface{}) (bytes []byte, err error) {
+	bytes, err = json.Marshal(map[string]interface{}{
+		"reply": reply,
+	})
+	if err != nil {
+		h.l.Error("could not encode reply", zap.Error(err))
+	}
+	return
+}

pkg/handler/json.go

@@ -0,0 +1,7 @@
+package handler
+
+import (
+	jsoniter "github.com/json-iterator/go"
+)
+
+var json = jsoniter.ConfigCompatibleWithStandardLibrary

pkg/handler/route.go

@@ -0,0 +1,17 @@
+package handler
+
+// Route type
+type Route string
+
+const (
+	// RouteGetURIs get uris, many at once, to keep it fast
+	RouteGetURIs Route = "getURIs"
+	// RouteGetContent get (site) content
+	RouteGetContent Route = "getContent"
+	// RouteGetNodes get nodes
+	RouteGetNodes Route = "getNodes"
+	// RouteUpdate update repo
+	RouteUpdate Route = "update"
+	// RouteGetRepo get the whole repo
+	RouteGetRepo Route = "getRepo"
+)

pkg/handler/socket.go

@@ -0,0 +1,269 @@
+package handler
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"net"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/foomo/contentserver/requests"
+	"go.uber.org/zap"
+
+	"github.com/foomo/contentserver/pkg/metrics"
+	"github.com/foomo/contentserver/pkg/repo"
+	"github.com/foomo/contentserver/responses"
+)
+
+const sourceSocketServer = "socketserver"
+
+type Socket struct {
+	l    *zap.Logger
+	repo *repo.Repo
+}
+
+// ------------------------------------------------------------------------------------------------
+// ~ Constructor
+// ------------------------------------------------------------------------------------------------
+
+// NewSocket returns a shiny new socket server
+func NewSocket(l *zap.Logger, repo *repo.Repo) *Socket {
+	inst := &Socket{
+		l:    l.Named("socket"),
+		repo: repo,
+	}
+
+	return inst
+}
+
+// ------------------------------------------------------------------------------------------------
+// ~ Public methods
+// ------------------------------------------------------------------------------------------------
+
+func (h *Socket) Serve(conn net.Conn) {
+
+	defer func() {
+		if r := recover(); r != nil {
+			h.l.Error("panic in handle connection", zap.String("error", fmt.Sprint(r)))
+		}
+	}()
+
+	h.l.Debug("socketServer.handleConnection")
+	metrics.NumSocketsGauge.WithLabelValues(conn.RemoteAddr().String()).Inc()
+
+	var (
+		headerBuffer [1]byte
+		header       = ""
+		i            = 0
+	)
+	for {
+		i++
+		// fmt.Println("---->", i)
+		// let us read with 1 byte steps on conn until we find "{"
+		_, readErr := conn.Read(headerBuffer[0:])
+		if readErr != nil {
+			h.l.Debug("looks like the client closed the connection", zap.Error(readErr))
+			metrics.NumSocketsGauge.WithLabelValues(conn.RemoteAddr().String()).Dec()
+			return
+		}
+		// read next byte
+		current := headerBuffer[0:]
+		if string(current) == "{" {
+			// json has started
+			handler, jsonLength, headerErr := h.extractHandlerAndJSONLentgh(header)
+			// reset header
+			header = ""
+			if headerErr != nil {
+				h.l.Error("invalid request could not read header", zap.Error(headerErr))
+				encodedErr, encodingErr := h.encodeReply(responses.NewError(4, "invalid header "+headerErr.Error()))
+				if encodingErr == nil {
+					h.writeResponse(conn, encodedErr)
+				} else {
+					h.l.Error("could not respond to invalid request", zap.Error(encodingErr))
+				}
+				return
+			}
+			h.l.Debug("found json", zap.Int("length", jsonLength))
+			if jsonLength > 0 {
+
+				var (
+					// let us try to read some json
+					jsonBytes         = make([]byte, jsonLength)
+					jsonLengthCurrent = 1
+					readRound         = 0
+				)
+
+				// that is "{"
+				jsonBytes[0] = 123
+
+				for jsonLengthCurrent < jsonLength {
+					readRound++
+					readLength, jsonReadErr := conn.Read(jsonBytes[jsonLengthCurrent:jsonLength])
+					if jsonReadErr != nil {
+						// @fixme we need to force a read timeout (SetReadDeadline?), if expected jsonLength is lower than really sent bytes (e.g. if client implements protocol wrong)
+						// @todo should we check for io.EOF here
+						h.l.Error("could not read json - giving up with this client connection", zap.Error(jsonReadErr))
+						metrics.NumSocketsGauge.WithLabelValues(conn.RemoteAddr().String()).Dec()
+						return
+					}
+					jsonLengthCurrent += readLength
+					h.l.Debug("read cycle status",
+						zap.Int("jsonLengthCurrent", jsonLengthCurrent),
+						zap.Int("jsonLength", jsonLength),
+						zap.Int("readRound", readRound),
+					)
+				}
+
+				h.l.Debug("read json", zap.Int("length", len(jsonBytes)))
+
+				h.writeResponse(conn, h.execute(handler, jsonBytes))
+				// note: connection remains open
+				continue
+			}
+			h.l.Error("can not read empty json")
+			metrics.NumSocketsGauge.WithLabelValues(conn.RemoteAddr().String()).Dec()
+			return
+		}
+		// adding to header byte by byte
+		header += string(headerBuffer[0:])
+	}
+}
+
+// ------------------------------------------------------------------------------------------------
+// ~ Private methods
+// ------------------------------------------------------------------------------------------------
+
+func (h *Socket) extractHandlerAndJSONLentgh(header string) (route Route, jsonLength int, err error) {
+	headerParts := strings.Split(header, ":")
+	if len(headerParts) != 2 {
+		return "", 0, errors.New("invalid header")
+	}
+	jsonLength, err = strconv.Atoi(headerParts[1])
+	if err != nil {
+		err = fmt.Errorf("could not parse length in header: %q", header)
+	}
+	return Route(headerParts[0]), jsonLength, err
+}
+
+func (h *Socket) execute(route Route, jsonBytes []byte) (reply []byte) {
+	h.l.Debug("incoming json buffer", zap.Int("length", len(jsonBytes)))
+
+	if route == RouteGetRepo {
+		var (
+			b bytes.Buffer
+		)
+		h.repo.WriteRepoBytes(&b)
+		return b.Bytes()
+	}
+
+	reply, handlingError := h.handleRequest(h.repo, route, jsonBytes, sourceSocketServer)
+	if handlingError != nil {
+		h.l.Error("socketServer.execute failed", zap.Error(handlingError))
+	}
+	return reply
+}
+
+func (h *Socket) writeResponse(conn net.Conn, reply []byte) {
+	headerBytes := []byte(strconv.Itoa(len(reply)))
+	reply = append(headerBytes, reply...)
+	h.l.Debug("replying", zap.String("reply", string(reply)))
+	n, writeError := conn.Write(reply)
+	if writeError != nil {
+		h.l.Error("socketServer.writeResponse: could not write reply", zap.Error(writeError))
+		return
+	}
+	if n < len(reply) {
+		h.l.Error("socketServer.writeResponse: write too short",
+			zap.Int("got", n),
+			zap.Int("expected", len(reply)),
+		)
+		return
+	}
+	h.l.Debug("replied. waiting for next request on open connection")
+}
+
+func (h *Socket) handleRequest(r *repo.Repo, route Route, jsonBytes []byte, source string) ([]byte, error) {
+	start := time.Now()
+
+	reply, err := h.executeRequest(r, route, jsonBytes, source)
+	result := "success"
+	if err != nil {
+		result = "error"
+	}
+
+	metrics.ServiceRequestCounter.WithLabelValues(string(route), result, source).Inc()
+	metrics.ServiceRequestDuration.WithLabelValues(string(route), result, source).Observe(time.Since(start).Seconds())
+
+	return reply, err
+}
+
+func (h *Socket) executeRequest(r *repo.Repo, route Route, jsonBytes []byte, source string) (replyBytes []byte, err error) {
+
+	var (
+		reply             interface{}
+		apiErr            error
+		jsonErr           error
+		processIfJSONIsOk = func(err error, processingFunc func()) {
+			if err != nil {
+				jsonErr = err
+				return
+			}
+			processingFunc()
+		}
+	)
+	metrics.ContentRequestCounter.WithLabelValues(source).Inc()
+
+	// handle and process
+	switch route {
+	// case RouteGetRepo: // This case is handled prior to handleRequest being called.
+	// since the resulting bytes are written directly in to the http.ResponseWriter / net.Connection
+	case RouteGetURIs:
+		getURIRequest := &requests.URIs{}
+		processIfJSONIsOk(json.Unmarshal(jsonBytes, &getURIRequest), func() {
+			reply = r.GetURIs(getURIRequest.Dimension, getURIRequest.IDs)
+		})
+	case RouteGetContent:
+		contentRequest := &requests.Content{}
+		processIfJSONIsOk(json.Unmarshal(jsonBytes, &contentRequest), func() {
+			reply, apiErr = r.GetContent(contentRequest)
+		})
+	case RouteGetNodes:
+		nodesRequest := &requests.Nodes{}
+		processIfJSONIsOk(json.Unmarshal(jsonBytes, &nodesRequest), func() {
+			reply = r.GetNodes(nodesRequest)
+		})
+	case RouteUpdate:
+		updateRequest := &requests.Update{}
+		processIfJSONIsOk(json.Unmarshal(jsonBytes, &updateRequest), func() {
+			reply = r.Update()
+		})
+
+	default:
+		reply = responses.NewError(1, "unknown handler: "+string(route))
+	}
+
+	// error handling
+	if jsonErr != nil {
+		h.l.Error("could not read incoming json", zap.Error(jsonErr))
+		reply = responses.NewError(2, "could not read incoming json "+jsonErr.Error())
+	} else if apiErr != nil {
+		h.l.Error("an API error occured", zap.Error(apiErr))
+		reply = responses.NewError(3, "internal error "+apiErr.Error())
+	}
+
+	return h.encodeReply(reply)
+}
+
+// encodeReply takes an interface and encodes it as JSON
+// it returns the resulting JSON and a marshalling error
+func (h *Socket) encodeReply(reply interface{}) (replyBytes []byte, err error) {
+	replyBytes, err = json.Marshal(map[string]interface{}{
+		"reply": reply,
+	})
+	if err != nil {
+		h.l.Error("could not encode reply", zap.Error(err))
+	}
+	return
+}

pkg/metrics/metrics.go

@@ -0,0 +1,100 @@
+package metrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+	namespace = "contentserver"
+
+	metricLabelHandler = "handler"
+	metricLabelStatus  = "status"
+	metricLabelSource  = "source"
+	metricLabelRemote  = "remote"
+)
+
+// Metrics is the structure that holds all prometheus metrics
+var (
+	// InvalidNodeTreeRequests counts the number of invalid tree node requests
+	InvalidNodeTreeRequests = newCounterVec(
+		"invalid_node_tree_request_count",
+		"Counts the number of invalid tree nodes for a specific node",
+	)
+	// ServiceRequestCounter count the number of requests for each service function
+	ServiceRequestCounter = newCounterVec(
+		"service_request_count",
+		"Count of requests for each handler",
+		metricLabelHandler, metricLabelStatus, metricLabelSource,
+	)
+	// ServiceRequestDuration observe the duration of requests for each service function
+	ServiceRequestDuration = newSummaryVec(
+		"service_request_duration_seconds",
+		"Seconds to unmarshal requests, execute a service function and marshal its reponses",
+		metricLabelHandler, metricLabelStatus, metricLabelSource,
+	)
+	// UpdatesCompletedCounter count the number of rejected updates
+	UpdatesCompletedCounter = newCounterVec(
+		"updates_completed_count",
+		"Number of updates that were successfully completed",
+	)
+	// UpdatesFailedCounter count the number of updates that had an error
+	UpdatesFailedCounter = newCounterVec(
+		"updates_failed_count",
+		"Number of updates that failed due to an error",
+	)
+	// UpdateDuration observe the duration of each repo.update() call
+	UpdateDuration = newSummaryVec(
+		"update_duration_seconds",
+		"Duration in seconds for each successful repo.update() call",
+	)
+	// ContentRequestCounter count the total number of content requests
+	ContentRequestCounter = newCounterVec(
+		"content_request_count",
+		"Number of requests for content",
+		metricLabelSource,
+	)
+	// NumSocketsGauge keep track of the total number of open sockets
+	NumSocketsGauge = newGaugeVec(
+		"num_sockets_total",
+		"Total number of currently open socket connections",
+		metricLabelRemote,
+	)
+	// HistoryPersistFailedCounter count the number of failed attempts to persist the content history
+	HistoryPersistFailedCounter = newCounterVec(
+		"history_persist_failed_count",
+		"Number of failures to store the content history on the filesystem",
+	)
+)
+
+func newSummaryVec(name, help string, labels ...string) *prometheus.SummaryVec {
+	vec := prometheus.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Namespace: namespace,
+			Name:      name,
+			Help:      help,
+		}, labels)
+	prometheus.MustRegister(vec)
+	return vec
+}
+
+func newCounterVec(name, help string, labels ...string) *prometheus.CounterVec {
+	vec := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: namespace,
+			Name:      name,
+			Help:      help,
+		}, labels)
+	prometheus.MustRegister(vec)
+	return vec
+}
+
+func newGaugeVec(name, help string, labels ...string) *prometheus.GaugeVec {
+	vec := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Namespace: namespace,
+			Name:      name,
+			Help:      help,
+		}, labels)
+	prometheus.MustRegister(vec)
+	return vec
+}

pkg/repo/dimension.go

@@ -0,0 +1,12 @@
+package repo
+
+import (
+	"github.com/foomo/contentserver/content"
+)
+
+// Dimension dimension in a repo
+type Dimension struct {
+	Directory    map[string]*content.RepoNode
+	URIDirectory map[string]*content.RepoNode
+	Node         *content.RepoNode
+}

pkg/repo/history.go

@@ -0,0 +1,172 @@
+package repo
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"sort"
+	"strings"
+	"time"
+
+	"go.uber.org/zap"
+)
+
+const (
+	HistoryRepoJSONPrefix = "contentserver-repo-"
+	HistoryRepoJSONSuffix = ".json"
+)
+
+type (
+	History struct {
+		l      *zap.Logger
+		max    int
+		varDir string
+	}
+	HistoryOption func(*History)
+)
+
+// ------------------------------------------------------------------------------------------------
+// ~ Options
+// ------------------------------------------------------------------------------------------------
+
+func HistoryWithMax(v int) HistoryOption {
+	return func(o *History) {
+		o.max = v
+	}
+}
+
+func HistoryWithVarDir(v string) HistoryOption {
+	return func(o *History) {
+		o.varDir = v
+	}
+}
+
+// ------------------------------------------------------------------------------------------------
+// ~ Constructor
+// ------------------------------------------------------------------------------------------------
+
+func NewHistory(l *zap.Logger, opts ...HistoryOption) *History {
+	inst := &History{
+		l:      l,
+		max:    2,
+		varDir: "/var/lib/contentserver",
+	}
+
+	for _, opt := range opts {
+		opt(inst)
+	}
+
+	return inst
+}
+
+// ------------------------------------------------------------------------------------------------
+// ~ Public methods
+// ------------------------------------------------------------------------------------------------
+
+func (h *History) Add(jsonBytes []byte) error {
+	var (
+		// historiy file name
+		filename = path.Join(h.varDir, HistoryRepoJSONPrefix+time.Now().Format(time.RFC3339Nano)+HistoryRepoJSONSuffix)
+		err      = os.WriteFile(filename, jsonBytes, 0600)
+	)
+	if err != nil {
+		return err
+	}
+
+	h.l.Debug("adding content backup", zap.String("file", filename))
+
+	// current filename
+	err = os.WriteFile(h.GetCurrentFilename(), jsonBytes, 0600)
+	if err != nil {
+		return err
+	}
+
+	err = h.cleanup()
+	if err != nil {
+		h.l.Error("an error occurred while cleaning up my history", zap.Error(err))
+		return err
+	}
+
+	return nil
+}
+
+func (h *History) GetCurrentFilename() string {
+	return path.Join(h.varDir, HistoryRepoJSONPrefix+"current"+HistoryRepoJSONSuffix)
+}
+
+func (h *History) GetCurrent(buf *bytes.Buffer) (err error) {
+	f, err := os.Open(h.GetCurrentFilename())
+	if err != nil {
+		return err
+	}
+	defer f.Close()
+	_, err = io.Copy(buf, f)
+	return err
+}
+
+// ------------------------------------------------------------------------------------------------
+// ~ Private methods
+// ------------------------------------------------------------------------------------------------
+
+func (h *History) getHistory() (files []string, err error) {
+	fileInfos, err := os.ReadDir(h.varDir)
+	if err != nil {
+		return
+	}
+	currentName := h.GetCurrentFilename()
+	for _, f := range fileInfos {
+		if !f.IsDir() {
+			filename := f.Name()
+			if filename != currentName && (strings.HasPrefix(filename, HistoryRepoJSONPrefix) && strings.HasSuffix(filename, HistoryRepoJSONSuffix)) {
+				files = append(files, path.Join(h.varDir, filename))
+			}
+		}
+	}
+	sort.Sort(sort.Reverse(sort.StringSlice(files)))
+	return
+}
+
+func (h *History) cleanup() error {
+	files, err := h.getFilesForCleanup(h.max)
+	if err != nil {
+		return err
+	}
+
+	for _, f := range files {
+		h.l.Debug("removing outdated backup", zap.String("file", f))
+		err := os.Remove(f)
+		if err != nil {
+			return fmt.Errorf("could not remove file %s : %s", f, err.Error())
+		}
+	}
+
+	return nil
+}
+
+func (h *History) getFilesForCleanup(historyVersions int) (files []string, err error) {
+	contentFiles, err := h.getHistory()
+	if err != nil {
+		return nil, errors.New("could not generate file cleanup list: " + err.Error())
+	}
+
+	// fmt.Println("contentFiles:")
+	// for _, f := range contentFiles {
+	// 	fmt.Println(f)
+	// }
+
+	// -1 to remove the current backup file from the number of items
+	// so that only files with a timestamp are compared
+	if len(contentFiles)-1 > historyVersions {
+		for i := historyVersions + 1; i < len(contentFiles); i++ {
+			// ignore current repository file to fall back on
+			if contentFiles[i] == h.GetCurrentFilename() {
+				continue
+			}
+			files = append(files, contentFiles[i])
+		}
+	}
+	return files, nil
+}

pkg/repo/history_test.go

@@ -0,0 +1,75 @@
+package repo
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.uber.org/zap/zaptest"
+)
+
+func TestHistoryCurrent(t *testing.T) {
+	var (
+		h    = testHistory(t)
+		test = []byte("test")
+		b    bytes.Buffer
+	)
+	err := h.Add(test)
+	require.NoError(t, err)
+	err = h.GetCurrent(&b)
+	require.NoError(t, err)
+	if !bytes.Equal(b.Bytes(), test) {
+		t.Fatalf("expected %q, got %q", string(test), b.String())
+	}
+}
+
+func TestHistoryCleanup(t *testing.T) {
+	h := testHistory(t)
+	for i := 0; i < 50; i++ {
+		err := h.Add([]byte(fmt.Sprint(i)))
+		require.NoError(t, err)
+		time.Sleep(time.Millisecond * 5)
+	}
+	err := h.cleanup()
+	require.NoError(t, err)
+	files, err := h.getHistory()
+	require.NoError(t, err)
+
+	// -1 for ignoring the current content backup file
+	if len(files)-1 != 2 {
+		t.Fatal("history too long", len(files), "instead of", 2)
+	}
+}
+
+func TestHistoryOrder(t *testing.T) {
+	h := testHistory(t)
+	h.varDir = "testdata/order"
+
+	files, err := h.getHistory()
+	require.NoError(t, err)
+	assert.Equal(t, "testdata/order/contentserver-repo-current.json", files[0])
+	assert.Equal(t, "testdata/order/contentserver-repo-2017-10-23.json", files[1])
+	assert.Equal(t, "testdata/order/contentserver-repo-2017-10-22.json", files[2])
+	assert.Equal(t, "testdata/order/contentserver-repo-2017-10-21.json", files[3])
+}
+
+func TestGetFilesForCleanup(t *testing.T) {
+	h := testHistory(t)
+	h.varDir = "testdata/order"
+
+	files, err := h.getFilesForCleanup(2)
+	require.NoError(t, err)
+	assert.Equal(t, "testdata/order/contentserver-repo-2017-10-21.json", files[0])
+}
+
+func testHistory(t *testing.T) *History {
+	t.Helper()
+	l := zaptest.NewLogger(t)
+	tempDir, err := os.MkdirTemp(os.TempDir(), "contentserver-history-test")
+	require.NoError(t, err)
+	return NewHistory(l, HistoryWithMax(2), HistoryWithVarDir(tempDir))
+}

pkg/repo/loader.go

@@ -3,13 +3,11 @@ package repo
 import (
 	"context"
 	"io"
-	"io/ioutil"
 	"net/http"
 	"time"
 
 	"github.com/foomo/contentserver/content"
-	"github.com/foomo/contentserver/logger"
+	"github.com/foomo/contentserver/pkg/metrics"
-	"github.com/foomo/contentserver/status"
 	"github.com/google/uuid"
 	jsoniter "github.com/json-iterator/go"
 	"github.com/pkg/errors"
@@ -19,7 +17,7 @@ import (
 
 var (
 	json              = jsoniter.ConfigCompatibleWithStandardLibrary
-	errUpdateRejected = errors.New("update rejected: queue full")
+	ErrUpdateRejected = errors.New("update rejected: queue full")
 )
 
 type updateResponse struct {
@@ -27,20 +25,47 @@ type updateResponse struct {
 	err         error
 }
 
-func (repo *Repo) updateRoutine() {
+func (r *Repo) PollRoutine(ctx context.Context) error {
-	for resChan := range repo.updateInProgressChannel {
+	l := r.l.Named("routine.poll")
-		log := logger.Log.With(zap.String("updateRunID", uuid.New().String()))
+	ticker := time.NewTicker(10 * time.Second)
-
+	for {
-		log.Info("Content update started")
+		select {
-		start := time.Now()
+		case <-ctx.Done():
-
+			l.Debug("routine canceled", zap.Error(ctx.Err()))
-		repoRuntime, err := repo.update(context.Background())
+			return nil
-		if err != nil {
+		case <-ticker.C:
-			log.Error("Content update failed", zap.Error(err))
+			chanReponse := make(chan updateResponse)
-			status.M.UpdatesFailedCounter.WithLabelValues().Inc()
+			r.updateInProgressChannel <- chanReponse
+			response := <-chanReponse
+			if response.err == nil {
+				l.Info("update success", zap.String("revision", r.pollVersion))
 			} else {
-			log.Info("Content update success")
+				l.Error("update failed", zap.Error(response.err))
-			status.M.UpdatesCompletedCounter.WithLabelValues().Inc()
+			}
+		}
+	}
+}
+
+func (r *Repo) UpdateRoutine(ctx context.Context) error {
+	l := r.l.Named("routine.update")
+	for {
+		select {
+		case <-ctx.Done():
+			l.Debug("routine canceled", zap.Error(ctx.Err()))
+			return nil
+		case resChan := <-r.updateInProgressChannel:
+			start := time.Now()
+			l := l.With(zap.String("run_id", uuid.New().String()))
+
+			l.Info("update started")
+
+			repoRuntime, err := r.update(context.Background())
+			if err != nil {
+				l.Error("update failed", zap.Error(err))
+				metrics.UpdatesFailedCounter.WithLabelValues().Inc()
+			} else {
+				l.Info("update success")
+				metrics.UpdatesCompletedCounter.WithLabelValues().Inc()
 			}
 
 			resChan <- updateResponse{
@@ -48,35 +73,43 @@ func (repo *Repo) updateRoutine() {
 				err:         err,
 			}
 
-		status.M.UpdateDuration.WithLabelValues().Observe(time.Since(start).Seconds())
+			metrics.UpdateDuration.WithLabelValues().Observe(time.Since(start).Seconds())
+		}
 	}
 }
 
-func (repo *Repo) dimensionUpdateRoutine() {
+func (r *Repo) DimensionUpdateRoutine(ctx context.Context) error {
-	for newDimension := range repo.dimensionUpdateChannel {
+	l := r.l.Named("routine.dimensionUpdate")
-		logger.Log.Info("dimensionUpdateRoutine received a new dimension", zap.String("dimension", newDimension.Dimension))
+	for {
+		select {
+		case <-ctx.Done():
+			l.Debug("routine canceled", zap.Error(ctx.Err()))
+			return nil
+		case newDimension := <-r.dimensionUpdateChannel:
+			l.Debug("received a new dimension", zap.String("dimension", newDimension.Dimension))
 
-		err := repo._updateDimension(newDimension.Dimension, newDimension.Node)
+			err := r._updateDimension(newDimension.Dimension, newDimension.Node)
-		logger.Log.Info("dimensionUpdateRoutine received result")
+			l.Info("received result")
 			if err != nil {
-			logger.Log.Debug("update dimension failed", zap.Error(err))
+				l.Debug("update failed", zap.Error(err))
+			}
+			r.dimensionUpdateDoneChannel <- err
 		}
-		repo.dimensionUpdateDoneChannel <- err
 	}
 }
 
-func (repo *Repo) updateDimension(dimension string, node *content.RepoNode) error {
+func (r *Repo) updateDimension(dimension string, node *content.RepoNode) error {
-	logger.Log.Debug("trying to push dimension into update channel", zap.String("dimension", dimension), zap.String("nodeName", node.Name))
+	r.l.Debug("trying to push dimension into update channel", zap.String("dimension", dimension), zap.String("nodeName", node.Name))
-	repo.dimensionUpdateChannel <- &repoDimension{
+	r.dimensionUpdateChannel <- &RepoDimension{
 		Dimension: dimension,
 		Node:      node,
 	}
-	logger.Log.Debug("waiting for done signal")
+	r.l.Debug("waiting for done signal")
-	return <-repo.dimensionUpdateDoneChannel
+	return <-r.dimensionUpdateDoneChannel
 }
 
 // do not call directly, but only through channel
-func (repo *Repo) _updateDimension(dimension string, newNode *content.RepoNode) error {
+func (r *Repo) _updateDimension(dimension string, newNode *content.RepoNode) error {
 	newNode.WireParents()
 
 	var (
@@ -97,7 +130,7 @@ func (repo *Repo) _updateDimension(dimension string, newNode *content.RepoNode)
 	// copy old datastructure to prevent concurrent map access
 	// collect other dimension in the Directory
 	newRepoDirectory := map[string]*Dimension{}
-	for d, D := range repo.Directory {
+	for d, D := range r.Directory {
 		if d != dimension {
 			newRepoDirectory[d] = D
 		}
@@ -109,7 +142,7 @@ func (repo *Repo) _updateDimension(dimension string, newNode *content.RepoNode)
 		Directory:    newDirectory,
 		URIDirectory: newURIDirectory,
 	}
-	repo.Directory = newRepoDirectory
+	r.Directory = newRepoDirectory
 
 	// ---------------------------------------------
 
@@ -126,9 +159,6 @@ func (repo *Repo) _updateDimension(dimension string, newNode *content.RepoNode)
 }
 
 func buildDirectory(dirNode *content.RepoNode, directory map[string]*content.RepoNode, uRIDirectory map[string]*content.RepoNode) error {
-
-	// Log.Debug("buildDirectory", zap.String("ID", dirNode.ID))
-
 	existingNode, ok := directory[dirNode.ID]
 	if ok {
 		return errors.New("duplicate node with id:" + existingNode.ID)
@@ -161,26 +191,30 @@ func wireAliases(directory map[string]*content.RepoNode) error {
 	return nil
 }
 
-func (repo *Repo) loadNodesFromJSON() (nodes map[string]*content.RepoNode, err error) {
+func (r *Repo) loadNodesFromJSON() (nodes map[string]*content.RepoNode, err error) {
 	nodes = make(map[string]*content.RepoNode)
-	err = json.Unmarshal(repo.jsonBuf.Bytes(), &nodes)
+	err = json.Unmarshal(r.jsonBuf.Bytes(), &nodes)
 	if err != nil {
-		logger.Log.Error("Failed to deserialize nodes", zap.Error(err))
+		r.l.Error("Failed to deserialize nodes", zap.Error(err))
 		return nil, errors.New("failed to deserialize nodes")
 	}
 	return nodes, nil
 }
 
-func (repo *Repo) tryToRestoreCurrent() (err error) {
+func (r *Repo) tryToRestoreCurrent() error {
-	err = repo.history.getCurrent(&repo.jsonBuf)
+	err := r.history.GetCurrent(&r.jsonBuf)
 	if err != nil {
 		return err
 	}
-	return repo.loadJSONBytes()
+	return r.loadJSONBytes()
 }
 
-func (repo *Repo) get(URL string) error {
+func (r *Repo) get(ctx context.Context, url string) error {
-	response, err := repo.httpClient.Get(URL)
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return errors.Wrap(err, "failed to create get repo request")
+	}
+	response, err := r.httpClient.Do(req)
 	if err != nil {
 		return errors.Wrap(err, "failed to get repo")
 	}
@@ -191,10 +225,10 @@ func (repo *Repo) get(URL string) error {
 	}
 
 	// Log.Info(ansi.Red + "RESETTING BUFFER" + ansi.Reset)
-	repo.jsonBuf.Reset()
+	r.jsonBuf.Reset()
 
 	// Log.Info(ansi.Green + "LOADING DATA INTO BUFFER" + ansi.Reset)
-	_, err = io.Copy(&repo.jsonBuf, response.Body)
+	_, err = io.Copy(&r.jsonBuf, response.Body)
 	if err != nil {
 		return errors.Wrap(err, "failed to copy IO stream")
 	}
@@ -202,12 +236,16 @@ func (repo *Repo) get(URL string) error {
 	return nil
 }
 
-func (repo *Repo) update(ctx context.Context) (repoRuntime int64, err error) {
+func (r *Repo) update(ctx context.Context) (repoRuntime int64, err error) {
 	startTimeRepo := time.Now().UnixNano()
 
-	repoURL := repo.server
+	repoURL := r.url
-	if repo.pollForUpdates {
+	if r.poll {
-		resp, err := repo.httpClient.Get(repo.server)
+		req, err := http.NewRequestWithContext(ctx, http.MethodGet, r.url, nil)
+		if err != nil {
+			return repoRuntime, err
+		}
+		resp, err := r.httpClient.Do(req)
 		if err != nil {
 			return repoRuntime, err
 		}
@@ -215,71 +253,71 @@ func (repo *Repo) update(ctx context.Context) (repoRuntime int64, err error) {
 		if resp.StatusCode != http.StatusOK {
 			return repoRuntime, errors.New("could not poll latest repo download url - non 200 response")
 		}
-		responseBytes, err := ioutil.ReadAll(resp.Body)
+		responseBytes, err := io.ReadAll(resp.Body)
 		if err != nil {
 			return repoRuntime, errors.New("could not poll latest repo download url, could not read body")
 		}
 		repoURL = string(responseBytes)
-		if repoURL == repo.pollVersion {
+		if repoURL == r.pollVersion {
-			logger.Log.Info(
+			r.l.Info(
 				"repo is up to date",
-				zap.String("pollVersion", repo.pollVersion),
+				zap.String("pollVersion", r.pollVersion),
 			)
 			// already up to date
 			return repoRuntime, nil
 		} else {
-			logger.Log.Info(
+			r.l.Info(
 				"new repo poll version",
-				zap.String("pollVersion", repo.pollVersion),
+				zap.String("pollVersion", r.pollVersion),
 			)
 		}
 	}
 
-	err = repo.get(repoURL)
+	err = r.get(ctx, repoURL)
 	repoRuntime = time.Now().UnixNano() - startTimeRepo
 	if err != nil {
 		// we have no json to load - the repo server did not reply
-		logger.Log.Debug("Failed to load json", zap.Error(err))
+		r.l.Debug("failed to load json", zap.Error(err))
 		return repoRuntime, err
 	}
-	logger.Log.Debug("loading json", zap.String("server", repoURL), zap.Int("length", len(repo.jsonBuf.Bytes())))
+	r.l.Debug("loading json", zap.String("server", repoURL), zap.Int("length", len(r.jsonBuf.Bytes())))
-	nodes, err := repo.loadNodesFromJSON()
+	nodes, err := r.loadNodesFromJSON()
 	if err != nil {
 		// could not load nodes from json
 		return repoRuntime, err
 	}
-	err = repo.loadNodes(nodes)
+	err = r.loadNodes(nodes)
 	if err != nil {
 		// repo failed to load nodes
 		return repoRuntime, err
 	}
-	if repo.pollForUpdates {
+	if r.poll {
-		repo.pollVersion = repoURL
+		r.pollVersion = repoURL
 	}
 	return repoRuntime, nil
 }
 
 // limit ressources and allow only one update request at once
-func (repo *Repo) tryUpdate() (repoRuntime int64, err error) {
+func (r *Repo) tryUpdate() (repoRuntime int64, err error) {
 	c := make(chan updateResponse)
 	select {
-	case repo.updateInProgressChannel <- c:
+	case r.updateInProgressChannel <- c:
-		logger.Log.Info("update request added to queue")
+		r.l.Debug("update request added to queue")
 		ur := <-c
 		return ur.repoRuntime, ur.err
 	default:
-		logger.Log.Info("update request accepted, will be processed after the previous update")
+		r.l.Info("update request accepted, will be processed after the previous update")
-		return 0, errUpdateRejected
+		return 0, ErrUpdateRejected
 	}
 }
 
-func (repo *Repo) loadJSONBytes() error {
+func (r *Repo) loadJSONBytes() error {
-	nodes, err := repo.loadNodesFromJSON()
+	nodes, err := r.loadNodesFromJSON()
 	if err != nil {
-		data := repo.jsonBuf.Bytes()
+		data := r.jsonBuf.Bytes()
 
 		if len(data) > 10 {
-			logger.Log.Debug("could not parse json",
+			r.l.Debug("could not parse json",
 				zap.String("jsonStart", string(data[:10])),
 				zap.String("jsonStart", string(data[len(data)-10:])),
 			)
@@ -287,26 +325,26 @@ func (repo *Repo) loadJSONBytes() error {
 		return err
 	}
 
-	err = repo.loadNodes(nodes)
+	err = r.loadNodes(nodes)
 	if err == nil {
-		errHistory := repo.history.add(repo.jsonBuf.Bytes())
+		errHistory := r.history.Add(r.jsonBuf.Bytes())
 		if errHistory != nil {
-			logger.Log.Error("Could not add valid JSON to history", zap.Error(errHistory))
+			r.l.Error("Could not add valid JSON to history", zap.Error(errHistory))
-			status.M.HistoryPersistFailedCounter.WithLabelValues().Inc()
+			metrics.HistoryPersistFailedCounter.WithLabelValues().Inc()
 		} else {
-			logger.Log.Info("Added valid JSON to history")
+			r.l.Info("added valid JSON to history")
 		}
 	}
 	return err
 }
 
-func (repo *Repo) loadNodes(newNodes map[string]*content.RepoNode) error {
+func (r *Repo) loadNodes(newNodes map[string]*content.RepoNode) error {
-	var newDimensions []string
 	var err error
+	newDimensions := make([]string, 0, len(newNodes))
 	for dimension, newNode := range newNodes {
 		newDimensions = append(newDimensions, dimension)
-		logger.Log.Debug("Loading nodes for dimension", zap.String("dimension", dimension))
+		r.l.Debug("loading nodes for dimension", zap.String("dimension", dimension))
-		errLoad := repo.updateDimension(dimension, newNode)
+		errLoad := r.updateDimension(dimension, newNode)
 		if errLoad != nil {
 			err = multierr.Append(err, errLoad)
 		}
@@ -323,10 +361,10 @@ func (repo *Repo) loadNodes(newNodes map[string]*content.RepoNode) error {
 		return false
 	}
 	// we need to throw away orphaned dimensions
-	for dimension := range repo.Directory {
+	for dimension := range r.Directory {
 		if !dimensionIsValid(dimension) {
-			logger.Log.Info("Removing orphaned dimension", zap.String("dimension", dimension))
+			r.l.Info("removing orphaned dimension", zap.String("dimension", dimension))
-			delete(repo.Directory, dimension)
+			delete(r.Directory, dimension)
 		}
 	}
 	return nil

pkg/repo/mock/mock.go

@@ -1,32 +1,33 @@
 package mock
 
 import (
-	"io/ioutil"
 	"net/http"
 	"net/http/httptest"
+	"os"
 	"path"
 	"runtime"
 	"testing"
 	"time"
 
 	"github.com/foomo/contentserver/requests"
+	"github.com/stretchr/testify/require"
 )
 
 // GetMockData mock data to run a repo
-func GetMockData(t testing.TB) (server *httptest.Server, varDir string) {
+func GetMockData(tb testing.TB) (*httptest.Server, string) {
-
+	tb.Helper()
 	_, filename, _, _ := runtime.Caller(0)
 	mockDir := path.Dir(filename)
 
-	server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 		time.Sleep(time.Millisecond * 50)
 		mockFilename := path.Join(mockDir, req.URL.Path[1:])
 		http.ServeFile(w, req, mockFilename)
 	}))
-	varDir, err := ioutil.TempDir("", "content-server-test")
+
-	if err != nil {
+	varDir, err := os.MkdirTemp("", "content-server-test")
-		panic(err)
+	require.NoError(tb, err)
-	}
+
 	return server, varDir
 }
 
@@ -37,7 +38,7 @@ func MakeNodesRequest() *requests.Nodes {
 			Dimensions: []string{"dimension_foo"},
 		},
 		Nodes: map[string]*requests.Node{
-			"test": &requests.Node{
+			"test": {
 				ID:         "id-root",
 				Dimension:  "dimension_foo",
 				MimeTypes:  []string{},
@@ -66,7 +67,7 @@ func MakeValidContentRequest() *requests.Content {
 			Groups:     []string{},
 		},
 		Nodes: map[string]*requests.Node{
-			"id-root": &requests.Node{
+			"id-root": {
 				ID:         "id-root",
 				Dimension:  dimensions[0],
 				MimeTypes:  []string{"application/x-node"},
@@ -75,5 +76,4 @@ func MakeValidContentRequest() *requests.Content {
 			},
 		},
 	}
-
 }

pkg/repo/repo.go

@@ -0,0 +1,483 @@
+package repo
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"strings"
+	"sync/atomic"
+	"time"
+
+	"github.com/foomo/contentserver/content"
+	"github.com/foomo/contentserver/pkg/metrics"
+	"github.com/foomo/contentserver/requests"
+	"github.com/foomo/contentserver/responses"
+	"github.com/pkg/errors"
+	"go.uber.org/zap"
+	"golang.org/x/sync/errgroup"
+)
+
+const maxGetURIForNodeRecursionLevel = 1000
+
+// Repo content repository
+type (
+	Repo struct {
+		l           *zap.Logger
+		url         string
+		poll        bool
+		pollVersion string
+		onStart     func()
+		loaded      *atomic.Bool
+		history     *History
+		httpClient  *http.Client
+		Directory   map[string]*Dimension
+		// updateLock        sync.Mutex
+		dimensionUpdateChannel     chan *RepoDimension
+		dimensionUpdateDoneChannel chan error
+		updateInProgressChannel    chan chan updateResponse
+		// jsonBytes []byte
+		jsonBuf bytes.Buffer
+	}
+	Option func(*Repo)
+)
+
+// ------------------------------------------------------------------------------------------------
+// ~ Constructor
+// ------------------------------------------------------------------------------------------------
+
+func New(l *zap.Logger, url string, history *History, opts ...Option) *Repo {
+	inst := &Repo{
+		l:                          l.Named("repo"),
+		url:                        url,
+		poll:                       false,
+		loaded:                     &atomic.Bool{},
+		history:                    history,
+		httpClient:                 http.DefaultClient,
+		Directory:                  map[string]*Dimension{},
+		dimensionUpdateChannel:     make(chan *RepoDimension),
+		dimensionUpdateDoneChannel: make(chan error),
+		updateInProgressChannel:    make(chan chan updateResponse),
+	}
+
+	for _, opt := range opts {
+		opt(inst)
+	}
+
+	return inst
+}
+
+// ------------------------------------------------------------------------------------------------
+// ~ Options
+// ------------------------------------------------------------------------------------------------
+
+func WithHTTPClient(v *http.Client) Option {
+	return func(o *Repo) {
+		o.httpClient = v
+	}
+}
+
+func WithPollForUpdates(v bool) Option {
+	return func(o *Repo) {
+		o.poll = v
+	}
+}
+
+// ------------------------------------------------------------------------------------------------
+// ~ Getter
+// ------------------------------------------------------------------------------------------------
+
+func (r *Repo) Loaded() bool {
+	return r.loaded.Load()
+}
+
+func (r *Repo) OnStart(fn func()) {
+	r.onStart = fn
+}
+
+// ------------------------------------------------------------------------------------------------
+// ~ Public methods
+// ------------------------------------------------------------------------------------------------
+
+// GetURIs get many uris at once
+func (r *Repo) GetURIs(dimension string, ids []string) map[string]string {
+	uris := map[string]string{}
+	for _, id := range ids {
+		uris[id] = r.getURI(dimension, id)
+	}
+	return uris
+}
+
+// GetNodes get nodes
+func (r *Repo) GetNodes(nodes *requests.Nodes) map[string]*content.Node {
+	return r.getNodes(nodes.Nodes, nodes.Env)
+}
+
+// GetContent resolves content and fetches nodes in one call. It combines those
+// two tasks for performance reasons.
+//
+// In the first step it uses r.URI to look up content in all given
+// r.Env.Dimensions of repo.Directory.
+//
+// In the second step it collects the requested nodes.
+//
+// those two steps are independent.
+func (r *Repo) GetContent(req *requests.Content) (*content.SiteContent, error) {
+	// add more input validation
+	err := r.validateContentRequest(req)
+	if err != nil {
+		return nil, errors.Wrap(err, "repo.GetContent invalid request")
+	}
+	r.l.Debug("repo.GetContent", zap.String("URI", req.URI))
+	c := content.NewSiteContent()
+	resolved, resolvedURI, resolvedDimension, node := r.resolveContent(req.Env.Dimensions, req.URI)
+	if resolved {
+		if !node.CanBeAccessedByGroups(req.Env.Groups) {
+			r.l.Warn("Resolved content cannot be accessed by specified group", zap.String("uri", req.URI))
+			c.Status = content.StatusForbidden
+		} else {
+			r.l.Info("Content resolved", zap.String("uri", req.URI))
+			c.Status = content.StatusOk
+			c.Data = node.Data
+		}
+		c.MimeType = node.MimeType
+		c.Dimension = resolvedDimension
+		c.URI = resolvedURI
+		c.Item = node.ToItem(req.DataFields)
+		c.Path = node.GetPath(req.PathDataFields)
+		// fetch URIs for all dimensions
+		uris := make(map[string]string)
+		for dimensionName := range r.Directory {
+			uris[dimensionName] = r.getURI(dimensionName, node.ID)
+		}
+		c.URIs = uris
+	} else {
+		r.l.Info("Content not found", zap.String("URI", req.URI))
+		c.Status = content.StatusNotFound
+		c.Dimension = req.Env.Dimensions[0]
+
+		r.l.Debug("Failed to resolve, falling back to default dimension",
+			zap.String("uri", req.URI),
+			zap.String("default_dimension", req.Env.Dimensions[0]),
+		)
+		// r.Env.Dimensions is validated => we can access it
+		resolvedDimension = req.Env.Dimensions[0]
+	}
+
+	// add navigation trees
+	for _, node := range req.Nodes {
+		if node.Dimension == "" {
+			node.Dimension = resolvedDimension
+		}
+	}
+	c.Nodes = r.getNodes(req.Nodes, req.Env)
+	return c, nil
+}
+
+// GetRepo get the whole repo in all dimensions
+func (r *Repo) GetRepo() map[string]*content.RepoNode {
+	response := make(map[string]*content.RepoNode)
+	for dimensionName, dimension := range r.Directory {
+		response[dimensionName] = dimension.Node
+	}
+	return response
+}
+
+// WriteRepoBytes get the whole repo in all dimensions
+// reads the JSON history file from the Filesystem and copies it directly in to the supplied buffer
+// the result is wrapped as service response, e.g: {"reply": <contentData>}
+func (r *Repo) WriteRepoBytes(w io.Writer) {
+	f, err := os.Open(r.history.GetCurrentFilename())
+	if err != nil {
+		r.l.Error("Failed to open Repo JSON", zap.Error(err))
+	}
+
+	_, _ = w.Write([]byte("{\"reply\":"))
+	_, err = io.Copy(w, f)
+	if err != nil {
+		r.l.Error("Failed to serve Repo JSON", zap.Error(err))
+	}
+	_, _ = w.Write([]byte("}"))
+}
+
+func (r *Repo) Update() (updateResponse *responses.Update) {
+	floatSeconds := func(nanoSeconds int64) float64 {
+		return float64(nanoSeconds) / float64(1000000000)
+	}
+
+	r.l.Info("Update triggered")
+	// Log.Info(ansi.Yellow + "BUFFER LENGTH BEFORE tryUpdate(): " + strconv.Itoa(len(repo.jsonBuf.Bytes())) + ansi.Reset)
+
+	start := time.Now()
+	updateRepotime, err := r.tryUpdate()
+	updateResponse = &responses.Update{}
+	updateResponse.Stats.RepoRuntime = floatSeconds(updateRepotime)
+
+	if err != nil {
+		updateResponse.Success = false
+		updateResponse.Stats.NumberOfNodes = -1
+		updateResponse.Stats.NumberOfURIs = -1
+
+		// let us try to restore the world from a file
+		// Log.Info(ansi.Yellow + "BUFFER LENGTH AFTER ERROR: " + strconv.Itoa(len(r.jsonBuf.Bytes())) + ansi.Reset)
+		// only try to restore if the update failed during processing
+
+		if !errors.Is(err, ErrUpdateRejected) {
+			updateResponse.ErrorMessage = err.Error()
+			r.l.Error("Failed to update repository", zap.Error(err))
+
+			restoreErr := r.tryToRestoreCurrent()
+			if restoreErr != nil {
+				r.l.Error("Failed to restore preceding repository version", zap.Error(restoreErr))
+			} else {
+				r.l.Info("Successfully restored current repository from local history")
+			}
+		}
+	} else {
+		updateResponse.Success = true
+		// persist the currently loaded one
+		historyErr := r.history.Add(r.jsonBuf.Bytes())
+		if historyErr != nil {
+			r.l.Error("Could not persist current repo in history", zap.Error(historyErr))
+			metrics.HistoryPersistFailedCounter.WithLabelValues().Inc()
+		}
+		// add some stats
+		for dimension := range r.Directory {
+			updateResponse.Stats.NumberOfNodes += len(r.Directory[dimension].Directory)
+			updateResponse.Stats.NumberOfURIs += len(r.Directory[dimension].URIDirectory)
+		}
+		r.loaded.Store(true)
+	}
+	updateResponse.Stats.OwnRuntime = floatSeconds(time.Since(start).Nanoseconds()) - updateResponse.Stats.RepoRuntime
+	return updateResponse
+}
+
+func (r *Repo) Start(ctx context.Context) error {
+	g, gCtx := errgroup.WithContext(ctx)
+
+	l := r.l.Named("start")
+
+	up := make(chan bool, 1)
+	g.Go(func() error {
+		l.Debug("starting update routine")
+		up <- true
+		return r.UpdateRoutine(gCtx)
+	})
+	l.Debug("waiting for UpdateRoutine")
+	<-up
+
+	g.Go(func() error {
+		l.Debug("starting dimension update routine")
+		up <- true
+		return r.DimensionUpdateRoutine(gCtx)
+	})
+	l.Debug("waiting for DimensionUpdateRoutine")
+	<-up
+
+	l.Debug("trying to restore previous repo")
+	if err := r.tryToRestoreCurrent(); errors.Is(err, os.ErrNotExist) {
+		l.Info("previous repo content file does not exist")
+	} else if err != nil {
+		l.Warn("could not restore previous repo content", zap.Error(err))
+	} else {
+		l.Info("restored previous repo")
+		r.loaded.Store(true)
+	}
+
+	if r.poll {
+		g.Go(func() error {
+			l.Debug("starting poll routine")
+			return r.PollRoutine(gCtx)
+		})
+	} else if !r.Loaded() {
+		l.Debug("trying to update initial state")
+		if resp := r.Update(); !resp.Success {
+			l.Fatal("failed to update",
+				zap.String("error", resp.ErrorMessage),
+				zap.Int("num_modes", resp.Stats.NumberOfNodes),
+				zap.Int("num_uris", resp.Stats.NumberOfURIs),
+				zap.Float64("own_runtime", resp.Stats.OwnRuntime),
+				zap.Float64("repo_runtime", resp.Stats.RepoRuntime),
+			)
+		}
+	}
+
+	if r.onStart != nil {
+		r.onStart()
+	}
+
+	return g.Wait()
+}
+
+// ------------------------------------------------------------------------------------------------
+// ~ Private methods
+// ------------------------------------------------------------------------------------------------
+
+func (r *Repo) getNodes(nodeRequests map[string]*requests.Node, env *requests.Env) map[string]*content.Node {
+	var (
+		path  []*content.Item
+		nodes = map[string]*content.Node{}
+	)
+	for nodeName, nodeRequest := range nodeRequests {
+		if nodeName == "" || nodeRequest.ID == "" {
+			r.l.Warn("invalid node request", zap.Error(errors.New("nodeName or nodeRequest.ID empty")))
+			continue
+		}
+		r.l.Debug("adding node", zap.String("name", nodeName), zap.String("requestID", nodeRequest.ID))
+
+		groups := env.Groups
+		if len(nodeRequest.Groups) > 0 {
+			groups = nodeRequest.Groups
+		}
+
+		dimensionNode, ok := r.Directory[nodeRequest.Dimension]
+		nodes[nodeName] = nil
+
+		if !ok && nodeRequest.Dimension == "" {
+			r.l.Debug("Could not get dimension root node", zap.String("dimension", nodeRequest.Dimension))
+			for _, dimension := range env.Dimensions {
+				dimensionNode, ok = r.Directory[dimension]
+				if ok {
+					r.l.Debug("Found root node in env.Dimensions", zap.String("dimension", dimension))
+					break
+				}
+				r.l.Debug("Could NOT find root node in env.Dimensions", zap.String("dimension", dimension))
+			}
+		}
+
+		if !ok {
+			r.l.Error("could not get dimension root node", zap.String("nodeRequest.Dimension", nodeRequest.Dimension))
+			continue
+		}
+
+		treeNode, ok := dimensionNode.Directory[nodeRequest.ID]
+		if !ok {
+			r.l.Error("Invalid tree node requested",
+				zap.String("nodeName", nodeName),
+				zap.String("nodeID", nodeRequest.ID),
+			)
+			metrics.InvalidNodeTreeRequests.WithLabelValues().Inc()
+			continue
+		}
+		nodes[nodeName] = r.getNode(treeNode, nodeRequest.Expand, nodeRequest.MimeTypes, path, 0, groups, nodeRequest.DataFields, nodeRequest.ExposeHiddenNodes)
+	}
+	return nodes
+}
+
+// resolveContent find content in a repository
+func (r *Repo) resolveContent(dimensions []string, uri string) (resolved bool, resolvedURI string, resolvedDimension string, repoNode *content.RepoNode) {
+	parts := strings.Split(uri, content.PathSeparator)
+	r.l.Debug("repo.ResolveContent", zap.String("URI", uri))
+	for i := len(parts); i > 0; i-- {
+		testURI := strings.Join(parts[0:i], content.PathSeparator)
+		if testURI == "" {
+			testURI = content.PathSeparator
+		}
+		for _, dimension := range dimensions {
+			if d, ok := r.Directory[dimension]; ok {
+				r.l.Debug("Checking node",
+					zap.String("dimension", dimension),
+					zap.String("URI", testURI),
+				)
+				if repoNode, ok := d.URIDirectory[testURI]; ok {
+					resolved = true
+					r.l.Debug("Node found", zap.String("URI", testURI), zap.String("destination", repoNode.DestinationID))
+					if len(repoNode.DestinationID) > 0 {
+						if destionationNode, destinationNodeOk := d.Directory[repoNode.DestinationID]; destinationNodeOk {
+							repoNode = destionationNode
+						}
+					}
+					return resolved, testURI, dimension, repoNode
+				}
+			}
+		}
+	}
+	return
+}
+
+func (r *Repo) getURIForNode(dimension string, repoNode *content.RepoNode, recursionLevel int64) (uri string) {
+	if len(repoNode.LinkID) == 0 {
+		uri = repoNode.URI
+		return
+	}
+	linkedNode, ok := r.Directory[dimension].Directory[repoNode.LinkID]
+	if ok {
+		if recursionLevel > maxGetURIForNodeRecursionLevel {
+			r.l.Error("maxGetURIForNodeRecursionLevel reached", zap.String("repoNode.ID", repoNode.ID), zap.String("linkID", repoNode.LinkID), zap.String("dimension", dimension))
+			return ""
+		}
+		return r.getURIForNode(dimension, linkedNode, recursionLevel+1)
+	}
+	return
+}
+
+func (r *Repo) getURI(dimension string, id string) string {
+	directory, ok := r.Directory[dimension]
+	if !ok {
+		return ""
+	}
+	repoNode, ok := directory.Directory[id]
+	if !ok {
+		return ""
+	}
+	return r.getURIForNode(dimension, repoNode, 0)
+}
+
+func (r *Repo) getNode(
+	repoNode *content.RepoNode,
+	expanded bool,
+	mimeTypes []string,
+	path []*content.Item,
+	level int,
+	groups []string,
+	dataFields []string,
+	exposeHiddenNodes bool,
+) *content.Node {
+	node := content.NewNode()
+	node.Item = repoNode.ToItem(dataFields)
+	r.l.Debug("getNode", zap.String("ID", repoNode.ID))
+	for _, childID := range repoNode.Index {
+		childNode := repoNode.Nodes[childID]
+		if (level == 0 || expanded || !expanded && childNode.InPath(path)) && (!childNode.Hidden || exposeHiddenNodes) && childNode.CanBeAccessedByGroups(groups) && childNode.IsOneOfTheseMimeTypes(mimeTypes) {
+			node.Nodes[childID] = r.getNode(childNode, expanded, mimeTypes, path, level+1, groups, dataFields, exposeHiddenNodes)
+			node.Index = append(node.Index, childID)
+		}
+	}
+	return node
+}
+
+func (r *Repo) validateContentRequest(req *requests.Content) (err error) {
+	if req == nil {
+		return errors.New("request must not be nil")
+	}
+	if len(req.URI) == 0 {
+		return errors.New("request URI must not be empty")
+	}
+	if req.Env == nil {
+		return errors.New("request.Env must not be nil")
+	}
+	if len(req.Env.Dimensions) == 0 {
+		return errors.New("request.Env.Dimensions must not be empty")
+	}
+	for _, envDimension := range req.Env.Dimensions {
+		if !r.hasDimension(envDimension) {
+			availableDimensions := make([]string, 0, len(r.Directory))
+			for availableDimension := range r.Directory {
+				availableDimensions = append(availableDimensions, availableDimension)
+			}
+			return errors.New(fmt.Sprint(
+				"unknown dimension ", envDimension,
+				" in r.Env must be one of ", availableDimensions,
+				" repo has ", len(r.Directory), " dimensions",
+			))
+		}
+	}
+	return nil
+}
+
+func (r *Repo) hasDimension(d string) bool {
+	_, hasDimension := r.Directory[d]
+	return hasDimension
+}

pkg/repo/repo_test.go

@@ -1,33 +1,29 @@
 package repo
 
 import (
+	"context"
 	"strings"
 	"testing"
 	"time"
 
-	. "github.com/foomo/contentserver/logger"
+	"github.com/foomo/contentserver/pkg/repo/mock"
-	_ "github.com/foomo/contentserver/logger"
-	"github.com/foomo/contentserver/repo/mock"
 	"github.com/foomo/contentserver/requests"
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"go.uber.org/zap"
+	"go.uber.org/zap/zaptest"
 )
 
-func init() {
+func NewTestRepo(l *zap.Logger, server, varDir string) *Repo {
-	SetupLogging(true, "contentserver_repo_test.log")
+	h := NewHistory(l, HistoryWithMax(2), HistoryWithVarDir(varDir))
-}
+	r := New(l, server, h)
-
+	go r.Start(context.Background()) //nolint:errcheck
-func NewTestRepo(server, varDir string) *Repo {
+	time.Sleep(100 * time.Millisecond)
-
-	r := NewRepo(server, varDir, 2*time.Minute, false)
-
-	// because the travis CI VMs are very slow,
-	// we need to add some delay to allow the server to startup
-	time.Sleep(1 * time.Second)
-
 	return r
 }
 
 func assertRepoIsEmpty(t *testing.T, r *Repo, empty bool) {
+	t.Helper()
 	if empty {
 		if len(r.Directory) > 0 {
 			t.Fatal("directory should have been empty, but is not")
@@ -40,10 +36,12 @@ func assertRepoIsEmpty(t *testing.T, r *Repo, empty bool) {
 }
 
 func TestLoad404(t *testing.T) {
+	l := zaptest.NewLogger(t)
+
 	var (
 		mockServer, varDir = mock.GetMockData(t)
 		server             = mockServer.URL + "/repo-no-have"
-		r                  = NewTestRepo(server, varDir)
+		r                  = NewTestRepo(l, server, varDir)
 	)
 
 	response := r.Update()
@@ -53,10 +51,12 @@ func TestLoad404(t *testing.T) {
 }
 
 func TestLoadBrokenRepo(t *testing.T) {
+	l := zaptest.NewLogger(t)
+
 	var (
 		mockServer, varDir = mock.GetMockData(t)
 		server             = mockServer.URL + "/repo-broken-json.json"
-		r                  = NewTestRepo(server, varDir)
+		r                  = NewTestRepo(l, server, varDir)
 	)
 
 	response := r.Update()
@@ -66,11 +66,12 @@ func TestLoadBrokenRepo(t *testing.T) {
 }
 
 func TestLoadRepo(t *testing.T) {
+	l := zaptest.NewLogger(t)
 
 	var (
 		mockServer, varDir = mock.GetMockData(t)
 		server             = mockServer.URL + "/repo-ok.json"
-		r                  = NewTestRepo(server, varDir)
+		r                  = NewTestRepo(l, server, varDir)
 	)
 	assertRepoIsEmpty(t, r, true)
 
@@ -83,22 +84,23 @@ func TestLoadRepo(t *testing.T) {
 	if response.Stats.OwnRuntime > response.Stats.RepoRuntime {
 		t.Fatal("how could all take less time, than me alone")
 	}
-	if response.Stats.RepoRuntime < float64(0.05) {
+	if response.Stats.RepoRuntime < 0.05 {
 		t.Fatal("the server was too fast")
 	}
 
 	// see what happens if we try to start it up again
-	nr := NewTestRepo(server, varDir)
+	// nr := NewTestRepo(l, server, varDir)
-	assertRepoIsEmpty(t, nr, false)
+	// assertRepoIsEmpty(t, nr, false)
 }
 
 func BenchmarkLoadRepo(b *testing.B) {
+	l := zaptest.NewLogger(b)
 
 	var (
 		t                  = &testing.T{}
 		mockServer, varDir = mock.GetMockData(t)
 		server             = mockServer.URL + "/repo-ok.json"
-		r                  = NewTestRepo(server, varDir)
+		r                  = NewTestRepo(l, server, varDir)
 	)
 	if len(r.Directory) > 0 {
 		b.Fatal("directory should have been empty, but is not")
@@ -119,114 +121,93 @@ func BenchmarkLoadRepo(b *testing.B) {
 }
 
 func TestLoadRepoDuplicateUris(t *testing.T) {
+	l := zaptest.NewLogger(t)
 
-	var (
+	mockServer, varDir := mock.GetMockData(t)
-		mockServer, varDir = mock.GetMockData(t)
+	server := mockServer.URL + "/repo-duplicate-uris.json"
-		server             = mockServer.URL + "/repo-duplicate-uris.json"
+	r := NewTestRepo(l, server, varDir)
-		r                  = NewTestRepo(server, varDir)
-	)
 
 	response := r.Update()
-	if response.Success {
+	require.False(t, response.Success, "there are duplicates, this repo update should have failed")
-		t.Fatal("there are duplicates, this repo update should have failed")
+
-	}
+	assert.True(t, strings.Contains(response.ErrorMessage, "update dimension"), "error message not as expected: "+response.ErrorMessage)
-	if !strings.Contains(response.ErrorMessage, "update dimension") {
-		t.Fatal("error message not as expected: " + response.ErrorMessage)
-	}
 }
 
 func TestDimensionHygiene(t *testing.T) {
+	l := zaptest.NewLogger(t)
 
-	var (
+	mockServer, varDir := mock.GetMockData(t)
-		mockServer, varDir = mock.GetMockData(t)
+	server := mockServer.URL + "/repo-two-dimensions.json"
-		server             = mockServer.URL + "/repo-two-dimensions.json"
+	r := NewTestRepo(l, server, varDir)
-		r                  = NewTestRepo(server, varDir)
-	)
 
 	response := r.Update()
-	if !response.Success {
+	require.True(t, response.Success, "well those two dimension should be fine")
-		t.Fatal("well those two dimension should be fine")
+
-	}
+	r.url = mockServer.URL + "/repo-ok.json"
-	r.server = mockServer.URL + "/repo-ok.json"
 	response = r.Update()
-	if !response.Success {
+	require.True(t, response.Success, "it is called repo ok")
-		t.Fatal("wtf it is called repo ok")
+
-	}
+	assert.Lenf(t, r.Directory, 1, "directory hygiene failed")
-	if len(r.Directory) != 1 {
-		t.Fatal("directory hygiene failed")
-	}
 }
 
-func getTestRepo(path string, t *testing.T) *Repo {
+func getTestRepo(t *testing.T, path string) *Repo {
+	t.Helper()
+	l := zaptest.NewLogger(t)
+
+	mockServer, varDir := mock.GetMockData(t)
+	server := mockServer.URL + path
+	r := NewTestRepo(l, server, varDir)
+	response := r.Update()
+
+	require.True(t, response.Success, "well those two dimension should be fine")
 
-	var (
-		mockServer, varDir = mock.GetMockData(t)
-		server             = mockServer.URL + path
-		r                  = NewTestRepo(server, varDir)
-		response           = r.Update()
-	)
-	if !response.Success {
-		t.Fatal("well those two dimension should be fine")
-	}
 	return r
 }
 
 func TestGetNodes(t *testing.T) {
-	var (
+	r := getTestRepo(t, "/repo-two-dimensions.json")
-		r            = getTestRepo("/repo-two-dimensions.json", t)
+	nodesRequest := mock.MakeNodesRequest()
-		nodesRequest = mock.MakeNodesRequest()
+	nodes := r.GetNodes(nodesRequest)
-		nodes        = r.GetNodes(nodesRequest)
+	testNode, ok := nodes["test"]
-		testNode, ok = nodes["test"]
+
-	)
+	require.True(t, ok, "should be a node")
-	if !ok {
+
-		t.Fatal("wtf that should be a node")
-	}
 	testData, ok := testNode.Item.Data["foo"]
+	require.True(t, ok, "failed to fetch test data")
+
 	t.Log("testData", testData)
-	if !ok {
-		t.Fatal("failed to fetch test data")
-	}
 }
 
 func TestGetNodesExposeHidden(t *testing.T) {
-	var (
+	r := getTestRepo(t, "/repo-ok-exposehidden.json")
-		r            = getTestRepo("/repo-ok-exposehidden.json", t)
+	nodesRequest := mock.MakeNodesRequest()
-		nodesRequest = mock.MakeNodesRequest()
-	)
 	nodesRequest.Nodes["test"].ExposeHiddenNodes = true
 	nodes := r.GetNodes(nodesRequest)
+
 	testNode, ok := nodes["test"]
-	if !ok {
+	require.True(t, ok, "should be a node")
-		t.Fatal("wtf that should be a node")
+
-	}
 	_, ok = testNode.Item.Data["foo"]
-	if !ok {
+	require.True(t, ok, "failed to fetch test data")
-		t.Fatal("failed to fetch test data")
+
-	}
 	require.Equal(t, 2, len(testNode.Nodes))
 }
+
 func TestResolveContent(t *testing.T) {
-
+	r := getTestRepo(t, "/repo-two-dimensions.json")
-	var (
+	contentRequest := mock.MakeValidContentRequest()
-		r                = getTestRepo("/repo-two-dimensions.json", t)
+	siteContent, err := r.GetContent(contentRequest)
-		contentRequest   = mock.MakeValidContentRequest()
+	require.NoError(t, err)
-		siteContent, err = r.GetContent(contentRequest)
+	assert.Equal(t, contentRequest.URI, siteContent.URI, "failed to resolve uri")
-	)
-
-	if siteContent.URI != contentRequest.URI {
-		t.Fatal("failed to resolve uri")
-	}
-	if err != nil {
-		t.Fatal(err)
-	}
 }
 
 func TestLinkIds(t *testing.T) {
+	l := zaptest.NewLogger(t)
 
 	var (
 		mockServer, varDir = mock.GetMockData(t)
 		server             = mockServer.URL + "/repo-link-ok.json"
-		r                  = NewTestRepo(server, varDir)
+		r                  = NewTestRepo(l, server, varDir)
 		response           = r.Update()
 	)
 
@@ -234,18 +215,16 @@ func TestLinkIds(t *testing.T) {
 		t.Fatal("those links should have been fine")
 	}
 
-	r.server = mockServer.URL + "/repo-link-broken.json"
+	r.url = mockServer.URL + "/repo-link-broken.json"
 	response = r.Update()
 
 	if response.Success {
 		t.Fatal("I do not think so")
 	}
-
 }
 
 func TestInvalidRequest(t *testing.T) {
-
+	r := getTestRepo(t, "/repo-two-dimensions.json")
-	r := getTestRepo("/repo-two-dimensions.json", t)
 
 	if r.validateContentRequest(mock.MakeValidContentRequest()) != nil {
 		t.Fatal("failed validation a valid request")

pkg/repo/repodimension.go

@@ -0,0 +1,10 @@
+package repo
+
+import (
+	"github.com/foomo/contentserver/content"
+)
+
+type RepoDimension struct {
+	Dimension string
+	Node      *content.RepoNode
+}

pkg/utils/url.go

@@ -0,0 +1,15 @@
+package utils
+
+import (
+	"net/url"
+)
+
+func IsValidUrl(str string) bool {
+	u, err := url.Parse(str)
+
+	if u.Scheme != "http" && u.Scheme != "https" {
+		return false
+	}
+
+	return err == nil && u.Scheme != "" && u.Host != ""
+}

prometheus/prometheus.yml

@@ -1,14 +0,0 @@
-# reference: https://prometheus.io/docs/prometheus/latest/configuration/configuration/
-
-global:
-  scrape_interval: 15s
-  scrape_timeout: 15s
-  #evaluation_interval: 15s
-
-scrape_configs:
-  - job_name: contentserver
-    metrics_path: /metrics
-    scheme: http
-    static_configs:
-      - targets:
-        - 127.0.0.1:9111

repo/history.go

@@ -1,137 +0,0 @@
-package repo
-
-import (
-	"bytes"
-	"errors"
-	"flag"
-	"fmt"
-	"io"
-	"io/ioutil"
-	"os"
-	"path"
-	"sort"
-	"strings"
-	"time"
-
-	. "github.com/foomo/contentserver/logger"
-	"go.uber.org/zap"
-)
-
-const (
-	historyRepoJSONPrefix = "contentserver-repo-"
-	historyRepoJSONSuffix = ".json"
-)
-
-var flagMaxHistoryVersions = flag.Int("max-history", 2, "set the maximum number of content backup files")
-
-type history struct {
-	varDir string
-}
-
-func newHistory(varDir string) *history {
-	return &history{
-		varDir: varDir,
-	}
-}
-
-func (h *history) add(jsonBytes []byte) error {
-
-	var (
-		// historiy file name
-		filename = path.Join(h.varDir, historyRepoJSONPrefix+time.Now().Format(time.RFC3339Nano)+historyRepoJSONSuffix)
-		err      = ioutil.WriteFile(filename, jsonBytes, 0644)
-	)
-	if err != nil {
-		return err
-	}
-
-	Log.Info("adding content backup", zap.String("file", filename))
-
-	// current filename
-	err = ioutil.WriteFile(h.getCurrentFilename(), jsonBytes, 0644)
-	if err != nil {
-		return err
-	}
-
-	err = h.cleanup()
-	if err != nil {
-		Log.Error("an error occured while cleaning up my history", zap.Error(err))
-		return err
-	}
-
-	return nil
-}
-
-func (h *history) getHistory() (files []string, err error) {
-	fileInfos, err := ioutil.ReadDir(h.varDir)
-	if err != nil {
-		return
-	}
-	currentName := h.getCurrentFilename()
-	for _, f := range fileInfos {
-		if !f.IsDir() {
-			filename := f.Name()
-			if filename != currentName && (strings.HasPrefix(filename, historyRepoJSONPrefix) && strings.HasSuffix(filename, historyRepoJSONSuffix)) {
-				files = append(files, path.Join(h.varDir, filename))
-			}
-		}
-	}
-	sort.Sort(sort.Reverse(sort.StringSlice(files)))
-	return
-}
-
-func (h *history) cleanup() error {
-	files, err := h.getFilesForCleanup(*flagMaxHistoryVersions)
-	if err != nil {
-		return err
-	}
-
-	for _, f := range files {
-		Log.Info("removing outdated backup", zap.String("file", f))
-		err := os.Remove(f)
-		if err != nil {
-			return fmt.Errorf("could not remove file %s : %s", f, err.Error())
-		}
-	}
-
-	return nil
-}
-
-func (h *history) getFilesForCleanup(historyVersions int) (files []string, err error) {
-	contentFiles, err := h.getHistory()
-	if err != nil {
-		return nil, errors.New("could not generate file cleanup list: " + err.Error())
-	}
-
-	// fmt.Println("contentFiles:")
-	// for _, f := range contentFiles {
-	// 	fmt.Println(f)
-	// }
-
-	// -1 to remove the current backup file from the number of items
-	// so that only files with a timestamp are compared
-	if len(contentFiles)-1 > historyVersions {
-		for i := historyVersions + 1; i < len(contentFiles); i++ {
-			// ignore current repository file to fall back on
-			if contentFiles[i] == h.getCurrentFilename() {
-				continue
-			}
-			files = append(files, contentFiles[i])
-		}
-	}
-	return files, nil
-}
-
-func (h *history) getCurrentFilename() string {
-	return path.Join(h.varDir, historyRepoJSONPrefix+"current"+historyRepoJSONSuffix)
-}
-
-func (h *history) getCurrent(buf *bytes.Buffer) (err error) {
-	f, err := os.Open(h.getCurrentFilename())
-	if err != nil {
-		return err
-	}
-	defer f.Close()
-	_, err = io.Copy(buf, f)
-	return err
-}

repo/history_test.go

@@ -1,92 +0,0 @@
-package repo
-
-import (
-	"bytes"
-	"fmt"
-	"io/ioutil"
-	"os"
-	"testing"
-	"time"
-)
-
-func testHistory() *history {
-	tempDir, err := ioutil.TempDir(os.TempDir(), "contentserver-history-test")
-	if err != nil {
-		panic(err)
-	}
-	return newHistory(tempDir)
-}
-
-func TestHistoryCurrent(t *testing.T) {
-	var (
-		h    = testHistory()
-		test = []byte("test")
-		b    bytes.Buffer
-	)
-	err := h.add(test)
-	if err != nil {
-		t.Fatal("failed to add: ", err)
-	}
-	err = h.getCurrent(&b)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if !bytes.Equal(b.Bytes(), test) {
-		t.Fatalf("expected %q, got %q", string(test), b.String())
-	}
-}
-
-func TestHistoryCleanup(t *testing.T) {
-	h := testHistory()
-	for i := 0; i < 50; i++ {
-		err := h.add([]byte(fmt.Sprint(i)))
-		if err != nil {
-			t.Fatal("failed to add: ", err)
-		}
-		time.Sleep(time.Millisecond * 5)
-	}
-	err := h.cleanup()
-	if err != nil {
-		t.Fatal("failed to run cleanup: ", err)
-	}
-	files, err := h.getHistory()
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	// -1 for ignoring the current content backup file
-	if len(files)-1 != *flagMaxHistoryVersions {
-		t.Fatal("history too long", len(files), "instead of", *flagMaxHistoryVersions)
-	}
-}
-
-func TestHistoryOrder(t *testing.T) {
-	h := testHistory()
-	h.varDir = "testdata/order"
-
-	files, err := h.getHistory()
-	if err != nil {
-		t.Fatal("error not expected")
-	}
-	assertStringEqual(t, "testdata/order/contentserver-repo-current.json", files[0])
-	assertStringEqual(t, "testdata/order/contentserver-repo-2017-10-23.json", files[1])
-	assertStringEqual(t, "testdata/order/contentserver-repo-2017-10-22.json", files[2])
-	assertStringEqual(t, "testdata/order/contentserver-repo-2017-10-21.json", files[3])
-}
-
-func TestGetFilesForCleanup(t *testing.T) {
-	h := testHistory()
-	h.varDir = "testdata/order"
-
-	files, err := h.getFilesForCleanup(2)
-	if err != nil {
-		t.Fatal("error not expected")
-	}
-	assertStringEqual(t, "testdata/order/contentserver-repo-2017-10-21.json", files[0])
-}
-
-func assertStringEqual(t *testing.T, expected, actual string) {
-	if expected != actual {
-		t.Errorf("expected string %s differs from the actual %s", expected, actual)
-	}
-}

repo/repo.go

@@ -1,440 +0,0 @@
-package repo
-
-import (
-	"bytes"
-	"crypto/tls"
-	"errors"
-	"fmt"
-	"io"
-	"net/http"
-	"os"
-	"strings"
-	"time"
-
-	"github.com/foomo/contentserver/status"
-
-	"go.uber.org/zap"
-
-	"github.com/foomo/contentserver/content"
-	"github.com/foomo/contentserver/logger"
-	"github.com/foomo/contentserver/requests"
-	"github.com/foomo/contentserver/responses"
-)
-
-const maxGetURIForNodeRecursionLevel = 1000
-
-// Dimension dimension in a repo
-type Dimension struct {
-	Directory    map[string]*content.RepoNode
-	URIDirectory map[string]*content.RepoNode
-	Node         *content.RepoNode
-}
-
-// Repo content repositiory
-type Repo struct {
-	pollForUpdates bool
-	pollVersion    string
-	server         string
-	recovered      bool
-	Directory      map[string]*Dimension
-	// updateLock        sync.Mutex
-	dimensionUpdateChannel     chan *repoDimension
-	dimensionUpdateDoneChannel chan error
-
-	history                 *history
-	updateInProgressChannel chan chan updateResponse
-
-	// jsonBytes []byte
-	jsonBuf bytes.Buffer
-
-	httpClient *http.Client
-}
-
-type repoDimension struct {
-	Dimension string
-	Node      *content.RepoNode
-}
-
-// NewRepo constructor
-func NewRepo(server string, varDir string, repositoryTimeout time.Duration, pollForUpdates bool) *Repo {
-
-	logger.Log.Info("creating new repo",
-		zap.String("server", server),
-		zap.String("varDir", varDir),
-	)
-	repo := &Repo{
-		pollForUpdates:             pollForUpdates,
-		recovered:                  false,
-		server:                     server,
-		Directory:                  map[string]*Dimension{},
-		history:                    newHistory(varDir),
-		dimensionUpdateChannel:     make(chan *repoDimension),
-		dimensionUpdateDoneChannel: make(chan error),
-		httpClient:                 getDefaultHTTPClient(repositoryTimeout),
-		updateInProgressChannel:    make(chan chan updateResponse),
-	}
-
-	go repo.updateRoutine()
-	go repo.dimensionUpdateRoutine()
-	if pollForUpdates {
-		go func() {
-			ticker := time.NewTicker(10 * time.Second)
-			for range ticker.C {
-				chanReponse := make(chan updateResponse)
-				repo.updateInProgressChannel <- chanReponse
-				response := <-chanReponse
-				if response.err == nil {
-					logger.Log.Info("poll update success", zap.String("revision", repo.pollVersion))
-				} else {
-					logger.Log.Info("poll error", zap.Error(response.err))
-				}
-			}
-		}()
-	}
-
-	logger.Log.Info("trying to restore previous state")
-	restoreErr := repo.tryToRestoreCurrent()
-	if restoreErr != nil {
-		logger.Log.Error("	could not restore previous repo content", zap.Error(restoreErr))
-	} else {
-		repo.recovered = true
-		logger.Log.Info("restored previous repo content")
-	}
-
-	return repo
-}
-
-func getDefaultHTTPClient(timeout time.Duration) *http.Client {
-	client := &http.Client{
-		Transport: &http.Transport{
-			DisableKeepAlives:   true,
-			TLSClientConfig:     &tls.Config{InsecureSkipVerify: true},
-			TLSHandshakeTimeout: 5 * time.Second,
-		},
-		Timeout: timeout,
-	}
-	return client
-}
-
-func (repo *Repo) Recovered() bool {
-	return repo.recovered
-}
-
-// GetURIs get many uris at once
-func (repo *Repo) GetURIs(dimension string, ids []string) map[string]string {
-	uris := map[string]string{}
-	for _, id := range ids {
-		uris[id] = repo.getURI(dimension, id)
-	}
-	return uris
-}
-
-// GetNodes get nodes
-func (repo *Repo) GetNodes(r *requests.Nodes) map[string]*content.Node {
-	return repo.getNodes(r.Nodes, r.Env)
-}
-
-func (repo *Repo) getNodes(nodeRequests map[string]*requests.Node, env *requests.Env) map[string]*content.Node {
-
-	var (
-		nodes = map[string]*content.Node{}
-		path  = []*content.Item{}
-	)
-	for nodeName, nodeRequest := range nodeRequests {
-		if nodeName == "" || nodeRequest.ID == "" {
-			logger.Log.Info("invalid node request", zap.Error(errors.New("nodeName or nodeRequest.ID empty")))
-			continue
-		}
-		logger.Log.Debug("adding node", zap.String("name", nodeName), zap.String("requestID", nodeRequest.ID))
-
-		groups := env.Groups
-		if len(nodeRequest.Groups) > 0 {
-			groups = nodeRequest.Groups
-		}
-
-		dimensionNode, ok := repo.Directory[nodeRequest.Dimension]
-		nodes[nodeName] = nil
-
-		if !ok && nodeRequest.Dimension == "" {
-			logger.Log.Debug("Could not get dimension root node", zap.String("dimension", nodeRequest.Dimension))
-			for _, dimension := range env.Dimensions {
-				dimensionNode, ok = repo.Directory[dimension]
-				if ok {
-					logger.Log.Debug("Found root node in env.Dimensions", zap.String("dimension", dimension))
-					break
-				}
-				logger.Log.Debug("Could NOT find root node in env.Dimensions", zap.String("dimension", dimension))
-			}
-		}
-
-		if !ok {
-			logger.Log.Error("could not get dimension root node", zap.String("nodeRequest.Dimension", nodeRequest.Dimension))
-			continue
-		}
-
-		treeNode, ok := dimensionNode.Directory[nodeRequest.ID]
-		if !ok {
-			logger.Log.Error("Invalid tree node requested",
-				zap.String("nodeName", nodeName),
-				zap.String("nodeID", nodeRequest.ID),
-			)
-			status.M.InvalidNodeTreeRequests.WithLabelValues().Inc()
-			continue
-		}
-		nodes[nodeName] = repo.getNode(treeNode, nodeRequest.Expand, nodeRequest.MimeTypes, path, 0, groups, nodeRequest.DataFields, nodeRequest.ExposeHiddenNodes)
-	}
-	return nodes
-}
-
-// GetContent resolves content and fetches nodes in one call. It combines those
-// two tasks for performance reasons.
-//
-// In the first step it uses r.URI to look up content in all given
-// r.Env.Dimensions of repo.Directory.
-//
-// In the second step it collects the requested nodes.
-//
-// those two steps are independent.
-func (repo *Repo) GetContent(r *requests.Content) (c *content.SiteContent, err error) {
-	// add more input validation
-	err = repo.validateContentRequest(r)
-	if err != nil {
-		logger.Log.Error("repo.GetContent invalid request", zap.Error(err))
-		return
-	}
-	logger.Log.Debug("repo.GetContent", zap.String("URI", r.URI))
-	c = content.NewSiteContent()
-	resolved, resolvedURI, resolvedDimension, node := repo.resolveContent(r.Env.Dimensions, r.URI)
-	if resolved {
-		if !node.CanBeAccessedByGroups(r.Env.Groups) {
-			logger.Log.Warn("Resolved content cannot be accessed by specified group", zap.String("URI", r.URI))
-			c.Status = content.StatusForbidden
-		} else {
-			logger.Log.Info("Content resolved", zap.String("URI", r.URI))
-			c.Status = content.StatusOk
-			c.Data = node.Data
-		}
-		c.MimeType = node.MimeType
-		c.Dimension = resolvedDimension
-		c.URI = resolvedURI
-		c.Item = node.ToItem(r.DataFields)
-		c.Path = node.GetPath(r.PathDataFields)
-		// fetch URIs for all dimensions
-		uris := make(map[string]string)
-		for dimensionName := range repo.Directory {
-			uris[dimensionName] = repo.getURI(dimensionName, node.ID)
-		}
-		c.URIs = uris
-	} else {
-		logger.Log.Info("Content not found", zap.String("URI", r.URI))
-		c.Status = content.StatusNotFound
-		c.Dimension = r.Env.Dimensions[0]
-
-		logger.Log.Debug("Failed to resolve, falling back to default dimension",
-			zap.String("URI", r.URI),
-			zap.String("defaultDimension", r.Env.Dimensions[0]),
-		)
-		// r.Env.Dimensions is validated => we can access it
-		resolvedDimension = r.Env.Dimensions[0]
-	}
-
-	// add navigation trees
-	for _, node := range r.Nodes {
-		if node.Dimension == "" {
-			node.Dimension = resolvedDimension
-		}
-	}
-	c.Nodes = repo.getNodes(r.Nodes, r.Env)
-	return c, nil
-}
-
-// GetRepo get the whole repo in all dimensions
-func (repo *Repo) GetRepo() map[string]*content.RepoNode {
-	response := make(map[string]*content.RepoNode)
-	for dimensionName, dimension := range repo.Directory {
-		response[dimensionName] = dimension.Node
-	}
-	return response
-}
-
-// WriteRepoBytes get the whole repo in all dimensions
-// reads the JSON history file from the Filesystem and copies it directly in to the supplied buffer
-// the result is wrapped as service response, e.g: {"reply": <contentData>}
-func (repo *Repo) WriteRepoBytes(w io.Writer) {
-
-	f, err := os.Open(repo.history.getCurrentFilename())
-	if err != nil {
-		logger.Log.Error("Failed to serve Repo JSON", zap.Error(err))
-	}
-
-	_, _ = w.Write([]byte("{\"reply\":"))
-	_, err = io.Copy(w, f)
-	if err != nil {
-		logger.Log.Error("Failed to serve Repo JSON", zap.Error(err))
-	}
-	_, _ = w.Write([]byte("}"))
-}
-
-// Update - reload contents of repository with json from repo.server
-func (repo *Repo) Update() (updateResponse *responses.Update) {
-	floatSeconds := func(nanoSeconds int64) float64 {
-		return float64(float64(nanoSeconds) / float64(1000000000.0))
-	}
-
-	logger.Log.Info("Update triggered")
-	// Log.Info(ansi.Yellow + "BUFFER LENGTH BEFORE tryUpdate(): " + strconv.Itoa(len(repo.jsonBuf.Bytes())) + ansi.Reset)
-
-	startTime := time.Now().UnixNano()
-	updateRepotime, updateErr := repo.tryUpdate()
-	updateResponse = &responses.Update{}
-	updateResponse.Stats.RepoRuntime = floatSeconds(updateRepotime)
-
-	if updateErr != nil {
-		updateResponse.Success = false
-		updateResponse.Stats.NumberOfNodes = -1
-		updateResponse.Stats.NumberOfURIs = -1
-
-		// let us try to restore the world from a file
-		// Log.Info(ansi.Yellow + "BUFFER LENGTH AFTER ERROR: " + strconv.Itoa(len(repo.jsonBuf.Bytes())) + ansi.Reset)
-		// only try to restore if the update failed during processing
-
-		if updateErr != errUpdateRejected {
-			updateResponse.ErrorMessage = updateErr.Error()
-			logger.Log.Error("Failed to update repository", zap.Error(updateErr))
-
-			restoreErr := repo.tryToRestoreCurrent()
-			if restoreErr != nil {
-				logger.Log.Error("Failed to restore preceding repository version", zap.Error(restoreErr))
-			} else {
-				logger.Log.Info("Successfully restored current repository from local history")
-			}
-		}
-	} else {
-		updateResponse.Success = true
-		// persist the currently loaded one
-		historyErr := repo.history.add(repo.jsonBuf.Bytes())
-		if historyErr != nil {
-			logger.Log.Error("Could not persist current repo in history", zap.Error(historyErr))
-			status.M.HistoryPersistFailedCounter.WithLabelValues().Inc()
-		}
-		// add some stats
-		for dimension := range repo.Directory {
-			updateResponse.Stats.NumberOfNodes += len(repo.Directory[dimension].Directory)
-			updateResponse.Stats.NumberOfURIs += len(repo.Directory[dimension].URIDirectory)
-		}
-	}
-	updateResponse.Stats.OwnRuntime = floatSeconds(time.Now().UnixNano()-startTime) - updateResponse.Stats.RepoRuntime
-	return updateResponse
-}
-
-// resolveContent find content in a repository
-func (repo *Repo) resolveContent(dimensions []string, URI string) (resolved bool, resolvedURI string, resolvedDimension string, repoNode *content.RepoNode) {
-	parts := strings.Split(URI, content.PathSeparator)
-	logger.Log.Debug("repo.ResolveContent", zap.String("URI", URI))
-	for i := len(parts); i > 0; i-- {
-		testURI := strings.Join(parts[0:i], content.PathSeparator)
-		if testURI == "" {
-			testURI = content.PathSeparator
-		}
-		for _, dimension := range dimensions {
-			if d, ok := repo.Directory[dimension]; ok {
-				logger.Log.Debug("Checking node",
-					zap.String("dimension", dimension),
-					zap.String("URI", testURI),
-				)
-				if repoNode, ok := d.URIDirectory[testURI]; ok {
-					resolved = true
-					logger.Log.Debug("Node found", zap.String("URI", testURI), zap.String("destination", repoNode.DestinationID))
-					if len(repoNode.DestinationID) > 0 {
-						if destionationNode, destinationNodeOk := d.Directory[repoNode.DestinationID]; destinationNodeOk {
-							repoNode = destionationNode
-						}
-					}
-					return true, testURI, dimension, repoNode
-				}
-			}
-		}
-	}
-	return
-}
-
-func (repo *Repo) getURIForNode(dimension string, repoNode *content.RepoNode, recursionLevel int64) (uri string) {
-	if len(repoNode.LinkID) == 0 {
-		uri = repoNode.URI
-		return
-	}
-	linkedNode, ok := repo.Directory[dimension].Directory[repoNode.LinkID]
-	if ok {
-		if recursionLevel > maxGetURIForNodeRecursionLevel {
-			logger.Log.Error("maxGetURIForNodeRecursionLevel reached", zap.String("repoNode.ID", repoNode.ID), zap.String("linkID", repoNode.LinkID), zap.String("dimension", dimension))
-			return ""
-		}
-		return repo.getURIForNode(dimension, linkedNode, recursionLevel+1)
-	}
-	return
-}
-
-func (repo *Repo) getURI(dimension string, id string) string {
-	repoNode, ok := repo.Directory[dimension].Directory[id]
-	if ok {
-		return repo.getURIForNode(dimension, repoNode, 0)
-	}
-	return ""
-}
-
-func (repo *Repo) getNode(
-	repoNode *content.RepoNode,
-	expanded bool,
-	mimeTypes []string,
-	path []*content.Item,
-	level int,
-	groups []string,
-	dataFields []string,
-	exposeHiddenNodes bool,
-) *content.Node {
-	node := content.NewNode()
-	node.Item = repoNode.ToItem(dataFields)
-	logger.Log.Debug("getNode", zap.String("ID", repoNode.ID))
-	for _, childID := range repoNode.Index {
-		childNode := repoNode.Nodes[childID]
-		if (level == 0 || expanded || !expanded && childNode.InPath(path)) && (!childNode.Hidden || exposeHiddenNodes) && childNode.CanBeAccessedByGroups(groups) && childNode.IsOneOfTheseMimeTypes(mimeTypes) {
-			node.Nodes[childID] = repo.getNode(childNode, expanded, mimeTypes, path, level+1, groups, dataFields, exposeHiddenNodes)
-			node.Index = append(node.Index, childID)
-		}
-	}
-	return node
-}
-
-func (repo *Repo) validateContentRequest(req *requests.Content) (err error) {
-	if req == nil {
-		return errors.New("request must not be nil")
-	}
-	if len(req.URI) == 0 {
-		return errors.New("request URI must not be empty")
-	}
-	if req.Env == nil {
-		return errors.New("request.Env must not be nil")
-	}
-	if len(req.Env.Dimensions) == 0 {
-		return errors.New("request.Env.Dimensions must not be empty")
-	}
-	for _, envDimension := range req.Env.Dimensions {
-		if !repo.hasDimension(envDimension) {
-			availableDimensions := []string{}
-			for availableDimension := range repo.Directory {
-				availableDimensions = append(availableDimensions, availableDimension)
-			}
-			return errors.New(fmt.Sprint(
-				"unknown dimension ", envDimension,
-				" in r.Env must be one of ", availableDimensions,
-				" repo has ", len(repo.Directory), " dimensions",
-			))
-		}
-	}
-	return nil
-}
-
-func (repo *Repo) hasDimension(d string) bool {
-	_, hasDimension := repo.Directory[d]
-	return hasDimension
-}

requests/content.go

@@ -0,0 +1,10 @@
+package requests
+
+// Content - the standard request to contentserver
+type Content struct {
+	Env            *Env             `json:"env"`
+	URI            string           `json:"URI"`
+	Nodes          map[string]*Node `json:"nodes"`
+	DataFields     []string         `json:"dataFields"`
+	PathDataFields []string         `json:"pathDataFields"`
+}

requests/env.go

@@ -0,0 +1,9 @@
+package requests
+
+// Env - abstract your server state
+type Env struct {
+	// when resolving conten these are processed in their order
+	Dimensions []string `json:"dimensions"`
+	// who is it for
+	Groups []string `json:"groups"`
+}

requests/itemmap.go

@@ -0,0 +1,7 @@
+package requests
+
+// ItemMap - map of items
+type ItemMap struct {
+	ID         string   `json:"id"`
+	DataFields []string `json:"dataFields"`
+}

requests/node.go

@@ -0,0 +1,19 @@
+package requests
+
+// Node - an abstract node request, use this one to request navigations
+type Node struct {
+	// this one should be obvious
+	ID string `json:"id"`
+	// from which dimension
+	Dimension string `json:"dimension"`
+	// allowed access groups
+	Groups []string `json:"groups"`
+	// what do you want to see in your navigations, folders, images or unicorns
+	MimeTypes []string `json:"mimeTypes"`
+	// expand the navigation tree or just the path to the resolved content
+	Expand bool `json:"expand"`
+	// Expose hidden nodes
+	ExposeHiddenNodes bool `json:"exposeHiddenNodes,omitempty"`
+	// filter with these
+	DataFields []string `json:"dataFields"`
+}

requests/nodes.go

@@ -0,0 +1,8 @@
+package requests
+
+// Nodes - which nodes in which dimensions
+type Nodes struct {
+	// map[dimension]*node
+	Nodes map[string]*Node `json:"nodes"`
+	Env   *Env             `json:"env"`
+}

requests/repo.go

@@ -0,0 +1,4 @@
+package requests
+
+// Repo - query repo
+type Repo struct{}

requests/requests.go

@@ -1,63 +0,0 @@
-package requests
-
-// Env - abstract your server state
-type Env struct {
-	// when resolving conten these are processed in their order
-	Dimensions []string `json:"dimensions"`
-	// who is it for
-	Groups []string `json:"groups"`
-}
-
-// Node - an abstract node request, use this one to request navigations
-type Node struct {
-	// this one should be obvious
-	ID string `json:"id"`
-	// from which dimension
-	Dimension string `json:"dimension"`
-	// allowed access groups
-	Groups []string `json:"groups"`
-	// what do you want to see in your navigations, folders, images or unicorns
-	MimeTypes []string `json:"mimeTypes"`
-	// expand the navigation tree or just the path to the resolved content
-	Expand bool `json:"expand"`
-	// Expose hidden nodes
-	ExposeHiddenNodes bool `json:"exposeHiddenNodes,omitempty"`
-	// filter with these
-	DataFields []string `json:"dataFields"`
-}
-
-// Nodes - which nodes in which dimensions
-type Nodes struct {
-	// map[dimension]*node
-	Nodes map[string]*Node `json:"nodes"`
-
-	Env *Env `json:"env"`
-}
-
-// Content - the standard request to contentserver
-type Content struct {
-	Env            *Env             `json:"env"`
-	URI            string           `json:"URI"`
-	Nodes          map[string]*Node `json:"nodes"`
-	DataFields     []string         `json:"dataFields"`
-	PathDataFields []string         `json:"pathDataFields"`
-}
-
-// Update - request an update
-type Update struct{}
-
-// Repo - query repo
-type Repo struct{}
-
-// ItemMap - map of items
-type ItemMap struct {
-	ID         string   `json:"id"`
-	DataFields []string `json:"dataFields"`
-}
-
-// URIs - request multiple URIs for a dimension use this resolve uris for links
-// in a document
-type URIs struct {
-	IDs       []string `json:"ids"`
-	Dimension string   `json:"dimension"`
-}

requests/update.go

@@ -0,0 +1,4 @@
+package requests
+
+// Update - request an update
+type Update struct{}

requests/uris.go

@@ -0,0 +1,8 @@
+package requests
+
+// URIs - request multiple URIs for a dimension use this resolve uris for links
+// in a document
+type URIs struct {
+	IDs       []string `json:"ids"`
+	Dimension string   `json:"dimension"`
+}

responses/error.go

@@ -1,6 +1,8 @@
 package responses
 
-import "fmt"
+import (
+	"fmt"
+)
 
 // Error describes an error for humans and machines
 type Error struct {
@@ -21,19 +23,3 @@ func NewError(code int, message string) *Error {
 		Message: message,
 	}
 }
-
-// Update - information about an update
-type Update struct {
-	// did it work or not
-	Success bool `json:"success"`
-	// this is for humand
-	ErrorMessage string `json:"errorMessage"`
-	Stats        struct {
-		NumberOfNodes int `json:"numberOfNodes"`
-		NumberOfURIs  int `json:"numberOfURIs"`
-		// seconds
-		RepoRuntime float64 `json:"repoRuntime"`
-		// seconds
-		OwnRuntime float64 `json:"ownRuntime"`
-	} `json:"stats"`
-}

responses/stats.go

@@ -0,0 +1,10 @@
+package responses
+
+type Stats struct {
+	NumberOfNodes int `json:"numberOfNodes"`
+	NumberOfURIs  int `json:"numberOfURIs"`
+	// seconds
+	RepoRuntime float64 `json:"repoRuntime"`
+	// seconds
+	OwnRuntime float64 `json:"ownRuntime"`
+}

responses/update.go

@@ -0,0 +1,10 @@
+package responses
+
+// Update - information about an update
+type Update struct {
+	// did it work or not
+	Success bool `json:"success"`
+	// this is for humans
+	ErrorMessage string `json:"errorMessage"`
+	Stats        Stats  `json:"stats"`
+}

server/handlerequest.go

@@ -1,97 +0,0 @@
-package server
-
-import (
-	"time"
-
-	"go.uber.org/zap"
-
-	. "github.com/foomo/contentserver/logger"
-	"github.com/foomo/contentserver/repo"
-	"github.com/foomo/contentserver/requests"
-	"github.com/foomo/contentserver/responses"
-	"github.com/foomo/contentserver/status"
-)
-
-func handleRequest(r *repo.Repo, handler Handler, jsonBytes []byte, source string) ([]byte, error) {
-	start := time.Now()
-
-	reply, err := executeRequest(r, handler, jsonBytes, source)
-	result := "success"
-	if err != nil {
-		result = "error"
-	}
-
-	status.M.ServiceRequestCounter.WithLabelValues(string(handler), result, source).Inc()
-	status.M.ServiceRequestDuration.WithLabelValues(string(handler), result, source).Observe(time.Since(start).Seconds())
-
-	return reply, err
-}
-
-func executeRequest(r *repo.Repo, handler Handler, jsonBytes []byte, source string) (replyBytes []byte, err error) {
-
-	var (
-		reply             interface{}
-		apiErr            error
-		jsonErr           error
-		processIfJSONIsOk = func(err error, processingFunc func()) {
-			if err != nil {
-				jsonErr = err
-				return
-			}
-			processingFunc()
-		}
-	)
-	status.M.ContentRequestCounter.WithLabelValues(source).Inc()
-
-	// handle and process
-	switch handler {
-	// case HandlerGetRepo: // This case is handled prior to handleRequest being called.
-	// since the resulting bytes are written directly in to the http.ResponseWriter / net.Connection
-	case HandlerGetURIs:
-		getURIRequest := &requests.URIs{}
-		processIfJSONIsOk(json.Unmarshal(jsonBytes, &getURIRequest), func() {
-			reply = r.GetURIs(getURIRequest.Dimension, getURIRequest.IDs)
-		})
-	case HandlerGetContent:
-		contentRequest := &requests.Content{}
-		processIfJSONIsOk(json.Unmarshal(jsonBytes, &contentRequest), func() {
-			reply, apiErr = r.GetContent(contentRequest)
-		})
-	case HandlerGetNodes:
-		nodesRequest := &requests.Nodes{}
-		processIfJSONIsOk(json.Unmarshal(jsonBytes, &nodesRequest), func() {
-			reply = r.GetNodes(nodesRequest)
-		})
-	case HandlerUpdate:
-		updateRequest := &requests.Update{}
-		processIfJSONIsOk(json.Unmarshal(jsonBytes, &updateRequest), func() {
-			reply = r.Update()
-		})
-
-	default:
-		reply = responses.NewError(1, "unknown handler: "+string(handler))
-	}
-
-	// error handling
-	if jsonErr != nil {
-		Log.Error("could not read incoming json", zap.Error(jsonErr))
-		reply = responses.NewError(2, "could not read incoming json "+jsonErr.Error())
-	} else if apiErr != nil {
-		Log.Error("an API error occured", zap.Error(apiErr))
-		reply = responses.NewError(3, "internal error "+apiErr.Error())
-	}
-
-	return encodeReply(reply)
-}
-
-// encodeReply takes an interface and encodes it as JSON
-// it returns the resulting JSON and a marshalling error
-func encodeReply(reply interface{}) (replyBytes []byte, err error) {
-	replyBytes, err = json.Marshal(map[string]interface{}{
-		"reply": reply,
-	})
-	if err != nil {
-		Log.Error("could not encode reply", zap.Error(err))
-	}
-	return
-}

server/server.go

@@ -1,142 +0,0 @@
-package server
-
-import (
-	"errors"
-	"fmt"
-	"net"
-	"net/http"
-	"os"
-	"time"
-
-	. "github.com/foomo/contentserver/logger"
-	"github.com/foomo/contentserver/repo"
-	jsoniter "github.com/json-iterator/go"
-	"go.uber.org/zap"
-)
-
-var (
-	json = jsoniter.ConfigCompatibleWithStandardLibrary
-)
-
-// Handler type
-type Handler string
-
-const (
-	// HandlerGetURIs get uris, many at once, to keep it fast
-	HandlerGetURIs Handler = "getURIs"
-	// HandlerGetContent get (site) content
-	HandlerGetContent = "getContent"
-	// HandlerGetNodes get nodes
-	HandlerGetNodes = "getNodes"
-	// HandlerUpdate update repo
-	HandlerUpdate = "update"
-	// HandlerGetRepo get the whole repo
-	HandlerGetRepo = "getRepo"
-
-	// DefaultRepositoryTimeout for the HTTP client towards the repo
-	DefaultRepositoryTimeout = 2 * time.Minute
-)
-
-// Run - let it run and enjoy on a socket near you
-func Run(server string, address string, varDir string, pollUpdates bool) error {
-	return RunServerSocketAndWebServer(server, address, "", "", varDir, DefaultRepositoryTimeout, pollUpdates)
-}
-
-func RunServerSocketAndWebServer(
-	server string,
-	address string,
-	webserverAddress string,
-	webserverPath string,
-	varDir string,
-	repositoryTimeout time.Duration,
-	pollForUpdates bool,
-) error {
-	if address == "" && webserverAddress == "" {
-		return errors.New("one of the addresses needs to be set")
-	}
-	Log.Info("building repo with content", zap.String("server", server))
-
-	r := repo.NewRepo(server, varDir, repositoryTimeout, pollForUpdates)
-
-	// start initial update and handle error
-	if !pollForUpdates {
-		go func() {
-			resp := r.Update()
-			if !resp.Success {
-				Log.Error("failed to update",
-					zap.String("error", resp.ErrorMessage),
-					zap.Int("NumberOfNodes", resp.Stats.NumberOfNodes),
-					zap.Int("NumberOfURIs", resp.Stats.NumberOfURIs),
-					zap.Float64("OwnRuntime", resp.Stats.OwnRuntime),
-					zap.Float64("RepoRuntime", resp.Stats.RepoRuntime),
-				)
-
-				//Exit only if it hasn't recovered
-				if !r.Recovered() {
-					os.Exit(1)
-				}
-			}
-
-		}()
-	}
-
-	// update can run in bg
-	chanErr := make(chan error)
-
-	if address != "" {
-		Log.Info("starting socketserver", zap.String("address", address))
-		go runSocketServer(r, address, chanErr)
-	}
-	if webserverAddress != "" {
-		Log.Info("starting webserver", zap.String("webserverAddress", webserverAddress))
-		go runWebserver(r, webserverAddress, webserverPath, chanErr)
-	}
-	return <-chanErr
-}
-
-func runWebserver(
-	r *repo.Repo,
-	address string,
-	path string,
-	chanErr chan error,
-) {
-	chanErr <- http.ListenAndServe(address, NewWebServer(path, r))
-}
-
-func runSocketServer(
-	repo *repo.Repo,
-	address string,
-	chanErr chan error,
-) {
-	// create socket server
-	s := newSocketServer(repo)
-
-	// listen on socket
-	ln, errListen := net.Listen("tcp", address)
-	if errListen != nil {
-		Log.Error("runSocketServer: could not start",
-			zap.String("address", address),
-			zap.Error(errListen),
-		)
-		chanErr <- errors.New("runSocketServer: could not start the on \"" + address + "\" - error: " + fmt.Sprint(errListen))
-		return
-	}
-
-	Log.Info("runSocketServer: started listening", zap.String("address", address))
-	for {
-		// this blocks until connection or error
-		conn, err := ln.Accept()
-		if err != nil {
-			Log.Error("runSocketServer: could not accept connection", zap.Error(err))
-			continue
-		}
-
-		// a goroutine handles conn so that the loop can accept other connections
-		go func() {
-			Log.Debug("accepted connection", zap.String("source", conn.RemoteAddr().String()))
-			s.handleConnection(conn)
-			conn.Close()
-			// log.Debug("connection closed")
-		}()
-	}
-}

server/socketserver.go

@@ -1,167 +0,0 @@
-package server
-
-import (
-	"bytes"
-	"errors"
-	"fmt"
-	"net"
-	"strconv"
-	"strings"
-
-	"go.uber.org/zap"
-
-	. "github.com/foomo/contentserver/logger"
-	"github.com/foomo/contentserver/repo"
-	"github.com/foomo/contentserver/responses"
-	"github.com/foomo/contentserver/status"
-)
-
-const sourceSocketServer = "socketserver"
-
-type socketServer struct {
-	repo *repo.Repo
-}
-
-// newSocketServer returns a shiny new socket server
-func newSocketServer(repo *repo.Repo) *socketServer {
-	return &socketServer{
-		repo: repo,
-	}
-}
-
-func extractHandlerAndJSONLentgh(header string) (handler Handler, jsonLength int, err error) {
-	headerParts := strings.Split(header, ":")
-	if len(headerParts) != 2 {
-		return "", 0, errors.New("invalid header")
-	}
-	jsonLength, err = strconv.Atoi(headerParts[1])
-	if err != nil {
-		err = fmt.Errorf("could not parse length in header: %q", header)
-	}
-	return Handler(headerParts[0]), jsonLength, err
-}
-
-func (s *socketServer) execute(handler Handler, jsonBytes []byte) (reply []byte) {
-	Log.Debug("incoming json buffer", zap.Int("length", len(jsonBytes)))
-
-	if handler == HandlerGetRepo {
-		var (
-			b bytes.Buffer
-		)
-		s.repo.WriteRepoBytes(&b)
-		return b.Bytes()
-	}
-
-	reply, handlingError := handleRequest(s.repo, handler, jsonBytes, sourceSocketServer)
-	if handlingError != nil {
-		Log.Error("socketServer.execute failed", zap.Error(handlingError))
-	}
-	return reply
-}
-
-func (s *socketServer) writeResponse(conn net.Conn, reply []byte) {
-	headerBytes := []byte(strconv.Itoa(len(reply)))
-	reply = append(headerBytes, reply...)
-	Log.Debug("replying", zap.String("reply", string(reply)))
-	n, writeError := conn.Write(reply)
-	if writeError != nil {
-		Log.Error("socketServer.writeResponse: could not write reply", zap.Error(writeError))
-		return
-	}
-	if n < len(reply) {
-		Log.Error("socketServer.writeResponse: write too short",
-			zap.Int("got", n),
-			zap.Int("expected", len(reply)),
-		)
-		return
-	}
-	Log.Debug("replied. waiting for next request on open connection")
-}
-
-func (s *socketServer) handleConnection(conn net.Conn) {
-	defer func() {
-		if r := recover(); r != nil {
-			Log.Error("panic in handle connection", zap.String("error", fmt.Sprint(r)))
-		}
-	}()
-
-	Log.Debug("socketServer.handleConnection")
-	status.M.NumSocketsGauge.WithLabelValues(conn.RemoteAddr().String()).Inc()
-
-	var (
-		headerBuffer [1]byte
-		header       = ""
-		i            = 0
-	)
-	for {
-		i++
-		// fmt.Println("---->", i)
-		// let us read with 1 byte steps on conn until we find "{"
-		_, readErr := conn.Read(headerBuffer[0:])
-		if readErr != nil {
-			Log.Debug("looks like the client closed the connection", zap.Error(readErr))
-			status.M.NumSocketsGauge.WithLabelValues(conn.RemoteAddr().String()).Dec()
-			return
-		}
-		// read next byte
-		current := headerBuffer[0:]
-		if string(current) == "{" {
-			// json has started
-			handler, jsonLength, headerErr := extractHandlerAndJSONLentgh(header)
-			// reset header
-			header = ""
-			if headerErr != nil {
-				Log.Error("invalid request could not read header", zap.Error(headerErr))
-				encodedErr, encodingErr := encodeReply(responses.NewError(4, "invalid header "+headerErr.Error()))
-				if encodingErr == nil {
-					s.writeResponse(conn, encodedErr)
-				} else {
-					Log.Error("could not respond to invalid request", zap.Error(encodingErr))
-				}
-				return
-			}
-			Log.Debug("found json", zap.Int("length", jsonLength))
-			if jsonLength > 0 {
-
-				var (
-					// let us try to read some json
-					jsonBytes         = make([]byte, jsonLength)
-					jsonLengthCurrent = 1
-					readRound         = 0
-				)
-
-				// that is "{"
-				jsonBytes[0] = 123
-
-				for jsonLengthCurrent < jsonLength {
-					readRound++
-					readLength, jsonReadErr := conn.Read(jsonBytes[jsonLengthCurrent:jsonLength])
-					if jsonReadErr != nil {
-						//@fixme we need to force a read timeout (SetReadDeadline?), if expected jsonLength is lower than really sent bytes (e.g. if client implements protocol wrong)
-						//@todo should we check for io.EOF here
-						Log.Error("could not read json - giving up with this client connection", zap.Error(jsonReadErr))
-						status.M.NumSocketsGauge.WithLabelValues(conn.RemoteAddr().String()).Dec()
-						return
-					}
-					jsonLengthCurrent += readLength
-					Log.Debug("read cycle status",
-						zap.Int("jsonLengthCurrent", jsonLengthCurrent),
-						zap.Int("jsonLength", jsonLength),
-						zap.Int("readRound", readRound),
-					)
-				}
-
-				Log.Debug("read json", zap.Int("length", len(jsonBytes)))
-
-				s.writeResponse(conn, s.execute(handler, jsonBytes))
-				// note: connection remains open
-				continue
-			}
-			Log.Error("can not read empty json")
-			status.M.NumSocketsGauge.WithLabelValues(conn.RemoteAddr().String()).Dec()
-			return
-		}
-		// adding to header byte by byte
-		header += string(headerBuffer[0:])
-	}
-}

server/webserver.go

@@ -1,59 +0,0 @@
-package server
-
-import (
-	"fmt"
-	"io/ioutil"
-	"net/http"
-	"strings"
-
-	"go.uber.org/zap"
-
-	. "github.com/foomo/contentserver/logger"
-	"github.com/foomo/contentserver/repo"
-)
-
-type webServer struct {
-	path string
-	r    *repo.Repo
-}
-
-// NewWebServer returns a shiny new web server
-func NewWebServer(path string, r *repo.Repo) http.Handler {
-	return &webServer{
-		path: path,
-		r:    r,
-	}
-}
-
-func (s *webServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	defer func() {
-		if r := recover(); r != nil {
-			Log.Error("Panic in handle connection", zap.String("error", fmt.Sprint(r)))
-		}
-	}()
-
-	if r.Body == nil {
-		http.Error(w, "no body", http.StatusBadRequest)
-		return
-	}
-	jsonBytes, readErr := ioutil.ReadAll(r.Body)
-	if readErr != nil {
-		http.Error(w, "failed to read incoming request", http.StatusBadRequest)
-		return
-	}
-	h := Handler(strings.TrimPrefix(r.URL.Path, s.path+"/"))
-	if h == HandlerGetRepo {
-		s.r.WriteRepoBytes(w)
-		w.Header().Set("Content-Type", "application/json")
-		return
-	}
-	reply, errReply := handleRequest(s.r, h, jsonBytes, "webserver")
-	if errReply != nil {
-		http.Error(w, errReply.Error(), http.StatusInternalServerError)
-		return
-	}
-	_, err := w.Write(reply)
-	if err != nil {
-		Log.Error("failed to write webServer reply", zap.Error(err))
-	}
-}

status/healthz.go

@@ -1,37 +0,0 @@
-package status
-
-import (
-	"fmt"
-	"net/http"
-
-	. "github.com/foomo/contentserver/logger"
-	jsoniter "github.com/json-iterator/go"
-	"go.uber.org/zap"
-)
-
-var (
-	json = jsoniter.ConfigCompatibleWithStandardLibrary
-)
-
-func RunHealthzHandlerListener(address string, serviceName string) {
-	Log.Info(fmt.Sprintf("starting healthz handler on '%s'" + address))
-	Log.Error("healthz server failed", zap.Error(http.ListenAndServe(address, HealthzHandler(serviceName))))
-}
-
-func HealthzHandler(serviceName string) http.Handler {
-	var (
-		data = map[string]string{
-			"service": serviceName,
-		}
-		status, _ = json.Marshal(data)
-		h         = http.NewServeMux()
-	)
-	h.Handle("/healthz", http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
-		_, err := w.Write(status)
-		if err != nil {
-			Log.Error("failed to write healthz status", zap.Error(err))
-		}
-	}))
-
-	return h
-}

status/metrics.go

@@ -1,119 +0,0 @@
-package status
-
-import (
-	"github.com/prometheus/client_golang/prometheus"
-)
-
-// M is the Metrics instance
-var M = newMetrics()
-
-const (
-	namespace = "contentserver"
-
-	metricLabelHandler = "handler"
-	metricLabelStatus  = "status"
-	metricLabelSource  = "source"
-	metricLabelRemote  = "remote"
-)
-
-// Metrics is the structure that holds all prometheus metrics
-type Metrics struct {
-	ServiceRequestCounter       *prometheus.CounterVec // count the number of requests for each service function
-	ServiceRequestDuration      *prometheus.SummaryVec // observe the duration of requests for each service function
-	UpdatesRejectedCounter      *prometheus.CounterVec // count the number of completed updates
-	UpdatesCompletedCounter     *prometheus.CounterVec // count the number of rejected updates
-	UpdatesFailedCounter        *prometheus.CounterVec // count the number of updates that had an error
-	UpdateDuration              *prometheus.SummaryVec // observe the duration of each repo.update() call
-	ContentRequestCounter       *prometheus.CounterVec // count the total number of content requests
-	NumSocketsGauge             *prometheus.GaugeVec   // keep track of the total number of open sockets
-	HistoryPersistFailedCounter *prometheus.CounterVec // count the number of failed attempts to persist the content history
-	InvalidNodeTreeRequests     *prometheus.CounterVec // counts the number of invalid tree node requests
-}
-
-// newMetrics can be used to instantiate a metrics instance
-// since this function will also register each metric and metrics should only be registered once
-// it is private
-// the package exposes the initialized Metrics instance as the variable M.
-func newMetrics() *Metrics {
-	return &Metrics{
-		InvalidNodeTreeRequests: newCounterVec("invalid_node_tree_request_count",
-			"Counts the number of invalid tree nodes for a specific node"),
-		ServiceRequestCounter: newCounterVec(
-			"service_request_count",
-			"Count of requests for each handler",
-			metricLabelHandler, metricLabelStatus, metricLabelSource,
-		),
-		ServiceRequestDuration: newSummaryVec(
-			"service_request_duration_seconds",
-			"Seconds to unmarshal requests, execute a service function and marshal its reponses",
-			metricLabelHandler, metricLabelStatus, metricLabelSource,
-		),
-		UpdatesRejectedCounter: newCounterVec(
-			"updates_rejected_count",
-			"Number of updates that were rejected because the queue was full",
-		),
-		UpdatesCompletedCounter: newCounterVec(
-			"updates_completed_count",
-			"Number of updates that were successfully completed",
-		),
-		UpdatesFailedCounter: newCounterVec(
-			"updates_failed_count",
-			"Number of updates that failed due to an error",
-		),
-		UpdateDuration: newSummaryVec(
-			"update_duration_seconds",
-			"Duration in seconds for each successful repo.update() call",
-		),
-		ContentRequestCounter: newCounterVec(
-			"content_request_count",
-			"Number of requests for content",
-			metricLabelSource,
-		),
-		NumSocketsGauge: newGaugeVec(
-			"num_sockets_total",
-			"Total number of currently open socket connections",
-			metricLabelRemote,
-		),
-		HistoryPersistFailedCounter: newCounterVec(
-			"history_persist_failed_count",
-			"Number of failures to store the content history on the filesystem",
-		),
-	}
-}
-
-/*
- *	Metric constructors
- */
-
-func newSummaryVec(name, help string, labels ...string) *prometheus.SummaryVec {
-	vec := prometheus.NewSummaryVec(
-		prometheus.SummaryOpts{
-			Namespace: namespace,
-			Name:      name,
-			Help:      help,
-		}, labels)
-	prometheus.MustRegister(vec)
-	return vec
-}
-
-func newCounterVec(name, help string, labels ...string) *prometheus.CounterVec {
-	vec := prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Namespace: namespace,
-			Name:      name,
-			Help:      help,
-		}, labels)
-	prometheus.MustRegister(vec)
-	return vec
-}
-
-func newGaugeVec(name, help string, labels ...string) *prometheus.GaugeVec {
-	vec := prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Namespace: namespace,
-			Name:      name,
-			Help:      help,
-		}, labels)
-	prometheus.MustRegister(vec)
-	return vec
-}

testing/client/client.go

@@ -1,57 +0,0 @@
-package main
-
-import (
-	"flag"
-	"log"
-	"time"
-
-	"github.com/foomo/contentserver/client"
-)
-
-var (
-	flagAddr    = flag.String("addr", "http://127.0.0.1:9191/contentserver", "set addr")
-	flagGetRepo = flag.Bool("getRepo", false, "get repo")
-	flagUpdate  = flag.Bool("update", true, "trigger content update")
-	flagNum     = flag.Int("num", 100, "num repititions")
-	flagDelay   = flag.Int("delay", 2, "delay in seconds")
-)
-
-func main() {
-
-	flag.Parse()
-
-	c, errClient := client.NewHTTPClient(*flagAddr)
-	if errClient != nil {
-		log.Fatal(errClient)
-	}
-
-	for i := 1; i <= *flagNum; i++ {
-
-		if *flagUpdate {
-			go func(num int) {
-				log.Println("start update")
-				resp, errUpdate := c.Update()
-				if errUpdate != nil {
-					log.Fatal(errUpdate)
-				}
-				log.Println(num, "update done", resp)
-			}(i)
-		}
-
-		if *flagGetRepo {
-			go func(num int) {
-				log.Println("GetRepo", num)
-				resp, err := c.GetRepo()
-				if err != nil {
-					// spew.Dump(resp)
-					log.Fatal("failed to get repo")
-				}
-				log.Println(num, "GetRepo done, got", len(resp), "dimensions")
-			}(i)
-		}
-
-		time.Sleep(time.Duration(*flagDelay) * time.Second)
-	}
-
-	log.Println("done!")
-}

testing/server/server.go

@@ -1,35 +0,0 @@
-package main
-
-import (
-	"flag"
-	"log"
-	"net/http"
-)
-
-type testServer struct {
-	file string
-}
-
-func main() {
-
-	var (
-		flagJSONFile = flag.String("json-file", "", "provide a json source file")
-		flagAddress  = flag.String("addr", ":1234", "set the webserver address")
-	)
-	flag.Parse()
-
-	if *flagJSONFile == "" {
-		log.Fatal("js source file must be provided")
-	}
-
-	ts := &testServer{
-		file: *flagJSONFile,
-	}
-
-	log.Println("start test server at", *flagAddress, "serving file:", ts.file)
-	log.Fatal(http.ListenAndServe(*flagAddress, ts))
-}
-
-func (ts *testServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
-	http.ServeFile(w, r, ts.file)
-}