diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..0977931 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,12 @@ +root = true + +[*] +charset = utf-8 +end_of_line = lf +indent_size = 2 +indent_style = tab +insert_final_newline = true +trim_trailing_whitespace = true + +[*.{yaml,yml,md,mdx}] +indent_style = space diff --git a/.gitignore b/.gitignore index 197a383..e99ac8f 100644 --- a/.gitignore +++ b/.gitignore @@ -1,11 +1,14 @@ -data -*.log -*.test -cprof-* -var .* -*~ +*.log +!.github/ +!.husky/ +!.editorconfig +!.gitignore +!.golangci.yml +!.goreleaser.yml +!.husky.yaml +!.yamllint.yaml /bin/ -/pkg/tmp/ -/vendor -!.git* +/coverage.out +/coverage.html +/tmp/ diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..5070508 --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,58 @@ +run: + timeout: 5m + +linters-settings: + gocritic: + disabled-checks: + - ifElseChain + - commentFormatting + +linters: + enable: + # Enabled by default linters: + - errcheck + - gosimple + - govet + - ineffassign + - staticcheck + + # Disabled by default linters: + - asciicheck # Simple linter to check that your code does not contain non-ASCII identifiers [fast: true, auto-fix: false] + - bidichk # Checks for dangerous unicode character sequences [fast: true, auto-fix: false] + #- dupl # Tool for code clone detection [fast: true, auto-fix: false] + - forcetypeassert # finds forced type assertions [fast: true, auto-fix: false] + #- gochecknoinits # Checks that no init functions are present in Go code [fast: true, auto-fix: false] + #- goconst # Finds repeated strings that could be replaced by a constant [fast: true, auto-fix: false] + - gocritic # Provides diagnostics that check for bugs, performance and style issues. [fast: false, auto-fix: false] + - goimports # In addition to fixing imports, goimports also formats your code in the same style as gofmt. [fast: true, auto-fix: true] + #- gomoddirectives # Manage the use of 'replace', 'retract', and 'excludes' directives in go.mod. [fast: true, auto-fix: false] + - gomodguard # Allow and block list linter for direct Go module dependencies. This is different from depguard where there are different block types for example version constraints and module recommendations. [fast: true, auto-fix: false] + - goprintffuncname # Checks that printf-like functions are named with `f` at the end [fast: true, auto-fix: false] + - gosec # (gas): Inspects source code for security problems [fast: false, auto-fix: false] + - grouper # An analyzer to analyze expression groups. [fast: true, auto-fix: false] + - importas # Enforces consistent import aliases [fast: false, auto-fix: false] + #- maintidx # maintidx measures the maintainability index of each function. [fast: true, auto-fix: false] + #- makezero # Finds slice declarations with non-zero initial length [fast: false, auto-fix: false] + - misspell # Finds commonly misspelled English words in comments [fast: true, auto-fix: true] + - nakedret # Finds naked returns in functions greater than a specified function length [fast: true, auto-fix: false] + #- nestif # Reports deeply nested if statements [fast: true, auto-fix: false] + - nilerr # Finds the code that returns nil even if it checks that the error is not nil. [fast: false, auto-fix: false] + - nilnil # Checks that there is no simultaneous return of `nil` error and an invalid value. [fast: false, auto-fix: false] + - noctx # noctx finds sending http request without context.Context [fast: false, auto-fix: false] + - nolintlint # Reports ill-formed or insufficient nolint directives [fast: true, auto-fix: false] + #- nonamedreturns # Reports all named returns [fast: false, auto-fix: false] + - nosprintfhostport # Checks for misuse of Sprintf to construct a host with port in a URL. [fast: true, auto-fix: false] + - prealloc # Finds slice declarations that could potentially be pre-allocated [fast: true, auto-fix: false] + - predeclared # find code that shadows one of Go's predeclared identifiers [fast: true, auto-fix: false] + - promlinter # Check Prometheus metrics naming via promlint [fast: true, auto-fix: false] + #- revive # Fast, configurable, extensible, flexible, and beautiful linter for Go. Drop-in replacement of golint. [fast: false, auto-fix: false] + #- tagliatelle # Checks the struct tags. [fast: true, auto-fix: false] + #- testpackage # linter that makes you use a separate _test package [fast: true, auto-fix: false] + - thelper # thelper detects golang test helpers without t.Helper() call and checks the consistency of test helpers [fast: false, auto-fix: false] + - unconvert # Remove unnecessary type conversions [fast: false, auto-fix: false] + - unparam # Reports unused function parameters [fast: false, auto-fix: false] + - usestdlibvars # A linter that detect the possibility to use variables/constants from the Go standard library. [fast: true, auto-fix: false] + - wastedassign # wastedassign finds wasted assignment statements. [fast: false, auto-fix: false] + - whitespace # Tool for detection of leading and trailing whitespace [fast: true, auto-fix: true] + disable: + - unused diff --git a/.goreleaser.yml b/.goreleaser.yml new file mode 100644 index 0000000..618358e --- /dev/null +++ b/.goreleaser.yml @@ -0,0 +1,165 @@ +project_name: contentserver + +release: + github: + owner: foomo + name: contentserver + prerelease: auto + +builds: + - binary: contentserver + goos: + - windows + - darwin + - linux + goarch: + - amd64 + - arm64 + goarm: + - 7 + env: + - CGO_ENABLED=0 + main: ./main.go + flags: + - -trimpath + ldflags: -s -w -X github.com/foomo/contentserver/cmd.version={{.Version}} + +archives: + - format: tar.gz + format_overrides: + - goos: windows + format: zip + files: + - LICENSE + - README.md + +changelog: + use: github-native + +brews: + # Repository to push the tap to. + - repository: + owner: foomo + name: homebrew-tap + caveats: "sesamy --help" + homepage: "https://github.com/foomo/contentserver" + description: "Serves content tree structures very quickly" + test: | + system "#{bin}/contentserver --version" + +docker_manifests: + # basic + - name_template: 'foomo/contentserver:latest' + image_templates: + - 'foomo/contentserver:{{ .Tag }}-amd64' + - 'foomo/contentserver:{{ .Tag }}-arm64' + + - name_template: 'foomo/contentserver:v{{ .Major }}.{{ .Minor }}' + image_templates: + - 'foomo/contentserver:v{{ .Major }}.{{ .Minor }}-amd64' + - 'foomo/contentserver:v{{ .Major }}.{{ .Minor }}-arm64' + + - name_template: 'foomo/contentserver:{{ .Tag }}' + image_templates: + - 'foomo/contentserver:{{ .Tag }}-amd64' + - 'foomo/contentserver:{{ .Tag }}-arm64' + + # alpine + - name_template: 'foomo/contentserver:latest-alpine' + image_templates: + - 'foomo/contentserver:{{ .Tag }}-alpine-amd64' + - 'foomo/contentserver:{{ .Tag }}-alpine-arm64' + + - name_template: 'foomo/contentserver:v{{ .Major }}.{{ .Minor }}-alpine' + image_templates: + - 'foomo/contentserver:v{{ .Major }}.{{ .Minor }}-alpine-amd64' + - 'foomo/contentserver:v{{ .Major }}.{{ .Minor }}-alpine-arm64' + + - name_template: 'foomo/contentserver:{{ .Tag }}-alpine' + image_templates: + - 'foomo/contentserver:{{ .Tag }}-alpine-amd64' + - 'foomo/contentserver:{{ .Tag }}-alpine-arm64' +dockers: + - use: buildx + goos: linux + goarch: amd64 + dockerfile: build/buildx.Dockerfile + image_templates: + - 'foomo/contentserver:latest-amd64' + - 'foomo/contentserver:{{ .Tag }}-amd64' + - 'foomo/contentserver:v{{ .Major }}.{{ .Minor }}-amd64' + build_flag_templates: + - '--pull' + # https://github.com/opencontainers/image-spec/blob/main/annotations.md#pre-defined-annotation-keys + - '--label=org.opencontainers.image.title={{.ProjectName}}' + - '--label=org.opencontainers.image.description=Serves content tree structures very quickly' + - '--label=org.opencontainers.image.source={{.GitURL}}' + - '--label=org.opencontainers.image.url={{.GitURL}}' + - '--label=org.opencontainers.image.documentation={{.GitURL}}' + - '--label=org.opencontainers.image.created={{.Date}}' + - '--label=org.opencontainers.image.revision={{.FullCommit}}' + - '--label=org.opencontainers.image.version={{.Version}}' + - '--platform=linux/amd64' + + - use: buildx + goos: linux + goarch: arm64 + dockerfile: build/buildx.Dockerfile + image_templates: + - 'foomo/contentserver:latest-arm64' + - 'foomo/contentserver:{{ .Tag }}-arm64' + - 'foomo/contentserver:v{{ .Major }}.{{ .Minor }}-arm64' + build_flag_templates: + - '--pull' + # https://github.com/opencontainers/image-spec/blob/main/annotations.md#pre-defined-annotation-keys + - '--label=org.opencontainers.image.title={{.ProjectName}}' + - '--label=org.opencontainers.image.description=Serves content tree structures very quickly' + - '--label=org.opencontainers.image.source={{.GitURL}}' + - '--label=org.opencontainers.image.url={{.GitURL}}' + - '--label=org.opencontainers.image.documentation={{.GitURL}}' + - '--label=org.opencontainers.image.created={{.Date}}' + - '--label=org.opencontainers.image.revision={{.FullCommit}}' + - '--label=org.opencontainers.image.version={{.Version}}' + - '--platform=linux/arm64' + + - use: buildx + goos: linux + goarch: amd64 + dockerfile: build/buildx-alpine.Dockerfile + image_templates: + - 'foomo/contentserver:latest-alpine-amd64' + - 'foomo/contentserver:{{ .Tag }}-alpine-amd64' + - 'foomo/contentserver:v{{ .Major }}.{{ .Minor }}-alpine-amd64' + build_flag_templates: + - '--pull' + # https://github.com/opencontainers/image-spec/blob/main/annotations.md#pre-defined-annotation-keys + - '--label=org.opencontainers.image.title={{.ProjectName}}' + - '--label=org.opencontainers.image.description=Serves content tree structures very quickly' + - '--label=org.opencontainers.image.source={{.GitURL}}' + - '--label=org.opencontainers.image.url={{.GitURL}}' + - '--label=org.opencontainers.image.documentation={{.GitURL}}' + - '--label=org.opencontainers.image.created={{.Date}}' + - '--label=org.opencontainers.image.revision={{.FullCommit}}' + - '--label=org.opencontainers.image.version={{.Version}}' + - '--platform=linux/amd64' + + - use: buildx + goos: linux + goarch: arm64 + dockerfile: build/buildx-alpine.Dockerfile + image_templates: + - 'foomo/contentserver:latest-alpine-arm64' + - 'foomo/contentserver:{{ .Tag }}-alpine-arm64' + - 'foomo/contentserver:v{{ .Major }}.{{ .Minor }}-alpine-arm64' + build_flag_templates: + - '--pull' + # https://github.com/opencontainers/image-spec/blob/main/annotations.md#pre-defined-annotation-keys + - '--label=org.opencontainers.image.title={{.ProjectName}}' + - '--label=org.opencontainers.image.description=Serves content tree structures very quickly' + - '--label=org.opencontainers.image.source={{.GitURL}}' + - '--label=org.opencontainers.image.url={{.GitURL}}' + - '--label=org.opencontainers.image.documentation={{.GitURL}}' + - '--label=org.opencontainers.image.created={{.Date}}' + - '--label=org.opencontainers.image.revision={{.FullCommit}}' + - '--label=org.opencontainers.image.version={{.Version}}' + - '--platform=linux/arm64' diff --git a/.travis.yml b/.travis.yml deleted file mode 100644 index 639e8ee..0000000 --- a/.travis.yml +++ /dev/null @@ -1,9 +0,0 @@ -language: go -go: "1.18" -os: - - linux -dist: trusty -sudo: false -install: true -script: - - make test diff --git a/Dockerfile b/Dockerfile deleted file mode 100644 index f79dbaf..0000000 --- a/Dockerfile +++ /dev/null @@ -1,39 +0,0 @@ -############################## -###### STAGE: BUILD ###### -############################## -FROM golang:1.18 AS build-env - -WORKDIR /src - -COPY ./go.mod ./go.sum ./ - -RUN --mount=type=cache,target=/root/.cache/go-build \ - --mount=type=cache,target=/go/pkg/mod \ - go mod download -x - -COPY ./ ./ - -RUN GOARCH=amd64 GOOS=linux CGO_ENABLED=0 go build -trimpath -o /contentserver - -############################## -###### STAGE: PACKAGE ###### -############################## -FROM alpine - -ENV CONTENT_SERVER_ADDR=0.0.0.0:80 -ENV CONTENT_SERVER_VAR_DIR=/var/lib/contentserver -ENV LOG_JSON=1 - -RUN apk add --update --no-cache ca-certificates curl bash && rm -rf /var/cache/apk/* - -COPY --from=build-env /contentserver /usr/sbin/contentserver - - -VOLUME $CONTENT_SERVER_VAR_DIR - -ENTRYPOINT ["/usr/sbin/contentserver"] - -CMD ["-address=$CONTENT_SERVER_ADDR", "-var-dir=$CONTENT_SERVER_VAR_DIR"] - -EXPOSE 80 -EXPOSE 9200 diff --git a/Makefile b/Makefile index b292ab9..dbfe0ac 100644 --- a/Makefile +++ b/Makefile @@ -1,92 +1,86 @@ -SHELL := /bin/bash +-include .makerc +.DEFAULT_GOAL:=help -TAG?=latest -IMAGE=foomo/contentserver +# --- Targets ----------------------------------------------------------------- -# Utils +## === Tasks === -all: build test -tag: - echo $(TAG) -clean: - rm -fv bin/contentserve* - -# Build - -build: clean - go build -o bin/contentserver - -build-arch: clean - GOOS=linux GOARCH=amd64 go build -o bin/contentserver-linux-amd64 - GOOS=darwin GOARCH=amd64 go build -o bin/contentserver-darwin-amd64 -build-docker: clean build-arch - curl https://curl.haxx.se/ca/cacert.pem > .cacert.pem - docker build -q . > .image_id - docker tag `cat .image_id` $(IMAGE):$(TAG) - echo "# tagged container `cat .image_id` as $(IMAGE):$(TAG)" - rm -vf .image_id .cacert.pem - -build-testclient: - go build -o bin/testclient -i github.com/foomo/contentserver/testing/client - -build-testserver: - go build -o bin/testserver -i github.com/foomo/contentserver/testing/server - -package: build - pkg/build.sh - -# Docker - -docker-build: - DOCKER_BUILDKIT=1 docker build -t $(IMAGE):$(TAG) --platform linux/amd64 --progress=plain . - -docker-push: - docker push $(IMAGE):$(TAG) - -# Testing / benchmarks +.PHONY: doc +## Open go docs +doc: + @open "http://localhost:6060/pkg/github.com/foomo/contentserver/" + @godoc -http=localhost:6060 -play +.PHONY: test +## Run tests test: - go test -v ./... + @go test -coverprofile=coverage.out -race -json ./... | gotestfmt -bench: - go test -run=none -bench=. ./... +.PHONY: test.update +## Run tests and update snapshots +test.update: + @go test -update -coverprofile=coverage.out -race -json ./... | gotestfmt -run-testserver: - bin/testserver -json-file var/cse-globus-stage-b-with-main-section.json +.PHONY: lint +## Run linter +lint: + @golangci-lint run -run-contentserver: - contentserver -var-dir var -webserver-address :9191 -address :9999 http://127.0.0.1:1234 +.PHONY: lint.fix +## Fix lint violations +lint.fix: + @golangci-lint run --fix -run-contentserver-freeosmem: - contentserver -var-dir var -webserver-address :9191 -address :9999 -free-os-mem 1 http://127.0.0.1:1234 +.PHONY: tidy +## Run go mod tidy +tidy: + @go mod tidy -run-prometheus: - prometheus --config.file=prometheus/prometheus.yml +.PHONY: outdated +## Show outdated direct dependencies +outdated: + @go list -u -m -json all | go-mod-outdated -update -direct -clean-var: - rm var/contentserver-repo-2019* +## Install binary +install: + @go build -o ${GOPATH}/bin/contentserver main.go -# Profiling +## Build binary +build: + @mkdir -p bin + @go build -o bin/sesamy main.go -test-cpu-profile: - go test -cpuprofile=cprof-client github.com/foomo/contentserver/client - go tool pprof --text client.test cprof-client +## === Utils === - go test -cpuprofile=cprof-repo github.com/foomo/contentserver/repo - go tool pprof --text repo.test cprof-repo - -test-gctrace: - GODEBUG=gctrace=1 go test ./... - -test-malloctrace: - GODEBUG=allocfreetrace=1 go test ./... - -trace: - curl http://localhost:6060/debug/pprof/trace?seconds=60 > cs-trace - go tool trace cs-trace - -pprof-heap-web: - go tool pprof -http=":8081" http://localhost:6060/debug/pprof/heap - -pprof-cpu-web: - go tool pprof -http=":8081" http://localhost:6060/debug/pprof/profile \ No newline at end of file +.PHONY: help +## Show help text +help: + @awk '{ \ + if ($$0 ~ /^.PHONY: [a-zA-Z\-\_0-9]+$$/) { \ + helpCommand = substr($$0, index($$0, ":") + 2); \ + if (helpMessage) { \ + printf "\033[36m%-23s\033[0m %s\n", \ + helpCommand, helpMessage; \ + helpMessage = ""; \ + } \ + } else if ($$0 ~ /^[a-zA-Z\-\_0-9.]+:/) { \ + helpCommand = substr($$0, 0, index($$0, ":")); \ + if (helpMessage) { \ + printf "\033[36m%-23s\033[0m %s\n", \ + helpCommand, helpMessage"\n"; \ + helpMessage = ""; \ + } \ + } else if ($$0 ~ /^##/) { \ + if (helpMessage) { \ + helpMessage = helpMessage"\n "substr($$0, 3); \ + } else { \ + helpMessage = substr($$0, 3); \ + } \ + } else { \ + if (helpMessage) { \ + print "\n "helpMessage"\n" \ + } \ + helpMessage = ""; \ + } \ + }' \ + $(MAKEFILE_LIST) diff --git a/README.md b/README.md index feecc52..a96ef2e 100644 --- a/README.md +++ b/README.md @@ -1,51 +1,60 @@ -[![Travis CI](https://travis-ci.org/foomo/contentserver.svg?branch=master)](https://travis-ci.org/foomo/contentserver) - # Content Server -Serves content tree structures very quickly through a json socket api +[![Build Status](https://github.com/foomo/contentserver/actions/workflows/test.yml/badge.svg?branch=main&event=push)](https://github.com/foomo/contentserver/actions/workflows/test.yml) +[![Go Report Card](https://goreportcard.com/badge/github.com/foomo/contentserver)](https://goreportcard.com/report/github.com/foomo/contentserver) +[![godoc](https://godoc.org/github.com/foomo/contentserver?status.svg)](https://godoc.org/github.com/foomo/contentserver) +[![goreleaser](https://github.com/foomo/contentserver/actions/workflows/release.yml/badge.svg)](https://github.com/foomo/contentserver/actions) + +Serves content tree structures very quickly. ## Concept -A Server written in GoLang to mix and resolve content from different content sources, e.g. CMS, Blog, Shop and many other more. The server provides a simple to use API for non blocking content repository updates, to resolve site content by an URI or to get deep-linking multilingual URIs for a given contentID. +A Server written in GoLang to mix and resolve content from different content sources, e.g. CMS, Blog, Shop and many +other more. The server provides a simple to use API for non blocking content repository updates, to resolve site content +by an URI or to get deep-linking multilingual URIs for a given contentID. -It's up to you how you use it and which data you want to export to the server. Our intention was to write a fast and cache hazzle-free content server to mix different content sources. +It's up to you how you use it and which data you want to export to the server. Our intention was to write a fast and +cache hazzle-free content server to mix different content sources. ### Overview - + ## Export Data All you have to do is to provide a tree of content nodes as a JSON encoded RepoNode. -| Attribute | Type | Usage | -|---------------|:----------------------:|----------------------------------------------------------------------------------:| -| Id | string | unique id to identify the node | -| MimeType | string | mime-type of the node, e.g. text/html, image/png, ... | -| LinkId | string | (symbolic) link/alias to another node | -| Groups | []string | access control | -| URI | string | URI | -| Name | string | name | -| Hidden | bool | hide in menu | -| DestinationId | string | alias / symlink handling | -| Data | map[string]interface{} | payload data | -| Nodes | map[string]*RepoNode | child nodes | -| Index | []string | contains the order of of nodes | +| Attribute | Type | Usage | +|---------------|:----------------------:|------------------------------------------------------:| +| Id | string | unique id to identify the node | +| MimeType | string | mime-type of the node, e.g. text/html, image/png, ... | +| LinkId | string | (symbolic) link/alias to another node | +| Groups | []string | access control | +| URI | string | URI | +| Name | string | name | +| Hidden | bool | hide in menu | +| DestinationId | string | alias / symlink handling | +| Data | map[string]interface{} | payload data | +| Nodes | map[string]*RepoNode | child nodes | +| Index | []string | contains the order of of nodes | ### Tips -- If you do not want to build a multi-market website define a generic market, e.g. call it *universe* -- keep it lean and do not export content which should not be accessible at all, e.g. you are working on a super secret fancy new category of your website -- Hidden nodes can be resolved by their uri, but are hidden on nodes -- To avoid duplicate content provide a DestinationId ( = ContentId of the node you want to reference) instead of URIs +- If you do not want to build a multi-market website define a generic market, e.g. call it *universe* +- keep it lean and do not export content which should not be accessible at all, e.g. you are working on a super secret + fancy new category of your website +- Hidden nodes can be resolved by their uri, but are hidden on nodes +- To avoid duplicate content provide a DestinationId ( = ContentId of the node you want to reference) instead of URIs ## Request Data -There is a PHP Proxy implementation for foomo in [Foomo.ContentServer](https://github.com/foomo/Foomo.ContentServer). Feel free to use it or to implement your own proxy in the language you love. The API should be easily to implement in every other framework and language, too. +There is a PHP Proxy implementation for foomo in [Foomo.ContentServer](https://github.com/foomo/Foomo.ContentServer). +Feel free to use it or to implement your own proxy in the language you love. The API should be easily to implement in +every other framework and language, too. ## Update Flowchart - + ### Usage diff --git a/build/buildx-alpine.Dockerfile b/build/buildx-alpine.Dockerfile new file mode 100644 index 0000000..f78e98e --- /dev/null +++ b/build/buildx-alpine.Dockerfile @@ -0,0 +1,30 @@ +# syntax=docker/dockerfile:1.4 +FROM golang:1.21-alpine + +# related to https://github.com/golangci/golangci-lint/issues/3107 +ENV GOROOT /usr/local/go + +# Allow to download a more recent version of Go. +# https://go.dev/doc/toolchain +# GOTOOLCHAIN=auto is shorthand for GOTOOLCHAIN=local+auto +ENV GOTOOLCHAIN auto + +# gcc is required to support cgo; +# git and mercurial are needed most times for go get`, etc. +# See https://github.com/docker-library/golang/issues/80 +RUN apk --no-cache add gcc musl-dev git mercurial + +# Set all directories as safe +RUN git config --global --add safe.directory '*' + +COPY contentserver /usr/bin/ +ENTRYPOINT ["/usr/bin/contentserver"] + +ENV CONTENT_SERVER_ADDRESS=0.0.0.0:8080 +ENV CONTENT_SERVER_VAR_DIR=/var/lib/contentserver +ENV LOG_JSON=1 + +EXPOSE 8080 +EXPOSE 9200 + +CMD ["-address=$CONTENT_SERVER_ADDRESS", "-var-dir=$CONTENT_SERVER_VAR_DIR"] diff --git a/build/buildx.Dockerfile b/build/buildx.Dockerfile new file mode 100644 index 0000000..16efe7f --- /dev/null +++ b/build/buildx.Dockerfile @@ -0,0 +1,25 @@ +# syntax=docker/dockerfile:1.4 +FROM golang:1.21 + +# related to https://github.com/golangci/golangci-lint/issues/3107 +ENV GOROOT /usr/local/go + +# Allow to download a more recent version of Go. +# https://go.dev/doc/toolchain +# GOTOOLCHAIN=auto is shorthand for GOTOOLCHAIN=local+auto +ENV GOTOOLCHAIN auto + +# Set all directories as safe +RUN git config --global --add safe.directory '*' + +COPY contentserver /usr/bin/ +ENTRYPOINT ["/usr/bin/contentserver"] + +ENV CONTENT_SERVER_ADDRESS=0.0.0.0:8080 +ENV CONTENT_SERVER_VAR_DIR=/var/lib/contentserver +ENV LOG_JSON=1 + +EXPOSE 8080 +EXPOSE 9200 + +CMD ["-address=$CONTENT_SERVER_ADDRESS", "-var-dir=$CONTENT_SERVER_VAR_DIR"] diff --git a/client/client.go b/client/client.go index c67ae02..9f428eb 100644 --- a/client/client.go +++ b/client/client.go @@ -1,96 +1,58 @@ package client import ( + "context" "errors" - "net/http" - "net/url" - "time" "github.com/foomo/contentserver/content" + "github.com/foomo/contentserver/pkg/handler" "github.com/foomo/contentserver/requests" "github.com/foomo/contentserver/responses" - "github.com/foomo/contentserver/server" +) + +var ( + ErrEmptyServerURL = errors.New("empty contentserver url provided") + ErrInvalidServerURL = errors.New("invalid contentserver url provided") ) // Client a content server client type Client struct { - t transport + t Transport } -func NewClient( - server string, - connectionPoolSize int, - waitTimeout time.Duration, -) (c *Client, err error) { - return NewClientWithTransport(NewSocketTransport(server, connectionPoolSize, waitTimeout)) -} +// ------------------------------------------------------------------------------------------------ +// ~ Constructor +// ------------------------------------------------------------------------------------------------ -func NewClientWithTransport( - transport transport, -) (c *Client, err error) { - c = &Client{ +func New(transport Transport) *Client { + return &Client{ t: transport, } - return } -var ( - ErrEmptyServerURL = errors.New("empty contentserver url provided") - ErrInvalidServerURL = errors.New("invalid contentserver url provided") -) - -func isValidUrl(str string) bool { - u, err := url.Parse(str) - - if u.Scheme != "http" && u.Scheme != "https" { - return false - } - - return err == nil && u.Scheme != "" && u.Host != "" -} - -// NewHTTPClient constructs a new client to talk to the contentserver. -// It returns an error if the provided url is empty or invalid. -func NewHTTPClient(server string) (c *Client, err error) { - - if server == "" { - return nil, ErrEmptyServerURL - } - - // validate url - if !isValidUrl(server) { - return nil, ErrInvalidServerURL - } - - return NewHTTPClientWithTransport(NewHTTPTransport(server, http.DefaultClient)) -} - -func NewHTTPClientWithTransport(transport transport) (c *Client, err error) { - c = &Client{ - t: transport, - } - return -} +// ------------------------------------------------------------------------------------------------ +// ~ Public methods +// ------------------------------------------------------------------------------------------------ // Update tell the server to update itself -func (c *Client) Update() (*responses.Update, error) { +func (c *Client) Update(ctx context.Context) (*responses.Update, error) { type serverResponse struct { Reply *responses.Update } resp := serverResponse{} - if err := c.t.call(server.HandlerUpdate, &requests.Update{}, &resp); err != nil { + if err := c.t.Call(ctx, handler.RouteUpdate, &requests.Update{}, &resp); err != nil { return nil, err } return resp.Reply, nil } // GetContent request site content -func (c *Client) GetContent(request *requests.Content) (*content.SiteContent, error) { +func (c *Client) GetContent(ctx context.Context, request *requests.Content) (*content.SiteContent, error) { type serverResponse struct { Reply *content.SiteContent } resp := serverResponse{} - if err := c.t.call(server.HandlerGetContent, request, &resp); err != nil { + if err := c.t.Call(ctx, handler.RouteGetContent, request, &resp); err != nil { return nil, err } @@ -98,20 +60,20 @@ func (c *Client) GetContent(request *requests.Content) (*content.SiteContent, er } // GetURIs resolve uris for ids in a dimension -func (c *Client) GetURIs(dimension string, IDs []string) (map[string]string, error) { +func (c *Client) GetURIs(ctx context.Context, dimension string, ids []string) (map[string]string, error) { type serverResponse struct { Reply map[string]string } resp := serverResponse{} - if err := c.t.call(server.HandlerGetURIs, &requests.URIs{Dimension: dimension, IDs: IDs}, &resp); err != nil { + if err := c.t.Call(ctx, handler.RouteGetURIs, &requests.URIs{Dimension: dimension, IDs: ids}, &resp); err != nil { return nil, err } return resp.Reply, nil } // GetNodes request nodes -func (c *Client) GetNodes(env *requests.Env, nodes map[string]*requests.Node) (map[string]*content.Node, error) { +func (c *Client) GetNodes(ctx context.Context, env *requests.Env, nodes map[string]*requests.Node) (map[string]*content.Node, error) { r := &requests.Nodes{ Env: env, Nodes: nodes, @@ -120,24 +82,24 @@ func (c *Client) GetNodes(env *requests.Env, nodes map[string]*requests.Node) (m Reply map[string]*content.Node } resp := serverResponse{} - if err := c.t.call(server.HandlerGetNodes, r, &resp); err != nil { + if err := c.t.Call(ctx, handler.RouteGetNodes, r, &resp); err != nil { return nil, err } return resp.Reply, nil } // GetRepo get the whole repo -func (c *Client) GetRepo() (map[string]*content.RepoNode, error) { +func (c *Client) GetRepo(ctx context.Context) (map[string]*content.RepoNode, error) { type serverResponse struct { Reply map[string]*content.RepoNode } resp := serverResponse{} - if err := c.t.call(server.HandlerGetRepo, &requests.Repo{}, &resp); err != nil { + if err := c.t.Call(ctx, handler.RouteGetRepo, &requests.Repo{}, &resp); err != nil { return nil, err } return resp.Reply, nil } -func (c *Client) ShutDown() { - c.t.shutdown() +func (c *Client) Close() { + c.t.Close() } diff --git a/client/client_test.go b/client/client_test.go index 2a252ee..209f3c2 100644 --- a/client/client_test.go +++ b/client/client_test.go @@ -1,210 +1,56 @@ -package client +package client_test import ( - "net" - "strconv" + "context" + "encoding/json" "sync" "testing" "time" + "github.com/foomo/contentserver/client" "github.com/foomo/contentserver/content" - . "github.com/foomo/contentserver/logger" - "github.com/foomo/contentserver/repo/mock" - "github.com/foomo/contentserver/requests" - "github.com/foomo/contentserver/server" + "github.com/foomo/contentserver/pkg/repo" + "github.com/foomo/contentserver/pkg/repo/mock" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" ) -const pathContentserver = "/contentserver" - -var ( - testServerSocketAddr string - testServerWebserverAddr string -) - -func init() { - SetupLogging(true, "contentserver_client_test.log") -} - -func TestInvalidHTTPClientInit(t *testing.T) { - c, err := NewHTTPClient("") - assert.Nil(t, c) - assert.Error(t, err) - - c, err = NewHTTPClient("bogus") - assert.Nil(t, c) - assert.Error(t, err) - - c, err = NewHTTPClient("htt:/notaurl") - assert.Nil(t, c) - assert.Error(t, err) - - c, err = NewHTTPClient("htts://notaurl") - assert.Nil(t, c) - assert.Error(t, err) - - c, err = NewHTTPClient("/path/segment/only") - assert.Nil(t, c) - assert.Error(t, err) -} - -func dump(t *testing.T, v interface{}) { - jsonBytes, err := json.MarshalIndent(v, "", " ") - if err != nil { - t.Fatal("could not dump v", v, "err", err) - return - } - t.Log(string(jsonBytes)) -} - -func getFreePort() int { - addr, err := net.ResolveTCPAddr("tcp", "localhost:0") - if err != nil { - panic(err) - } - l, err := net.ListenTCP("tcp", addr) - if err != nil { - panic(err) - } - defer l.Close() - return l.Addr().(*net.TCPAddr).Port -} - -func getAvailableAddr() string { - return "127.0.0.1:" + strconv.Itoa(getFreePort()) -} - -func initTestServer(t testing.TB) (socketAddr, webserverAddr string) { - socketAddr = getAvailableAddr() - webserverAddr = getAvailableAddr() - testServer, varDir := mock.GetMockData(t) - - go func() { - err := server.RunServerSocketAndWebServer( - testServer.URL+"/repo-two-dimensions.json", - socketAddr, - webserverAddr, - pathContentserver, - varDir, - server.DefaultRepositoryTimeout, - false, - ) - if err != nil { - t.Fatal("test server crashed: ", err) - } - }() - - socketClient, errClient := NewClient(socketAddr, 1, time.Duration(time.Millisecond*100)) - if errClient != nil { - panic(errClient) - } - i := 0 - for { - time.Sleep(time.Millisecond * 100) - r, err := socketClient.GetRepo() - if err != nil { - continue - } - - if len(r) == 0 { - t.Fatal("received empty JSON from GetRepo") - } - - if r["dimension_foo"].Nodes["id-a"].Data["baz"].(float64) == float64(1) { - break - } - if i > 100 { - panic("this is taking too long") - } - i++ - } - return -} - -func getTestClients(t testing.TB) (socketClient *Client, httpClient *Client) { - if testServerSocketAddr == "" { - socketAddr, webserverAddr := initTestServer(t) - testServerSocketAddr = socketAddr - testServerWebserverAddr = webserverAddr - } - socketClient, errClient := NewClient(testServerSocketAddr, 25, time.Duration(time.Millisecond*100)) - if errClient != nil { - t.Log(errClient) - t.Fail() - } - httpClient, errHTTPClient := NewHTTPClient("http://" + testServerWebserverAddr + pathContentserver) - if errHTTPClient != nil { - t.Log(errHTTPClient) - t.Fail() - } - return -} - -func testWithClients(t *testing.T, testFunc func(c *Client)) { - socketClient, httpClient := getTestClients(t) - defer socketClient.ShutDown() - defer httpClient.ShutDown() - testFunc(socketClient) - testFunc(httpClient) -} - func TestUpdate(t *testing.T) { - testWithClients(t, func(c *Client) { - response, err := c.Update() - if err != nil { - t.Fatal("unexpected err", err) - } - if !response.Success { - t.Fatal("update has to return .Sucesss true", response) - } - stats := response.Stats - if !(stats.RepoRuntime > float64(0.0)) || !(stats.OwnRuntime > float64(0.0)) { - t.Fatal("stats invalid") - } + testWithClients(t, func(c *client.Client) { + response, err := c.Update(context.TODO()) + require.NoError(t, err) + require.True(t, response.Success, "update has to return .Sucesss true") + assert.Greater(t, response.Stats.OwnRuntime, 0.0) + assert.Greater(t, response.Stats.RepoRuntime, 0.0) }) } func TestGetURIs(t *testing.T) { - testWithClients(t, func(c *Client) { - defer c.ShutDown() + testWithClients(t, func(c *client.Client) { request := mock.MakeValidURIsRequest() - uriMap, err := c.GetURIs(request.Dimension, request.IDs) - if err != nil { - t.Fatal(err) - } - if uriMap[request.IDs[0]] != "/a" { - t.Fatal(uriMap) - } + uriMap, err := c.GetURIs(context.TODO(), request.Dimension, request.IDs) + time.Sleep(100 * time.Millisecond) + require.NoError(t, err) + assert.Equal(t, "/a", uriMap[request.IDs[0]]) }) } func TestGetRepo(t *testing.T) { - testWithClients(t, func(c *Client) { - - time.Sleep(time.Millisecond * 100) - - r, err := c.GetRepo() - if err != nil { - t.Fatal(err) - } - - if len(r) == 0 { - t.Fatal("received empty JSON from GetRepo") - } - - if r["dimension_foo"].Nodes["id-a"].Data["baz"].(float64) != float64(1) { - t.Fatal("failed to drill deep for data") + testWithClients(t, func(c *client.Client) { + r, err := c.GetRepo(context.TODO()) + require.NoError(t, err) + if assert.NotEmpty(t, r, "received empty JSON from GetRepo") { + assert.Equal(t, 1.0, r["dimension_foo"].Nodes["id-a"].Data["baz"].(float64), "failed to drill deep for data") //nolint:forcetypeassert } }) } func TestGetNodes(t *testing.T) { - testWithClients(t, func(c *Client) { + testWithClients(t, func(c *client.Client) { nodesRequest := mock.MakeNodesRequest() - nodes, err := c.GetNodes(nodesRequest.Env, nodesRequest.Nodes) - if err != nil { - t.Fatal(err) - } + nodes, err := c.GetNodes(context.TODO(), nodesRequest.Env, nodesRequest.Nodes) + require.NoError(t, err) testNode, ok := nodes["test"] if !ok { t.Fatal("that should be a node") @@ -220,9 +66,9 @@ func TestGetNodes(t *testing.T) { } func TestGetContent(t *testing.T) { - testWithClients(t, func(c *Client) { + testWithClients(t, func(c *client.Client) { request := mock.MakeValidContentRequest() - response, err := c.GetContent(request) + response, err := c.GetContent(context.TODO(), request) if err != nil { t.Fatal("unexpected err", err) } @@ -237,17 +83,8 @@ func TestGetContent(t *testing.T) { }) } -func BenchmarkSocketClientAndServerGetContent(b *testing.B) { - socketClient, _ := getTestClients(b) - benchmarkServerAndClientGetContent(b, 30, 100, socketClient) - -} -func BenchmarkWebClientAndServerGetContent(b *testing.B) { - _, httpClient := getTestClients(b) - benchmarkServerAndClientGetContent(b, 30, 100, httpClient) -} - func benchmarkServerAndClientGetContent(b *testing.B, numGroups, numCalls int, client GetContentClient) { + b.Helper() b.ResetTimer() for i := 0; i < b.N; i++ { start := time.Now() @@ -258,30 +95,55 @@ func benchmarkServerAndClientGetContent(b *testing.B, numGroups, numCalls int, c } } -type GetContentClient interface { - GetContent(request *requests.Content) (response *content.SiteContent, err error) -} - -func benchmarkClientAndServerGetContent(b testing.TB, numGroups, numCalls int, client GetContentClient) { +func benchmarkClientAndServerGetContent(tb testing.TB, numGroups, numCalls int, client GetContentClient) { + tb.Helper() var wg sync.WaitGroup wg.Add(numGroups) for group := 0; group < numGroups; group++ { - go func(g int) { + go func() { defer wg.Done() request := mock.MakeValidContentRequest() for i := 0; i < numCalls; i++ { - response, err := client.GetContent(request) + response, err := client.GetContent(context.TODO(), request) if err == nil { if request.URI != response.URI { - b.Fatal("uri mismatch") + tb.Fatal("uri mismatch") } if response.Status != content.StatusOk { - b.Fatal("unexpected status") + tb.Fatal("unexpected status") } } } - }(group) + }() } // Wait for all HTTP fetches to complete. wg.Wait() } + +func initRepo(tb testing.TB, l *zap.Logger) *repo.Repo { + tb.Helper() + testRepoServer, varDir := mock.GetMockData(tb) + r := repo.New(l, + testRepoServer.URL+"/repo-two-dimensions.json", + repo.NewHistory(l, + repo.HistoryWithVarDir(varDir), + ), + ) + up := make(chan bool, 1) + r.OnStart(func() { + up <- true + }) + go r.Start(context.TODO()) //nolint:errcheck + <-up + return r +} + +func dump(t *testing.T, v interface{}) { + t.Helper() + jsonBytes, err := json.MarshalIndent(v, "", " ") + if err != nil { + t.Fatal("could not dump v", v, "err", err) + return + } + t.Log(string(jsonBytes)) +} diff --git a/client/connectionpool.go b/client/connectionpool.go index efcc348..e3c98b3 100644 --- a/client/connectionpool.go +++ b/client/connectionpool.go @@ -6,16 +6,16 @@ import ( ) type connectionPool struct { - server string + url string // conn net.Conn chanConnGet chan chan net.Conn chanConnReturn chan connReturn chanDrainPool chan int } -func newConnectionPool(server string, connectionPoolSize int, waitTimeout time.Duration) *connectionPool { +func newConnectionPool(url string, connectionPoolSize int, waitTimeout time.Duration) *connectionPool { connPool := &connectionPool{ - server: server, + url: url, chanConnGet: make(chan chan net.Conn), chanConnReturn: make(chan connReturn), chanDrainPool: make(chan int), @@ -92,7 +92,7 @@ RunLoop: // refill connection pool for _, poolEntry := range connectionPool { if poolEntry.conn == nil { - newConn, errDial := net.Dial("tcp", c.server) + newConn, errDial := net.Dial("tcp", c.url) poolEntry.err = errDial poolEntry.conn = newConn } @@ -131,5 +131,5 @@ RunLoop: c.chanDrainPool = nil c.chanConnReturn = nil c.chanConnGet = nil - //fmt.Println("runloop is done", waitPool) + // fmt.Println("runloop is done", waitPool) } diff --git a/client/httptransport.go b/client/httptransport.go index 379407b..54b1d32 100644 --- a/client/httptransport.go +++ b/client/httptransport.go @@ -2,45 +2,86 @@ package client import ( "bytes" + "context" "errors" - "io/ioutil" + "io" "net/http" - "github.com/foomo/contentserver/server" + "github.com/foomo/contentserver/pkg/handler" + "github.com/foomo/contentserver/pkg/utils" ) -type httpTransport struct { - client *http.Client - endpoint string -} +type ( + HTTPTransport struct { + httpClient *http.Client + endpoint string + } + HTTPTransportOption func(*HTTPTransport) +) + +// ------------------------------------------------------------------------------------------------ +// ~ Constructor +// ------------------------------------------------------------------------------------------------ // NewHTTPTransport will create a new http transport for the given server and client. // Caution: the provided server url is not validated! -func NewHTTPTransport(server string, client *http.Client) transport { - return &httpTransport{ - endpoint: server, - client: client, +func NewHTTPTransport(server string, opts ...HTTPTransportOption) *HTTPTransport { + inst := &HTTPTransport{ + endpoint: server, + httpClient: http.DefaultClient, + } + + for _, opt := range opts { + opt(inst) + } + + return inst +} + +// NewHTTPClient constructs a new client to talk to the contentserver. +// It returns an error if the provided url is empty or invalid. +func NewHTTPClient(url string) (c *Client, err error) { + if url == "" { + return nil, ErrEmptyServerURL + } + + // validate url + if !utils.IsValidUrl(url) { + return nil, ErrInvalidServerURL + } + + return New(NewHTTPTransport(url)), nil +} + +// ------------------------------------------------------------------------------------------------ +// ~ Options +// ------------------------------------------------------------------------------------------------ + +func HTTPTransportWithHTTPClient(v *http.Client) HTTPTransportOption { + return func(o *HTTPTransport) { + o.httpClient = v } } -func (ht *httpTransport) shutdown() { - // nothing to do here -} +// ------------------------------------------------------------------------------------------------ +// ~ Public methods +// ------------------------------------------------------------------------------------------------ -func (ht *httpTransport) call(handler server.Handler, request interface{}, response interface{}) error { +func (t *HTTPTransport) Call(ctx context.Context, route handler.Route, request interface{}, response interface{}) error { requestBytes, errMarshal := json.Marshal(request) if errMarshal != nil { return errMarshal } - req, errNewRequest := http.NewRequest( + req, errNewRequest := http.NewRequestWithContext( + ctx, http.MethodPost, - ht.endpoint+"/"+string(handler), + t.endpoint+"/"+string(route), bytes.NewBuffer(requestBytes), ) if errNewRequest != nil { return errNewRequest } - httpResponse, errDo := ht.client.Do(req) + httpResponse, errDo := t.httpClient.Do(req) if errDo != nil { return errDo } @@ -52,9 +93,13 @@ func (ht *httpTransport) call(handler server.Handler, request interface{}, respo if httpResponse.Body == nil { return errors.New("empty response body") } - responseBytes, errRead := ioutil.ReadAll(httpResponse.Body) + responseBytes, errRead := io.ReadAll(httpResponse.Body) if errRead != nil { return errRead } return json.Unmarshal(responseBytes, response) } + +func (t *HTTPTransport) Close() { + // nothing to do here +} diff --git a/client/httptransport_test.go b/client/httptransport_test.go new file mode 100644 index 0000000..ed12e34 --- /dev/null +++ b/client/httptransport_test.go @@ -0,0 +1,77 @@ +package client_test + +import ( + "context" + "net/http/httptest" + "testing" + + "github.com/foomo/contentserver/client" + "github.com/foomo/contentserver/content" + "github.com/foomo/contentserver/pkg/handler" + "github.com/foomo/contentserver/requests" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "go.uber.org/zap/zaptest" +) + +const pathContentserver = "/contentserver" + +func TestInvalidHTTPClientInit(t *testing.T) { + c, err := client.NewHTTPClient("") + assert.Nil(t, c) + assert.Error(t, err) + + c, err = client.NewHTTPClient("bogus") + assert.Nil(t, c) + assert.Error(t, err) + + c, err = client.NewHTTPClient("htt:/notaurl") + assert.Nil(t, c) + assert.Error(t, err) + + c, err = client.NewHTTPClient("htts://notaurl") + assert.Nil(t, c) + assert.Error(t, err) + + c, err = client.NewHTTPClient("/path/segment/only") + assert.Nil(t, c) + assert.Error(t, err) +} + +func BenchmarkWebClientAndServerGetContent(b *testing.B) { + l := zaptest.NewLogger(b) + server := initHTTPRepoServer(b, l) + httpClient := newHTTPClient(b, server) + benchmarkServerAndClientGetContent(b, 30, 100, httpClient) +} + +type GetContentClient interface { + GetContent(ctx context.Context, request *requests.Content) (response *content.SiteContent, err error) +} + +func newHTTPClient(tb testing.TB, server *httptest.Server) *client.Client { + tb.Helper() + c, err := client.NewHTTPClient(server.URL + pathContentserver) + require.NoError(tb, err) + return c +} + +func testWithClients(t *testing.T, testFunc func(c *client.Client)) { + t.Helper() + l := zaptest.NewLogger(t) + httpRepoServer := initHTTPRepoServer(t, l) + socketRepoServer := initSocketRepoServer(t, l) + httpClient := newHTTPClient(t, httpRepoServer) + socketClient := newSocketClient(t, socketRepoServer) + defer httpClient.Close() + defer socketClient.Close() + testFunc(httpClient) + testFunc(socketClient) +} + +func initHTTPRepoServer(tb testing.TB, l *zap.Logger) *httptest.Server { + tb.Helper() + r := initRepo(tb, l) + return httptest.NewServer(handler.NewHTTP(l, r)) +} diff --git a/client/json.go b/client/json.go new file mode 100644 index 0000000..a67967b --- /dev/null +++ b/client/json.go @@ -0,0 +1,7 @@ +package client + +import ( + jsoniter "github.com/json-iterator/go" +) + +var json = jsoniter.ConfigCompatibleWithStandardLibrary diff --git a/client/sockettransport.go b/client/sockettransport.go index 84ff550..667abf5 100644 --- a/client/sockettransport.go +++ b/client/sockettransport.go @@ -1,6 +1,7 @@ package client import ( + "context" "errors" "fmt" "io" @@ -8,36 +9,35 @@ import ( "strconv" "time" + "github.com/foomo/contentserver/pkg/handler" "github.com/foomo/contentserver/responses" - "github.com/foomo/contentserver/server" - jsoniter "github.com/json-iterator/go" ) -var json = jsoniter.ConfigCompatibleWithStandardLibrary - type connReturn struct { conn net.Conn err error } -type socketTransport struct { +type SocketTransport struct { connPool *connectionPool } -func NewSocketTransport(server string, connectionPoolSize int, waitTimeout time.Duration) transport { - return &socketTransport{ - connPool: newConnectionPool(server, connectionPoolSize, waitTimeout), +// ------------------------------------------------------------------------------------------------ +// ~ Constructor +// ------------------------------------------------------------------------------------------------ + +func NewSocketTransport(url string, connectionPoolSize int, waitTimeout time.Duration) *SocketTransport { + return &SocketTransport{ + connPool: newConnectionPool(url, connectionPoolSize, waitTimeout), } } -func (st *socketTransport) shutdown() { - if st.connPool.chanDrainPool != nil { - st.connPool.chanDrainPool <- 1 - } -} +// ------------------------------------------------------------------------------------------------ +// ~ Public methods +// ------------------------------------------------------------------------------------------------ -func (c *socketTransport) call(handler server.Handler, request interface{}, response interface{}) error { - if c.connPool.chanDrainPool == nil { +func (t *SocketTransport) Call(ctx context.Context, route handler.Route, request interface{}, response interface{}) error { + if t.connPool.chanDrainPool == nil { return errors.New("connection pool has been drained, client is dead") } jsonBytes, err := json.Marshal(request) @@ -45,19 +45,19 @@ func (c *socketTransport) call(handler server.Handler, request interface{}, resp return fmt.Errorf("could not marshal request : %q", err) } netChan := make(chan net.Conn) - c.connPool.chanConnGet <- netChan + t.connPool.chanConnGet <- netChan conn := <-netChan if conn == nil { return errors.New("could not get a connection") } returnConn := func(err error) { - c.connPool.chanConnReturn <- connReturn{ + t.connPool.chanConnReturn <- connReturn{ conn: conn, err: err, } } // write header result will be like handler:2{} - jsonBytes = append([]byte(fmt.Sprintf("%s:%d", handler, len(jsonBytes))), jsonBytes...) + jsonBytes = append([]byte(fmt.Sprintf("%s:%d", route, len(jsonBytes))), jsonBytes...) // send request var ( @@ -125,3 +125,9 @@ func (c *socketTransport) call(handler server.Handler, request interface{}, resp returnConn(nil) return nil } + +func (t *SocketTransport) Close() { + if t.connPool.chanDrainPool != nil { + t.connPool.chanDrainPool <- 1 + } +} diff --git a/client/sockettransport_test.go b/client/sockettransport_test.go new file mode 100644 index 0000000..4e20820 --- /dev/null +++ b/client/sockettransport_test.go @@ -0,0 +1,56 @@ +package client_test + +import ( + "testing" + "time" + + "github.com/foomo/contentserver/client" + "github.com/foomo/contentserver/pkg/handler" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "go.uber.org/zap/zaptest" + "golang.org/x/net/nettest" +) + +func BenchmarkSocketClientAndServerGetContent(b *testing.B) { + l := zaptest.NewLogger(b) + server := initSocketRepoServer(b, l) + socketClient := newSocketClient(b, server) + benchmarkServerAndClientGetContent(b, 30, 100, socketClient) +} + +func newSocketClient(tb testing.TB, address string) *client.Client { + tb.Helper() + return client.New(client.NewSocketTransport(address, 25, 100*time.Millisecond)) +} + +func initSocketRepoServer(tb testing.TB, l *zap.Logger) string { + tb.Helper() + r := initRepo(tb, l) + h := handler.NewSocket(l, r) + + // listen on socket + ln, err := nettest.NewLocalListener("tcp") + + require.NoError(tb, err) + + go func() { + for { + // this blocks until connection or error + conn, err := ln.Accept() + if err != nil { + tb.Error("runSocketServer: could not accept connection", err.Error()) + continue + } + + // a goroutine handles conn so that the loop can accept other connections + go func() { + l.Debug("accepted connection", zap.String("source", conn.RemoteAddr().String())) + h.Serve(conn) + require.NoError(tb, conn.Close()) + }() + } + }() + + return ln.Addr().String() +} diff --git a/client/transport.go b/client/transport.go index 963d98c..638124a 100644 --- a/client/transport.go +++ b/client/transport.go @@ -1,8 +1,12 @@ package client -import "github.com/foomo/contentserver/server" +import ( + "context" -type transport interface { - call(handler server.Handler, request interface{}, response interface{}) error - shutdown() + "github.com/foomo/contentserver/pkg/handler" +) + +type Transport interface { + Call(ctx context.Context, route handler.Route, request interface{}, response interface{}) error + Close() } diff --git a/cmd/flags.go b/cmd/flags.go new file mode 100644 index 0000000..a5f58fa --- /dev/null +++ b/cmd/flags.go @@ -0,0 +1,57 @@ +package cmd + +import ( + "github.com/spf13/cobra" + "github.com/spf13/viper" +) + +func addAddressFlag(cmd *cobra.Command, v *viper.Viper) { + cmd.Flags().String("address", "localhost:8080", "Address to bind to (host:port)") + _ = v.BindPFlag("address", cmd.Flags().Lookup("address")) +} + +func addBasePathFlag(cmd *cobra.Command, v *viper.Viper) { + cmd.Flags().String("base-path", "/contentserver", "Base path to export the webserver on") + _ = v.BindPFlag("base_path", cmd.Flags().Lookup("base_path")) +} + +func addPollFlag(cmd *cobra.Command, v *viper.Viper) { + cmd.Flags().Bool("poll", false, "If true, the address arg will be used to periodically poll the content url") + _ = v.BindPFlag("poll", cmd.Flags().Lookup("poll")) +} + +func addHistoryDirFlag(cmd *cobra.Command, v *viper.Viper) { + cmd.Flags().String("history-dir", "/var/lib/contentserver", "Where to put my data") + _ = v.BindPFlag("history.dir", cmd.Flags().Lookup("history-dir")) + _ = v.BindEnv("history.dir", "CONTENT_SERVER_HISTORY_DIR") +} + +func addHistoryLimitFlag(cmd *cobra.Command, v *viper.Viper) { + cmd.Flags().Int("history-limit", 2, "Number of history records to keep") + _ = v.BindPFlag("history.limit", cmd.Flags().Lookup("history-limit")) +} + +func addGracefulTimeoutFlag(cmd *cobra.Command, v *viper.Viper) { + cmd.Flags().Duration("graceful-timeout", 0, "Timeout duration for graceful shutdown") + _ = v.BindPFlag("graceful.timeout", cmd.Flags().Lookup("graceful-timeout")) +} + +func addShutdownTimeoutFlag(cmd *cobra.Command, v *viper.Viper) { + cmd.Flags().Duration("shutdown-timeout", 0, "Timeout duration for shutdown") + _ = v.BindPFlag("shutdown.timeout", cmd.Flags().Lookup("shutdown-timeout")) +} + +func addOtelEnabledFlag(cmd *cobra.Command, v *viper.Viper) { + cmd.Flags().Bool("otel-enabled", false, "Enable otel service") + _ = v.BindPFlag("otel.enabled", cmd.Flags().Lookup("otel-enabled")) +} + +func addHealthzEnabledFlag(cmd *cobra.Command, v *viper.Viper) { + cmd.Flags().Bool("healthz-enabled", false, "Enable healthz service") + _ = v.BindPFlag("healthz.enabled", cmd.Flags().Lookup("healthz-enabled")) +} + +func addPrometheusEnabledFlag(cmd *cobra.Command, v *viper.Viper) { + cmd.Flags().Bool("prometheus-enabled", false, "Enable prometheus service") + _ = v.BindPFlag("prometheus.enabled", cmd.Flags().Lookup("prometheus-enabled")) +} diff --git a/cmd/http.go b/cmd/http.go new file mode 100644 index 0000000..93c3a06 --- /dev/null +++ b/cmd/http.go @@ -0,0 +1,104 @@ +package cmd + +import ( + "context" + "errors" + + "github.com/foomo/contentserver/pkg/handler" + "github.com/foomo/contentserver/pkg/repo" + "github.com/foomo/keel" + "github.com/foomo/keel/healthz" + keelhttp "github.com/foomo/keel/net/http" + "github.com/foomo/keel/net/http/middleware" + "github.com/foomo/keel/service" + "github.com/spf13/cobra" + "go.uber.org/zap" +) + +func NewHTTPCommand() *cobra.Command { + v := NewViper() + cmd := &cobra.Command{ + Use: "http ", + Short: "Start http server", + Args: cobra.ExactArgs(1), + ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + var comps []string + if len(args) == 0 { + comps = cobra.AppendActiveHelp(comps, "You must specify the URL for the repository you are adding") + } else { + comps = cobra.AppendActiveHelp(comps, "This command does not take any more arguments") + } + return comps, cobra.ShellCompDirectiveNoFileComp + }, + RunE: func(cmd *cobra.Command, args []string) error { + svr := keel.NewServer( + keel.WithLogger(logger), + keel.WithHTTPReadmeService(true), + keel.WithHTTPPrometheusService(v.GetBool("prometheus.enabled")), + keel.WithHTTPHealthzService(v.GetBool("healthz.enabled")), + keel.WithPrometheusMeter(v.GetBool("prometheus.enabled")), + keel.WithOTLPGRPCTracer(v.GetBool("otel.enabled")), + keel.WithGracefulTimeout(v.GetDuration("graceful.timeout")), + keel.WithShutdownTimeout(v.GetDuration("shutdown.timeout")), + ) + + l := svr.Logger() + + r := repo.New(l, + args[0], + repo.NewHistory(l, + repo.HistoryWithVarDir(v.GetString("history.dir")), + repo.HistoryWithMax(v.GetInt("history.limit")), + ), + repo.WithHTTPClient( + keelhttp.NewHTTPClient( + keelhttp.HTTPClientWithTelemetry(), + ), + ), + repo.WithPollForUpdates(v.GetBool("poll")), + ) + + // start initial update and handle error + svr.AddReadinessHealthzers(healthz.NewHealthzerFn(func(ctx context.Context) error { + if !r.Loaded() { + return errors.New("repo not ready yet") + } + return nil + })) + + svr.AddServices( + service.NewHTTP(l, "http", v.GetString("address"), + handler.NewHTTP(l, r, handler.WithPath(v.GetString("path"))), + middleware.Telemetry(), + middleware.Logger(), + middleware.Recover(), + ), + service.NewGoRoutine(l, "repo", func(ctx context.Context, l *zap.Logger) error { + return r.Start(ctx) + }), + ) + + svr.Run() + return nil + }, + } + + addAddressFlag(cmd, v) + addBasePathFlag(cmd, v) + addPollFlag(cmd, v) + addHistoryDirFlag(cmd, v) + addHistoryLimitFlag(cmd, v) + addGracefulTimeoutFlag(cmd, v) + addShutdownTimeoutFlag(cmd, v) + addOtelEnabledFlag(cmd, v) + addHealthzEnabledFlag(cmd, v) + addPrometheusEnabledFlag(cmd, v) + // cmd.Flags().SetNormalizeFunc(normalizeFlag) + + return cmd +} + +func init() { + rootCmd.AddCommand(NewHTTPCommand()) + +} diff --git a/cmd/init.go b/cmd/init.go new file mode 100644 index 0000000..83f7bdc --- /dev/null +++ b/cmd/init.go @@ -0,0 +1,27 @@ +package cmd + +import ( + "strings" + + "github.com/spf13/viper" + "go.uber.org/zap" +) + +// initConfig reads in config file and ENV variables if set. +func initConfig() { + viper.EnvKeyReplacer(strings.NewReplacer(".", "_")) +} + +// initConfig reads in config file and ENV variables if set. +func initLogger() { + var err error + c := zap.NewProductionConfig() + c.Level, err = zap.ParseAtomicLevel(logLevel) + if err != nil { + panic(err) + } + logger, err = c.Build() + if err != nil { + panic(err) + } +} diff --git a/cmd/root.go b/cmd/root.go new file mode 100644 index 0000000..e6966e4 --- /dev/null +++ b/cmd/root.go @@ -0,0 +1,49 @@ +package cmd + +import ( + "os" + + "github.com/spf13/cobra" + "github.com/spf13/viper" + "go.uber.org/zap" +) + +var ( + logger *zap.Logger + logLevel string +) + +// rootCmd represents the base command when called without any subcommands +var rootCmd = &cobra.Command{ + Use: "contentserver", + Short: "Serves content tree structures very quickly", +} + +// Execute adds all child commands to the root command and sets flags appropriately. +// This is called by main.main(). It only needs to happen once to the rootCmd. +func Execute() { + err := rootCmd.Execute() + if err != nil { + os.Exit(1) + } +} + +func init() { + cobra.OnInitialize(initConfig, initLogger) + + // Here you will define your flags and configuration settings. + // Cobra supports persistent flags, which, if defined here, + // will be global for your application. + + rootCmd.PersistentFlags().StringVar(&logLevel, "log-level", "info", "log level") + + // Cobra also supports local flags, which will only run + // when this action is called directly. + // rootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") +} + +func NewViper() *viper.Viper { + v := viper.New() + v.AutomaticEnv() + return v +} diff --git a/cmd/socket.go b/cmd/socket.go new file mode 100644 index 0000000..ef5fd74 --- /dev/null +++ b/cmd/socket.go @@ -0,0 +1,96 @@ +package cmd + +import ( + "context" + "net" + + "github.com/foomo/contentserver/pkg/handler" + "github.com/foomo/contentserver/pkg/repo" + keelhttp "github.com/foomo/keel/net/http" + "github.com/spf13/cobra" + "github.com/spf13/viper" + "go.uber.org/zap" +) + +func NewSocketCommand() *cobra.Command { + v := viper.New() + cmd := &cobra.Command{ + Use: "socket ", + Short: "Start socket server", + Args: cobra.ExactArgs(1), + ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + var comps []string + if len(args) == 0 { + comps = cobra.AppendActiveHelp(comps, "You must specify the URL for the repository you are adding") + } else { + comps = cobra.AppendActiveHelp(comps, "This command does not take any more arguments") + } + return comps, cobra.ShellCompDirectiveNoFileComp + }, + RunE: func(cmd *cobra.Command, args []string) error { + l := logger + + r := repo.New(l, + args[0], + repo.NewHistory(l, + repo.HistoryWithVarDir(v.GetString("history.dir")), + repo.HistoryWithMax(v.GetInt("history.limit")), + ), + repo.WithHTTPClient( + keelhttp.NewHTTPClient( + keelhttp.HTTPClientWithTelemetry(), + ), + ), + repo.WithPollForUpdates(v.GetBool("poll")), + ) + + // create socket server + handle := handler.NewSocket(l, r) + + // listen on socket + ln, err := net.Listen("tcp", v.GetString("address")) + if err != nil { + return err + } + + // start repo + up := make(chan bool, 1) + r.OnStart(func() { + up <- true + }) + go r.Start(context.Background()) //nolint:errcheck + <-up + + l.Info("started listening", zap.String("address", v.GetString("address"))) + + for { + // this blocks until connection or error + conn, err := ln.Accept() + if err != nil { + l.Error("runSocketServer: could not accept connection", zap.Error(err)) + continue + } + + // a goroutine handles conn so that the loop can accept other connections + go func() { + l.Debug("accepted connection", zap.String("source", conn.RemoteAddr().String())) + handle.Serve(conn) + if err := conn.Close(); err != nil { + l.Warn("failed to close connection", zap.Error(err)) + } + }() + } + }, + } + + addAddressFlag(cmd, v) + addPollFlag(cmd, v) + addHistoryDirFlag(cmd, v) + addHistoryLimitFlag(cmd, v) + + return cmd +} + +func init() { + rootCmd.AddCommand(NewSocketCommand()) +} diff --git a/cmd/version.go b/cmd/version.go new file mode 100644 index 0000000..92815b1 --- /dev/null +++ b/cmd/version.go @@ -0,0 +1,22 @@ +package cmd + +import ( + "fmt" + + "github.com/spf13/cobra" +) + +// Populated by goreleaser during build +var version = "latest" + +var versionCmd = &cobra.Command{ + Use: "version", + Short: "Print version information", + Run: func(cmd *cobra.Command, args []string) { + fmt.Println(version) + }, +} + +func init() { + rootCmd.AddCommand(versionCmd) +} diff --git a/content/reponode.go b/content/reponode.go index b5cf917..b368513 100644 --- a/content/reponode.go +++ b/content/reponode.go @@ -25,23 +25,23 @@ type RepoNode struct { // // NewRepoNode constructor // func NewRepoNode() *RepoNode { // return &RepoNode{ -// Data: make(map[string]interface{}, 0), // set initial size to zero explicitely? +// Data: make(map[string]interface{}, 0), // set initial size to zero explicitly? // Nodes: make(map[string]*RepoNode, 0), // } // } // WireParents helper method to reference from child to parent in a tree // recursively -func (node *RepoNode) WireParents() { - for _, childNode := range node.Nodes { - childNode.parent = node +func (n *RepoNode) WireParents() { + for _, childNode := range n.Nodes { + childNode.parent = n childNode.WireParents() } } // InPath is the given node in a path -func (node *RepoNode) InPath(path []*Item) bool { - myParentID := node.parent.ID +func (n *RepoNode) InPath(path []*Item) bool { + myParentID := n.parent.ID for _, pathItem := range path { if pathItem.ID == myParentID { return true @@ -51,17 +51,16 @@ func (node *RepoNode) InPath(path []*Item) bool { } // GetPath get a path for a repo node -func (node *RepoNode) GetPath(dataFields []string) []*Item { - +func (n *RepoNode) GetPath(dataFields []string) []*Item { var ( - parentNode = node.parent + parentNode = n.parent pathLength = 0 ) for parentNode != nil { parentNode = parentNode.parent pathLength++ } - parentNode = node.parent + parentNode = n.parent var ( i = 0 @@ -81,19 +80,19 @@ func (node *RepoNode) GetPath(dataFields []string) []*Item { } // ToItem convert a repo node to a simple repo item -func (node *RepoNode) ToItem(dataFields []string) *Item { +func (n *RepoNode) ToItem(dataFields []string) *Item { item := NewItem() - item.ID = node.ID - item.Name = node.Name - item.MimeType = node.MimeType - item.Hidden = node.Hidden - item.URI = node.URI - item.Groups = node.Groups + item.ID = n.ID + item.Name = n.Name + item.MimeType = n.MimeType + item.Hidden = n.Hidden + item.URI = n.URI + item.Groups = n.Groups if dataFields == nil { - item.Data = node.Data + item.Data = n.Data } else { for _, dataField := range dataFields { - if data, ok := node.Data[dataField]; ok { + if data, ok := n.Data[dataField]; ok { item.Data[dataField] = data } } @@ -102,23 +101,23 @@ func (node *RepoNode) ToItem(dataFields []string) *Item { } // GetParent get the parent node of a node -func (node *RepoNode) GetParent() *RepoNode { - return node.parent +func (n *RepoNode) GetParent() *RepoNode { + return n.parent } // AddNode adds a named child node -func (node *RepoNode) AddNode(name string, childNode *RepoNode) *RepoNode { - node.Nodes[name] = childNode - return node +func (n *RepoNode) AddNode(name string, childNode *RepoNode) *RepoNode { + n.Nodes[name] = childNode + return n } // IsOneOfTheseMimeTypes is the node one of the given mime types -func (node *RepoNode) IsOneOfTheseMimeTypes(mimeTypes []string) bool { +func (n *RepoNode) IsOneOfTheseMimeTypes(mimeTypes []string) bool { if len(mimeTypes) == 0 { return true } for _, mimeType := range mimeTypes { - if mimeType == node.MimeType { + if mimeType == n.MimeType { return true } } @@ -127,14 +126,14 @@ func (node *RepoNode) IsOneOfTheseMimeTypes(mimeTypes []string) bool { // CanBeAccessedByGroups can this node be accessed by at least one the given // groups -func (node *RepoNode) CanBeAccessedByGroups(groups []string) bool { +func (n *RepoNode) CanBeAccessedByGroups(groups []string) bool { // no groups set on node => anybody can access it - if len(node.Groups) == 0 { + if len(n.Groups) == 0 { return true } for _, group := range groups { - for _, myGroup := range node.Groups { + for _, myGroup := range n.Groups { if group == myGroup { return true } @@ -144,10 +143,10 @@ func (node *RepoNode) CanBeAccessedByGroups(groups []string) bool { } // PrintNode essentially a recursive dump -func (node *RepoNode) PrintNode(id string, level int) { +func (n *RepoNode) PrintNode(id string, level int) { prefix := strings.Repeat(Indent, level) - fmt.Printf("%s %s %s:\n", prefix, id, node.Name) - for key, childNode := range node.Nodes { + fmt.Printf("%s %s %s:\n", prefix, id, n.Name) + for key, childNode := range n.Nodes { childNode.PrintNode(key, level+1) } } diff --git a/content/sitecontent.go b/content/sitecontent.go index 68a78e2..c53f11a 100644 --- a/content/sitecontent.go +++ b/content/sitecontent.go @@ -1,17 +1,5 @@ package content -// Status status type SiteContent respnses -type Status int - -const ( - // StatusOk we found content - StatusOk Status = 200 - // StatusForbidden we found content but you mst not access it - StatusForbidden = 403 - // StatusNotFound we did not find content - StatusNotFound = 404 -) - // SiteContent resolved content for a site type SiteContent struct { Status Status `json:"status"` diff --git a/content/status.go b/content/status.go new file mode 100644 index 0000000..63843db --- /dev/null +++ b/content/status.go @@ -0,0 +1,13 @@ +package content + +// Status status type SiteContent respnses +type Status int + +const ( + // StatusOk we found content + StatusOk Status = 200 + // StatusForbidden we found content but you mst not access it + StatusForbidden = 403 + // StatusNotFound we did not find content + StatusNotFound = 404 +) diff --git a/contentserver.go b/contentserver.go deleted file mode 100644 index 26edacb..0000000 --- a/contentserver.go +++ /dev/null @@ -1,111 +0,0 @@ -package main - -import ( - "flag" - "fmt" - _ "net/http/pprof" - "os" - "runtime/debug" - "time" - - . "github.com/foomo/contentserver/logger" - "github.com/foomo/contentserver/metrics" - "github.com/foomo/contentserver/server" - "github.com/foomo/contentserver/status" - "go.uber.org/zap" -) - -const ( - ServiceName = "Content Server" - DefaultHealthzHandlerAddress = ":8080" - DefaultPrometheusListener = ":9200" -) - -var ( - flagAddress = flag.String("address", "", "address to bind socket server host:port") - flagWebserverAddress = flag.String("webserver-address", "", "address to bind web server host:port, when empty no webserver will be spawned") - flagWebserverPath = flag.String("webserver-path", "/contentserver", "path to export the webserver on - useful when behind a proxy") - flagVarDir = flag.String("var-dir", "/var/lib/contentserver", "where to put my data") - flagPrometheusListener = flag.String("prometheus-listener", getenv("PROMETHEUS_LISTENER", DefaultPrometheusListener), "address for the prometheus listener") - flagRepositoryTimeoutDuration = flag.Duration("repository-timeout-duration", server.DefaultRepositoryTimeout, "timeout duration for the contentserver") - - flagPoll = flag.Bool("poll", false, "if true, the address arg will be used to periodically poll the content url") - - // debugging / profiling - flagDebug = flag.Bool("debug", false, "toggle debug mode") - flagFreeOSMem = flag.Int("free-os-mem", 0, "free OS mem every X minutes") - flagHeapDump = flag.Int("heap-dump", 0, "dump heap every X minutes") -) - -func getenv(env, fallback string) string { - if value, ok := os.LookupEnv(env); ok { - return value - } - return fallback -} - -func exitUsage(code int) { - fmt.Println("Usage:", os.Args[0], "http(s)://your-content-server/path/to/content.json") - flag.PrintDefaults() - os.Exit(code) -} - -func main() { - flag.Parse() - - SetupLogging(*flagDebug, "contentserver.log") - - if *flagFreeOSMem > 0 { - Log.Info("freeing OS memory every $interval minutes", zap.Int("interval", *flagFreeOSMem)) - go func() { - for { - time.Sleep(time.Duration(*flagFreeOSMem) * time.Minute) - Log.Info("FreeOSMemory") - debug.FreeOSMemory() - } - }() - } - - if *flagHeapDump > 0 { - Log.Info("dumping heap every $interval minutes", zap.Int("interval", *flagHeapDump)) - go func() { - for { - time.Sleep(time.Duration(*flagFreeOSMem) * time.Minute) - Log.Info("HeapDump") - f, err := os.Create("heapdump") - if err != nil { - panic("failed to create heap dump file") - } - debug.WriteHeapDump(f.Fd()) - err = f.Close() - if err != nil { - panic("failed to create heap dump file") - } - } - }() - } - - if len(flag.Args()) == 1 { - fmt.Println(*flagAddress, flag.Arg(0)) - - // kickoff metric handlers - go metrics.RunPrometheusHandler(*flagPrometheusListener) - go status.RunHealthzHandlerListener(DefaultHealthzHandlerAddress, ServiceName) - - err := server.RunServerSocketAndWebServer( - flag.Arg(0), - *flagAddress, - *flagWebserverAddress, - *flagWebserverPath, - *flagVarDir, - *flagRepositoryTimeoutDuration, - *flagPoll, - ) - if err != nil { - fmt.Println("exiting with error", err) - os.Exit(1) - } - } else { - exitUsage(1) - } -} diff --git a/graphics/Horizontal Update.svg b/docs/assets/Horizontal Update.svg similarity index 100% rename from graphics/Horizontal Update.svg rename to docs/assets/Horizontal Update.svg diff --git a/graphics/Horizontal.svg b/docs/assets/Horizontal.svg similarity index 100% rename from graphics/Horizontal.svg rename to docs/assets/Horizontal.svg diff --git a/graphics/Overview.svg b/docs/assets/Overview.svg similarity index 100% rename from graphics/Overview.svg rename to docs/assets/Overview.svg diff --git a/graphics/Update-Flow.svg b/docs/assets/Update-Flow.svg similarity index 100% rename from graphics/Update-Flow.svg rename to docs/assets/Update-Flow.svg diff --git a/go.mod b/go.mod index 084c691..2cd29e4 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/foomo/contentserver go 1.21 require ( + github.com/foomo/keel v0.17.4-0.20240315155218-1be83011f59e // #190 (1be8301) update otel github.com/google/uuid v1.6.0 github.com/json-iterator/go v1.1.12 github.com/pkg/errors v0.9.1 @@ -13,16 +14,117 @@ require ( ) require ( + github.com/spf13/cobra v1.8.0 + github.com/spf13/viper v1.18.2 + golang.org/x/sync v0.6.0 +) + +require ( + cloud.google.com/go v0.111.0 // indirect + cloud.google.com/go/compute v1.23.3 // indirect + cloud.google.com/go/compute/metadata v0.2.3 // indirect + cloud.google.com/go/firestore v1.14.0 // indirect + cloud.google.com/go/longrunning v0.5.4 // indirect + github.com/armon/go-metrics v0.4.1 // indirect + github.com/avast/retry-go v3.0.0+incompatible // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect + github.com/coreos/go-semver v0.3.0 // indirect + github.com/coreos/go-systemd/v22 v22.3.2 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/fatih/color v1.14.1 // indirect + github.com/fbiville/markdown-table-formatter v0.3.0 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/fsnotify/fsnotify v1.7.0 // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/go-ole/go-ole v1.2.6 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang-jwt/jwt v3.2.2+incompatible // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/google/s2a-go v0.1.7 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect + github.com/googleapis/gax-go/v2 v2.12.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0 // indirect + github.com/hashicorp/consul/api v1.25.1 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-hclog v1.5.0 // indirect + github.com/hashicorp/go-immutable-radix v1.3.1 // indirect + github.com/hashicorp/go-rootcerts v1.0.2 // indirect + github.com/hashicorp/golang-lru v0.5.4 // indirect + github.com/hashicorp/hcl v1.0.0 // indirect + github.com/hashicorp/serf v0.10.1 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/klauspost/compress v1.17.2 // indirect + github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect + github.com/magiconair/properties v1.8.7 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.17 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_model v0.5.0 // indirect + github.com/nats-io/nats.go v1.33.1 // indirect + github.com/nats-io/nkeys v0.4.7 // indirect + github.com/nats-io/nuid v1.0.1 // indirect + github.com/pelletier/go-toml/v2 v2.1.0 // indirect + github.com/philhofer/fwd v1.1.2 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c // indirect + github.com/prometheus/client_model v0.6.0 // indirect github.com/prometheus/common v0.48.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect - golang.org/x/sys v0.16.0 // indirect + github.com/sagikazarmark/crypt v0.17.0 // indirect + github.com/sagikazarmark/locafero v0.4.0 // indirect + github.com/sagikazarmark/slog-shim v0.1.0 // indirect + github.com/shirou/gopsutil/v3 v3.24.1 // indirect + github.com/shoenig/go-m1cpu v0.1.6 // indirect + github.com/sony/gobreaker v0.5.0 // indirect + github.com/sourcegraph/conc v0.3.0 // indirect + github.com/spf13/afero v1.11.0 // indirect + github.com/spf13/cast v1.6.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/subosito/gotenv v1.6.0 // indirect + github.com/tinylib/msgp v1.1.9 // indirect + github.com/tklauser/go-sysconf v0.3.12 // indirect + github.com/tklauser/numcpus v0.6.1 // indirect + github.com/yusufpapurcu/wmi v1.2.3 // indirect + go.etcd.io/etcd/api/v3 v3.5.10 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.5.10 // indirect + go.etcd.io/etcd/client/v2 v2.305.10 // indirect + go.etcd.io/etcd/client/v3 v3.5.10 // indirect + go.opencensus.io v0.24.0 // indirect + go.opentelemetry.io/contrib/instrumentation/host v0.49.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect + go.opentelemetry.io/contrib/instrumentation/runtime v0.49.0 // indirect + go.opentelemetry.io/otel v1.24.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.24.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.24.0 // indirect + go.opentelemetry.io/otel/exporters/prometheus v0.46.0 // indirect + go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.24.0 // indirect + go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.24.0 // indirect + go.opentelemetry.io/otel/metric v1.24.0 // indirect + go.opentelemetry.io/otel/sdk v1.24.0 // indirect + go.opentelemetry.io/otel/sdk/metric v1.24.0 // indirect + go.opentelemetry.io/otel/trace v1.24.0 // indirect + go.opentelemetry.io/proto/otlp v1.1.0 // indirect + golang.org/x/crypto v0.21.0 // indirect + golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect + golang.org/x/net v0.21.0 // indirect + golang.org/x/oauth2 v0.16.0 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/text v0.14.0 // indirect + golang.org/x/time v0.5.0 // indirect + google.golang.org/api v0.153.0 // indirect + google.golang.org/appengine v1.6.8 // indirect + google.golang.org/genproto v0.0.0-20231212172506-995d672761c0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20240102182953-50ed04b92917 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240102182953-50ed04b92917 // indirect + google.golang.org/grpc v1.61.1 // indirect google.golang.org/protobuf v1.32.0 // indirect + gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index ca5c44e..b25ce2f 100644 --- a/go.sum +++ b/go.sum @@ -1,56 +1,549 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.111.0 h1:YHLKNupSD1KqjDbQ3+LVdQ81h/UJbJyZG203cEfnQgM= +cloud.google.com/go v0.111.0/go.mod h1:0mibmpKP1TyOOFYQY5izo0LnT+ecvOQ0Sg3OdmMiNRU= +cloud.google.com/go/compute v1.23.3 h1:6sVlXXBmbd7jNX0Ipq0trII3e4n1/MsADLK6a+aiVlk= +cloud.google.com/go/compute v1.23.3/go.mod h1:VCgBUoMnIVIR0CscqQiPJLAG25E3ZRZMzcFZeQ+h8CI= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= +cloud.google.com/go/firestore v1.14.0 h1:8aLcKnMPoldYU3YHgu4t2exrKhLQkqaXAGqT0ljrFVw= +cloud.google.com/go/firestore v1.14.0/go.mod h1:96MVaHLsEhbvkBEdZgfN+AS/GIkco1LRpH9Xp9YZfzQ= +cloud.google.com/go/longrunning v0.5.4 h1:w8xEcbZodnA2BbW6sVirkkoC+1gP8wS57EUUgGS0GVg= +cloud.google.com/go/longrunning v0.5.4/go.mod h1:zqNVncI0BOP8ST6XQD1+VcvuShMmq7+xFSzOL++V0dI= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= +github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= +github.com/armon/go-metrics v0.4.1 h1:hR91U9KYmb6bLBYLQjyM+3j+rcd/UhE+G78SFnF8gJA= +github.com/armon/go-metrics v0.4.1/go.mod h1:E6amYzXo6aW1tqzoZGT755KkbgrJsSdpwZ+3JqfkOG4= +github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/armon/go-radix v1.0.0/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8= +github.com/avast/retry-go v3.0.0+incompatible h1:4SOWQ7Qs+oroOTQOYnAHqelpCO0biHSxpiH9JdtuBj0= +github.com/avast/retry-go v3.0.0+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= +github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= +github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= +github.com/circonus-labs/circonusllhist v0.1.3/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= +github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fatih/color v1.9.0/go.mod h1:eQcE1qtQxscV5RaZvpXrrb8Drkc3/DdQ+uUYCNjL+zU= +github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fatih/color v1.14.1 h1:qfhVLaG5s+nCROl1zJsZRxFeYrHLqWroPOQ8BWiNb4w= +github.com/fatih/color v1.14.1/go.mod h1:2oHN61fhTpgcxD3TSWCgKDiH1+x4OiDVVGH8WlgGZGg= +github.com/fbiville/markdown-table-formatter v0.3.0 h1:PIm1UNgJrFs8q1htGTw+wnnNYvwXQMMMIKNZop2SSho= +github.com/fbiville/markdown-table-formatter v0.3.0/go.mod h1:q89TDtSEVDdTaufgSbfHpNVdPU/bmfvqNkrC5HagmLY= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/foomo/keel v0.17.4-0.20240315155218-1be83011f59e h1:cAUtpoQqHHhuXPMY930rG3zzBx2Dp86/3l9gp04yPHU= +github.com/foomo/keel v0.17.4-0.20240315155218-1be83011f59e/go.mod h1:+90rU3I9pErPp3n28ZaSB708n+nfPEf5cVP1lqn5Sf8= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= +github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= +github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= +github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= +github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= +github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= +github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= +github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= +github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= +github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= +github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w= +github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0= +github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= +github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= +github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= +github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= +github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= +github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= +github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= +github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0 h1:Wqo399gCIufwto+VfwCSvsnfGpF/w5E9CNxSwbpD6No= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.0/go.mod h1:qmOFXW2epJhM0qSnUUYpldc7gVz2KMQwJ/QYCDIa7XU= +github.com/hashicorp/consul/api v1.25.1 h1:CqrdhYzc8XZuPnhIYZWH45toM0LB9ZeYr/gvpLVI3PE= +github.com/hashicorp/consul/api v1.25.1/go.mod h1:iiLVwR/htV7mas/sy0O+XSuEnrdBUUydemjxcUrAt4g= +github.com/hashicorp/consul/sdk v0.14.1 h1:ZiwE2bKb+zro68sWzZ1SgHF3kRMBZ94TwOCFRF4ylPs= +github.com/hashicorp/consul/sdk v0.14.1/go.mod h1:vFt03juSzocLRFo59NkeQHHmQa6+g7oU0pfzdI1mUhg= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I= +github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v1.5.0 h1:bI2ocEMgcVlz55Oj1xZNBsVi900c7II+fWDyV9o+13c= +github.com/hashicorp/go-hclog v1.5.0/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc= +github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= +github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-msgpack v0.5.5 h1:i9R9JSrqIz0QVLz3sz+i3YJdT7TTSLcfLLzJi9aZTuI= +github.com/hashicorp/go-msgpack v0.5.5/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= +github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= +github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= +github.com/hashicorp/go-rootcerts v1.0.2 h1:jzhAVGtqPKbwpyCPELlgNWhE1znq+qwJtW5Oi2viEzc= +github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= +github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= +github.com/hashicorp/go-sockaddr v1.0.2 h1:ztczhD1jLxIRjVejw8gFomI1BQZOe2WoVOu0SyteCQc= +github.com/hashicorp/go-sockaddr v1.0.2/go.mod h1:rB4wwRAUzs07qva3c5SdrY/NEtAUjGlgmH/UkBUC97A= +github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= +github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.3 h1:2gKiV6YVmrJ1i2CKKa9obLvRieoRGviZFL26PcT/Co8= +github.com/hashicorp/go-uuid v1.0.3/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-version v1.2.1 h1:zEfKbn2+PDgroKdiOzqiE8rsmLqU2uwi5PB5pBJ3TkI= +github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.4 h1:YDjusn29QI/Das2iO9M0BHnIbxPeyuCHsjMW+lJfyTc= +github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= +github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= +github.com/hashicorp/mdns v1.0.4/go.mod h1:mtBihi+LeNXGtG8L9dX59gAEa12BDtBQSp4v/YAJqrc= +github.com/hashicorp/memberlist v0.5.0 h1:EtYPN8DpAURiapus508I4n9CzHs2W+8NZGbmmR/prTM= +github.com/hashicorp/memberlist v0.5.0/go.mod h1:yvyXLpo0QaGE59Y7hDTsTzDD25JYBZ4mHgHUZ8lrOI0= +github.com/hashicorp/serf v0.10.1 h1:Z1H2J60yRKvfDYAOZLd2MU0ND4AH/WDz7xYHDWQsIPY= +github.com/hashicorp/serf v0.10.1/go.mod h1:yL2t6BqATOLGc5HF7qbFkTfXoPIY0WZdWHfEvMqbG+4= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= +github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= +github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= +github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= +github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= +github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOAqxQCu2WE= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/miekg/dns v1.1.26/go.mod h1:bPDLeHnStXmXAq1m/Ch/hvfNHr14JKNPMBo3VZKjuso= +github.com/miekg/dns v1.1.41 h1:WMszZWJG0XmzbK9FEmzH2TVcqYzFesusSIB41b8KHxY= +github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= +github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= +github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= +github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= +github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nats-io/nats.go v1.33.1 h1:8TxLZZ/seeEfR97qV0/Bl939tpDnt2Z2fK3HkPypj70= +github.com/nats-io/nats.go v1.33.1/go.mod h1:Ubdu4Nh9exXdSz0RVWRFBbRfrbSxOYd26oF0wkWclB8= +github.com/nats-io/nkeys v0.4.7 h1:RwNJbbIdYCoClSDNY7QVKZlyb/wfT6ugvFCiKy6vDvI= +github.com/nats-io/nkeys v0.4.7/go.mod h1:kqXRgRDPlGy7nGaEDMuYzmiJCIAAWDK0IMBtDmGD0nc= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= +github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= +github.com/pelletier/go-toml/v2 v2.1.0 h1:FnwAJ4oYMvbT/34k9zzHuZNrhlz48GB3/s6at6/MHO4= +github.com/pelletier/go-toml/v2 v2.1.0/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc= +github.com/philhofer/fwd v1.1.2 h1:bnDivRJ1EWPjUIRXV5KfORO897HTbpFAQddBdE8t7Gw= +github.com/philhofer/fwd v1.1.2/go.mod h1:qkPdfjR2SIEbspLqpe1tO4n5yICnr2DY7mqEx2tUTP0= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= +github.com/posener/complete v1.2.3/go.mod h1:WZIdtGGp+qx0sLrYKtIRAruyNpv6hFCicSgv7Sy7s/s= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c h1:ncq/mPwQF4JjgDlrVEn3C11VoGHZN7m8qihwgMEtzYw= +github.com/power-devops/perfstat v0.0.0-20210106213030-5aafc221ea8c/go.mod h1:OmDBASR4679mdNQnz2pUhc2G8CO2JrUAVFDRBDP/hJE= +github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= +github.com/prometheus/client_golang v1.4.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_golang v1.19.0 h1:ygXvpU1AoN1MhdzckN+PyD9QJOSD4x7kmXYlnfbA6JU= github.com/prometheus/client_golang v1.19.0/go.mod h1:ZRM9uEAypZakd+q/x7+gmsvXdURP+DABIEIjnmDdp+k= -github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= -github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= +github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.6.0 h1:k1v3CzpSRUTrKMppY35TLwPvxHqBu0bYgxZzqGIgaos= +github.com/prometheus/client_model v0.6.0/go.mod h1:NTQHnmxFpouOD0DpvP4XujX3CdOAGQPoaGhyTchlyt8= +github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= +github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/common v0.48.0 h1:QO8U2CdOzSn1BBsmXJXduaaW+dY/5QLjfB8svtSzKKE= github.com/prometheus/common v0.48.0/go.mod h1:0/KsvlIEfPQCQ5I2iNSAWKPZziNCvRs5EC6ILDTlAPc= +github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/sagikazarmark/crypt v0.17.0 h1:ZA/7pXyjkHoK4bW4mIdnCLvL8hd+Nrbiw7Dqk7D4qUk= +github.com/sagikazarmark/crypt v0.17.0/go.mod h1:SMtHTvdmsZMuY/bpZoqokSoChIrcJ/epOxZN58PbZDg= +github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= +github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= +github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= +github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= +github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/shirou/gopsutil/v3 v3.24.1 h1:R3t6ondCEvmARp3wxODhXMTLC/klMa87h2PHUw5m7QI= +github.com/shirou/gopsutil/v3 v3.24.1/go.mod h1:UU7a2MSBQa+kW1uuDq8DeEBS8kmrnQwsv2b5O513rwU= +github.com/shoenig/go-m1cpu v0.1.6 h1:nxdKQNcEB6vzgA2E2bvzKIYRuNj7XNJ4S/aRSwKzFtM= +github.com/shoenig/go-m1cpu v0.1.6/go.mod h1:1JJMcUBvfNwpq05QDQVAnx3gUHr9IYF7GNg9SUEw2VQ= +github.com/shoenig/test v0.6.4 h1:kVTaSd7WLz5WZ2IaoM0RSzRsUD+m8wRR+5qvntpn4LU= +github.com/shoenig/test v0.6.4/go.mod h1:byHiCGXqrVaflBLAMq/srcZIHynQPQgeyvkvXnjqq0k= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sony/gobreaker v0.5.0 h1:dRCvqm0P490vZPmy7ppEk2qCnCieBooFJ+YoXGYB+yg= +github.com/sony/gobreaker v0.5.0/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= +github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= +github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= +github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= +github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= +github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= +github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= +github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= +github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/viper v1.18.2 h1:LUXCnvUvSM6FXAsj6nnfc8Q2tp1dIgUfY9Kc8GsSOiQ= +github.com/spf13/viper v1.18.2/go.mod h1:EKmWIqdnk5lOcmR72yw6hS+8OPYcwD0jteitLMVB+yk= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= +github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/subosito/gotenv v1.6.0 h1:9NlTDc1FTs4qu0DDq7AEtTPNw6SVm7uBMsUCUjABIf8= +github.com/subosito/gotenv v1.6.0/go.mod h1:Dk4QP5c2W3ibzajGcXpNraDfq2IrhjMIvMSWPKKo0FU= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tinylib/msgp v1.1.9 h1:SHf3yoO2sGA0veCJeCBYLHuttAVFHGm2RHgNodW7wQU= +github.com/tinylib/msgp v1.1.9/go.mod h1:BCXGB54lDD8qUEPmiG0cQQUANC4IUQyB2ItS2UDlO/k= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= +github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= +github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= +github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= +go.etcd.io/etcd/api/v3 v3.5.10 h1:szRajuUUbLyppkhs9K6BRtjY37l66XQQmw7oZRANE4k= +go.etcd.io/etcd/api/v3 v3.5.10/go.mod h1:TidfmT4Uycad3NM/o25fG3J07odo4GBB9hoxaodFCtI= +go.etcd.io/etcd/client/pkg/v3 v3.5.10 h1:kfYIdQftBnbAq8pUWFXfpuuxFSKzlmM5cSn76JByiT0= +go.etcd.io/etcd/client/pkg/v3 v3.5.10/go.mod h1:DYivfIviIuQ8+/lCq4vcxuseg2P2XbHygkKwFo9fc8U= +go.etcd.io/etcd/client/v2 v2.305.10 h1:MrmRktzv/XF8CvtQt+P6wLUlURaNpSDJHFZhe//2QE4= +go.etcd.io/etcd/client/v2 v2.305.10/go.mod h1:m3CKZi69HzilhVqtPDcjhSGp+kA1OmbNn0qamH80xjA= +go.etcd.io/etcd/client/v3 v3.5.10 h1:W9TXNZ+oB3MCd/8UjxHTWK5J9Nquw9fQBLJd5ne5/Ao= +go.etcd.io/etcd/client/v3 v3.5.10/go.mod h1:RVeBnDz2PUEZqTpgqwAtUd8nAPf5kjyFyND7P1VkOKc= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/contrib/instrumentation/host v0.49.0 h1:PHK4Cnis16iENFfqnzvuak5vfRl5L0UaTG2Z03vr3iI= +go.opentelemetry.io/contrib/instrumentation/host v0.49.0/go.mod h1:0XQuDAhohvWG6+cdmjX6aFbC4mGMjYf1xILFh5OUcEg= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= +go.opentelemetry.io/contrib/instrumentation/runtime v0.49.0 h1:dg9y+7ArpumB6zwImJv47RHfdgOGQ1EMkzP5vLkEnTU= +go.opentelemetry.io/contrib/instrumentation/runtime v0.49.0/go.mod h1:Ul4MtXqu/hJBM+v7a6dCF0nHwckPMLpIpLeCi4+zfdw= +go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= +go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0 h1:t6wl9SPayj+c7lEIFgm4ooDBZVb01IhLB4InpomhRw8= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.24.0/go.mod h1:iSDOcsnSA5INXzZtwaBPrKp/lWu/V14Dd+llD0oI2EA= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.24.0 h1:Mw5xcxMwlqoJd97vwPxA8isEaIoxsta9/Q51+TTJLGE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.24.0/go.mod h1:CQNu9bj7o7mC6U7+CA/schKEYakYXWr79ucDHTMGhCM= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.24.0 h1:Xw8U6u2f8DK2XAkGRFV7BBLENgnTGX9i4rQRxJf+/vs= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.24.0/go.mod h1:6KW1Fm6R/s6Z3PGXwSJN2K4eT6wQB3vXX6CVnYX9NmM= +go.opentelemetry.io/otel/exporters/prometheus v0.46.0 h1:I8WIFXR351FoLJYuloU4EgXbtNX2URfU/85pUPheIEQ= +go.opentelemetry.io/otel/exporters/prometheus v0.46.0/go.mod h1:ztwVUHe5DTR/1v7PeuGRnU5Bbd4QKYwApWmuutKsJSs= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.24.0 h1:JYE2HM7pZbOt5Jhk8ndWZTUWYOVift2cHjXVMkPdmdc= +go.opentelemetry.io/otel/exporters/stdout/stdoutmetric v1.24.0/go.mod h1:yMb/8c6hVsnma0RpsBMNo0fEiQKeclawtgaIaOp2MLY= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.24.0 h1:s0PHtIkN+3xrbDOpt2M8OTG92cWqUESvzh2MxiR5xY8= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.24.0/go.mod h1:hZlFbDbRt++MMPCCfSJfmhkGIWnX1h3XjkfxZUjLrIA= +go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= +go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= +go.opentelemetry.io/otel/sdk v1.24.0 h1:YMPPDNymmQN3ZgczicBY3B6sf9n62Dlj9pWD3ucgoDw= +go.opentelemetry.io/otel/sdk v1.24.0/go.mod h1:KVrIYw6tEubO9E96HQpcmpTKDVn9gdv35HoYiQWGDFg= +go.opentelemetry.io/otel/sdk/metric v1.24.0 h1:yyMQrPzF+k88/DbH7o4FMAs80puqd+9osbiBrJrz/w8= +go.opentelemetry.io/otel/sdk/metric v1.24.0/go.mod h1:I6Y5FjH6rvEnTTAYQz3Mmv2kl6Ek5IIrmwTLqMrrOE0= +go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= +go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= +go.opentelemetry.io/proto/otlp v1.1.0 h1:2Di21piLrCqJ3U3eXGCTPHE9R8Nh+0uglSnOyxikMeI= +go.opentelemetry.io/proto/otlp v1.1.0/go.mod h1:GpBHCBWiqvVLDqmHZsoMM3C5ySeKTC7ej/RNTae6MdY= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= -golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210410081132-afb366fc7cd1/go.mod h1:9tjilg8BloeKEkVJvy7fQ90B1CfIiPueXVOjqfkSzI8= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4= +golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.16.0 h1:aDkGMBSYxElaoP81NpoUoz2oo2R2wHdZpGToUxfyQrQ= +golang.org/x/oauth2 v0.16.0/go.mod h1:hqZ+0LWXsiVoZpeld6jVt06P3adbS2Uu911W1SsJv2o= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190922100055-0a153f010e69/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210303074136-134d130e1a04/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190907020128-2ca718005c18/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.153.0 h1:N1AwGhielyKFaUqH07/ZSIQR3uNPcV7NVw0vj+j4iR4= +google.golang.org/api v0.153.0/go.mod h1:3qNJX5eOmhiWYc67jRA/3GsDw97UFb5ivv7Y2PrriAY= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= +google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20231212172506-995d672761c0 h1:YJ5pD9rF8o9Qtta0Cmy9rdBwkSjrTCT6XTiUQVOtIos= +google.golang.org/genproto v0.0.0-20231212172506-995d672761c0/go.mod h1:l/k7rMz0vFTBPy+tFSGvXEd3z+BcoG1k7EHbqm+YBsY= +google.golang.org/genproto/googleapis/api v0.0.0-20240102182953-50ed04b92917 h1:rcS6EyEaoCO52hQDupoSfrxI3R6C2Tq741is7X8OvnM= +google.golang.org/genproto/googleapis/api v0.0.0-20240102182953-50ed04b92917/go.mod h1:CmlNWB9lSezaYELKS5Ym1r44VrrbPUa7JTvw+6MbpJ0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240102182953-50ed04b92917 h1:6G8oQ016D88m1xAKljMlBOOGWDZkes4kMhgGFlf8WcQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240102182953-50ed04b92917/go.mod h1:xtjpI3tXFPP051KaWnhvxkiubL/6dJ18vLVf7q2pTOU= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= +google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.61.1 h1:kLAiWrZs7YeDM6MumDe7m3y4aM6wacLzM1Y/wiLP9XY= +google.golang.org/grpc v1.61.1/go.mod h1:VUbo7IFqmF1QtCAstipjG0GIoq49KvMe9+h1jFLBNJs= +google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= +google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= +google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= +google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= +google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= +google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= +google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= +gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/logger/log.go b/logger/log.go deleted file mode 100644 index e5a7b3f..0000000 --- a/logger/log.go +++ /dev/null @@ -1,43 +0,0 @@ -package logger - -import ( - "log" - "os" - - "go.uber.org/zap" -) - -var ( - // Log is the logger instance exposed by this package - // call Setup() prior to using it - // want JSON output? Set LOG_JSON env var to 1! - Log *zap.Logger -) - -// SetupLogging configures the logger -func SetupLogging(debug bool, outputPath string) { - - var ( - zc zap.Config - err error - ) - - if debug { - zc = zap.NewDevelopmentConfig() - zc.OutputPaths = append(zc.OutputPaths, outputPath) - } else { - zc = zap.NewProductionConfig() - zc.OutputPaths = append(zc.OutputPaths, outputPath) - } - - if os.Getenv("LOG_JSON") == "1" { - zc.Encoding = "json" - } else { - zc.Encoding = "console" - } - - Log, err = zc.Build() - if err != nil { - log.Fatalf("can't initialize zap logger: %v", err) - } -} diff --git a/main.go b/main.go new file mode 100644 index 0000000..f68089f --- /dev/null +++ b/main.go @@ -0,0 +1,9 @@ +package main + +import ( + "github.com/foomo/contentserver/cmd" +) + +func main() { + cmd.Execute() +} diff --git a/metrics/prometheus.go b/metrics/prometheus.go deleted file mode 100644 index bcf892c..0000000 --- a/metrics/prometheus.go +++ /dev/null @@ -1,21 +0,0 @@ -package metrics - -import ( - "net/http" - - . "github.com/foomo/contentserver/logger" - "github.com/prometheus/client_golang/prometheus/promhttp" - "go.uber.org/zap" -) - -const metricsRoute = "/metrics" - -func RunPrometheusHandler(listener string) { - Log.Info("starting prometheus handler on", - zap.String("address", listener), - zap.String("route", metricsRoute), - ) - Log.Error("prometheus listener failed", - zap.Error(http.ListenAndServe(listener, promhttp.Handler())), - ) -} diff --git a/pkg/README.md b/pkg/README.md deleted file mode 100644 index cec37b8..0000000 --- a/pkg/README.md +++ /dev/null @@ -1,44 +0,0 @@ -Packaging & Deployment ----------------------- - -In order to build packages and upload to Package Cloud, please install the following requirements and run the make task. - -[Package Cloud Command Line Client](https://packagecloud.io/docs#cli_install) - -``` -$ gem install package_cloud -``` - -[FPM](https://github.com/jordansissel/fpm) - -``` -$ gem install fpm -``` - -Building package - -``` -$ make package -``` - -*NOTE: you will be prompted for Package Cloud credentials.* - -Testing -------- - -``` -$ git clone https://github.com/foomo/contentserver.git -$ cd contentserver -$ make test -``` - -Contributing ------------- - -In lieu of a formal styleguide, take care to maintain the existing coding style. Add unit tests and examples for any new or changed functionality. - -1. Fork it -2. Create your feature branch (`git checkout -b my-new-feature`\) -3. Commit your changes (`git commit -am 'Add some feature'`\) -4. Push to the branch (`git push origin my-new-feature`\) -5. Create new Pull Request diff --git a/pkg/build.sh b/pkg/build.sh deleted file mode 100755 index b4f4826..0000000 --- a/pkg/build.sh +++ /dev/null @@ -1,61 +0,0 @@ -#!/bin/bash - -USER="foomo" -NAME="content-server" -URL="http://www.foomo.org" -DESCRIPTION="Serves content tree structures very quickly through a json socket api." -LICENSE="LGPL-3.0" - -# get version -VERSION=`bin/content-server --version | sed 's/content-server //'` - -# create temp dir -TEMP=`pwd`/pkg/tmp -mkdir -p $TEMP - -package() -{ - OS=$1 - ARCH=$2 - TYPE=$3 - TARGET=$4 - - # copy license file - cp LICENSE $LICENSE - - # define source dir - SOURCE=`pwd`/pkg/${TYPE} - - # create build folder - BUILD=${TEMP}/${NAME}-${VERSION} - #rsync -rv --exclude **/.git* --exclude /*.sh $SOURCE/ $BUILD/ - - # build binary - GOOS=$OS GOARCH=$ARCH go build -o $BUILD/usr/local/bin/${NAME} - - # create package - fpm -s dir \ - -t $TYPE \ - --name $NAME \ - --maintainer $USER \ - --version $VERSION \ - --license $LICENSE \ - --description "${DESCRIPTION}" \ - --architecture $ARCH \ - --package $TEMP \ - --url "${URL}" \ - -C $BUILD \ - . - - # push - package_cloud push $TARGET $TEMP/${NAME}_${VERSION}_${ARCH}.${TYPE} - - # cleanup - rm -rf $TEMP - rm $LICENSE -} - -package linux amd64 deb foomo/content-server/ubuntu/precise -package linux amd64 deb foomo/content-server/ubuntu/trusty - -#package linux amd64 rpm diff --git a/pkg/handler/http.go b/pkg/handler/http.go new file mode 100644 index 0000000..5586712 --- /dev/null +++ b/pkg/handler/http.go @@ -0,0 +1,175 @@ +package handler + +import ( + "io" + "net/http" + "strings" + "time" + + "github.com/foomo/contentserver/pkg/metrics" + "github.com/foomo/contentserver/pkg/repo" + "github.com/foomo/contentserver/requests" + "github.com/foomo/contentserver/responses" + httputils "github.com/foomo/keel/utils/net/http" + "github.com/pkg/errors" + "go.uber.org/zap" +) + +type ( + HTTP struct { + l *zap.Logger + path string + repo *repo.Repo + } + HTTPOption func(*HTTP) +) + +// ------------------------------------------------------------------------------------------------ +// ~ Constructor +// ------------------------------------------------------------------------------------------------ + +// NewHTTP returns a shiny new web server +func NewHTTP(l *zap.Logger, repo *repo.Repo, opts ...HTTPOption) http.Handler { + inst := &HTTP{ + l: l.Named("http"), + path: "/contentserver", + repo: repo, + } + + for _, opt := range opts { + opt(inst) + } + + return inst +} + +// ------------------------------------------------------------------------------------------------ +// ~ Options +// ------------------------------------------------------------------------------------------------ + +func WithPath(v string) HTTPOption { + return func(o *HTTP) { + o.path = v + } +} + +// ------------------------------------------------------------------------------------------------ +// ~ Public methods +// ------------------------------------------------------------------------------------------------ + +func (h *HTTP) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodPost { + httputils.ServerError(h.l, w, r, http.StatusMethodNotAllowed, errors.New("method not allowed")) + return + } + if r.Body == nil { + httputils.BadRequestServerError(h.l, w, r, errors.New("empty request body")) + return + } + + bytes, err := io.ReadAll(r.Body) + if err != nil { + httputils.BadRequestServerError(h.l, w, r, errors.Wrap(err, "failed to read incoming request")) + return + } + + route := Route(strings.TrimPrefix(r.URL.Path, h.path+"/")) + if route == RouteGetRepo { + h.repo.WriteRepoBytes(w) + w.Header().Set("Content-Type", "application/json") + return + } + + reply, errReply := h.handleRequest(h.repo, route, bytes, "webserver") + if errReply != nil { + http.Error(w, errReply.Error(), http.StatusInternalServerError) + return + } + _, _ = w.Write(reply) +} + +// ------------------------------------------------------------------------------------------------ +// ~ Private methods +// ------------------------------------------------------------------------------------------------ + +func (h *HTTP) handleRequest(r *repo.Repo, handler Route, jsonBytes []byte, source string) ([]byte, error) { + start := time.Now() + + reply, err := h.executeRequest(r, handler, jsonBytes, source) + result := "success" + if err != nil { + result = "error" + } + + metrics.ServiceRequestCounter.WithLabelValues(string(handler), result, source).Inc() + metrics.ServiceRequestDuration.WithLabelValues(string(handler), result, source).Observe(time.Since(start).Seconds()) + + return reply, err +} + +func (h *HTTP) executeRequest(r *repo.Repo, handler Route, jsonBytes []byte, source string) (replyBytes []byte, err error) { + var ( + reply interface{} + apiErr error + jsonErr error + processIfJSONIsOk = func(err error, processingFunc func()) { + if err != nil { + jsonErr = err + return + } + processingFunc() + } + ) + metrics.ContentRequestCounter.WithLabelValues(source).Inc() + + // handle and process + switch handler { + // case HandlerGetRepo: // This case is handled prior to handleRequest being called. + // since the resulting bytes are written directly in to the http.ResponseWriter / net.Connection + case RouteGetURIs: + getURIRequest := &requests.URIs{} + processIfJSONIsOk(json.Unmarshal(jsonBytes, &getURIRequest), func() { + reply = r.GetURIs(getURIRequest.Dimension, getURIRequest.IDs) + }) + case RouteGetContent: + contentRequest := &requests.Content{} + processIfJSONIsOk(json.Unmarshal(jsonBytes, &contentRequest), func() { + reply, apiErr = r.GetContent(contentRequest) + }) + case RouteGetNodes: + nodesRequest := &requests.Nodes{} + processIfJSONIsOk(json.Unmarshal(jsonBytes, &nodesRequest), func() { + reply = r.GetNodes(nodesRequest) + }) + case RouteUpdate: + updateRequest := &requests.Update{} + processIfJSONIsOk(json.Unmarshal(jsonBytes, &updateRequest), func() { + reply = r.Update() + }) + default: + reply = responses.NewError(1, "unknown handler: "+string(handler)) + } + + // error handling + if jsonErr != nil { + h.l.Error("could not read incoming json", zap.Error(jsonErr)) + reply = responses.NewError(2, "could not read incoming json "+jsonErr.Error()) + } else if apiErr != nil { + h.l.Error("an API error occurred", zap.Error(apiErr)) + reply = responses.NewError(3, "internal error "+apiErr.Error()) + } + + return h.encodeReply(reply) +} + +// encodeReply takes an interface and encodes it as JSON +// it returns the resulting JSON and a marshalling error +func (h *HTTP) encodeReply(reply interface{}) (bytes []byte, err error) { + bytes, err = json.Marshal(map[string]interface{}{ + "reply": reply, + }) + if err != nil { + h.l.Error("could not encode reply", zap.Error(err)) + } + return +} diff --git a/pkg/handler/json.go b/pkg/handler/json.go new file mode 100644 index 0000000..ad4efda --- /dev/null +++ b/pkg/handler/json.go @@ -0,0 +1,7 @@ +package handler + +import ( + jsoniter "github.com/json-iterator/go" +) + +var json = jsoniter.ConfigCompatibleWithStandardLibrary diff --git a/pkg/handler/route.go b/pkg/handler/route.go new file mode 100644 index 0000000..be5cff3 --- /dev/null +++ b/pkg/handler/route.go @@ -0,0 +1,17 @@ +package handler + +// Route type +type Route string + +const ( + // RouteGetURIs get uris, many at once, to keep it fast + RouteGetURIs Route = "getURIs" + // RouteGetContent get (site) content + RouteGetContent Route = "getContent" + // RouteGetNodes get nodes + RouteGetNodes Route = "getNodes" + // RouteUpdate update repo + RouteUpdate Route = "update" + // RouteGetRepo get the whole repo + RouteGetRepo Route = "getRepo" +) diff --git a/pkg/handler/socket.go b/pkg/handler/socket.go new file mode 100644 index 0000000..6649bff --- /dev/null +++ b/pkg/handler/socket.go @@ -0,0 +1,269 @@ +package handler + +import ( + "bytes" + "errors" + "fmt" + "net" + "strconv" + "strings" + "time" + + "github.com/foomo/contentserver/requests" + "go.uber.org/zap" + + "github.com/foomo/contentserver/pkg/metrics" + "github.com/foomo/contentserver/pkg/repo" + "github.com/foomo/contentserver/responses" +) + +const sourceSocketServer = "socketserver" + +type Socket struct { + l *zap.Logger + repo *repo.Repo +} + +// ------------------------------------------------------------------------------------------------ +// ~ Constructor +// ------------------------------------------------------------------------------------------------ + +// NewSocket returns a shiny new socket server +func NewSocket(l *zap.Logger, repo *repo.Repo) *Socket { + inst := &Socket{ + l: l.Named("socket"), + repo: repo, + } + + return inst +} + +// ------------------------------------------------------------------------------------------------ +// ~ Public methods +// ------------------------------------------------------------------------------------------------ + +func (h *Socket) Serve(conn net.Conn) { + + defer func() { + if r := recover(); r != nil { + h.l.Error("panic in handle connection", zap.String("error", fmt.Sprint(r))) + } + }() + + h.l.Debug("socketServer.handleConnection") + metrics.NumSocketsGauge.WithLabelValues(conn.RemoteAddr().String()).Inc() + + var ( + headerBuffer [1]byte + header = "" + i = 0 + ) + for { + i++ + // fmt.Println("---->", i) + // let us read with 1 byte steps on conn until we find "{" + _, readErr := conn.Read(headerBuffer[0:]) + if readErr != nil { + h.l.Debug("looks like the client closed the connection", zap.Error(readErr)) + metrics.NumSocketsGauge.WithLabelValues(conn.RemoteAddr().String()).Dec() + return + } + // read next byte + current := headerBuffer[0:] + if string(current) == "{" { + // json has started + handler, jsonLength, headerErr := h.extractHandlerAndJSONLentgh(header) + // reset header + header = "" + if headerErr != nil { + h.l.Error("invalid request could not read header", zap.Error(headerErr)) + encodedErr, encodingErr := h.encodeReply(responses.NewError(4, "invalid header "+headerErr.Error())) + if encodingErr == nil { + h.writeResponse(conn, encodedErr) + } else { + h.l.Error("could not respond to invalid request", zap.Error(encodingErr)) + } + return + } + h.l.Debug("found json", zap.Int("length", jsonLength)) + if jsonLength > 0 { + + var ( + // let us try to read some json + jsonBytes = make([]byte, jsonLength) + jsonLengthCurrent = 1 + readRound = 0 + ) + + // that is "{" + jsonBytes[0] = 123 + + for jsonLengthCurrent < jsonLength { + readRound++ + readLength, jsonReadErr := conn.Read(jsonBytes[jsonLengthCurrent:jsonLength]) + if jsonReadErr != nil { + // @fixme we need to force a read timeout (SetReadDeadline?), if expected jsonLength is lower than really sent bytes (e.g. if client implements protocol wrong) + // @todo should we check for io.EOF here + h.l.Error("could not read json - giving up with this client connection", zap.Error(jsonReadErr)) + metrics.NumSocketsGauge.WithLabelValues(conn.RemoteAddr().String()).Dec() + return + } + jsonLengthCurrent += readLength + h.l.Debug("read cycle status", + zap.Int("jsonLengthCurrent", jsonLengthCurrent), + zap.Int("jsonLength", jsonLength), + zap.Int("readRound", readRound), + ) + } + + h.l.Debug("read json", zap.Int("length", len(jsonBytes))) + + h.writeResponse(conn, h.execute(handler, jsonBytes)) + // note: connection remains open + continue + } + h.l.Error("can not read empty json") + metrics.NumSocketsGauge.WithLabelValues(conn.RemoteAddr().String()).Dec() + return + } + // adding to header byte by byte + header += string(headerBuffer[0:]) + } +} + +// ------------------------------------------------------------------------------------------------ +// ~ Private methods +// ------------------------------------------------------------------------------------------------ + +func (h *Socket) extractHandlerAndJSONLentgh(header string) (route Route, jsonLength int, err error) { + headerParts := strings.Split(header, ":") + if len(headerParts) != 2 { + return "", 0, errors.New("invalid header") + } + jsonLength, err = strconv.Atoi(headerParts[1]) + if err != nil { + err = fmt.Errorf("could not parse length in header: %q", header) + } + return Route(headerParts[0]), jsonLength, err +} + +func (h *Socket) execute(route Route, jsonBytes []byte) (reply []byte) { + h.l.Debug("incoming json buffer", zap.Int("length", len(jsonBytes))) + + if route == RouteGetRepo { + var ( + b bytes.Buffer + ) + h.repo.WriteRepoBytes(&b) + return b.Bytes() + } + + reply, handlingError := h.handleRequest(h.repo, route, jsonBytes, sourceSocketServer) + if handlingError != nil { + h.l.Error("socketServer.execute failed", zap.Error(handlingError)) + } + return reply +} + +func (h *Socket) writeResponse(conn net.Conn, reply []byte) { + headerBytes := []byte(strconv.Itoa(len(reply))) + reply = append(headerBytes, reply...) + h.l.Debug("replying", zap.String("reply", string(reply))) + n, writeError := conn.Write(reply) + if writeError != nil { + h.l.Error("socketServer.writeResponse: could not write reply", zap.Error(writeError)) + return + } + if n < len(reply) { + h.l.Error("socketServer.writeResponse: write too short", + zap.Int("got", n), + zap.Int("expected", len(reply)), + ) + return + } + h.l.Debug("replied. waiting for next request on open connection") +} + +func (h *Socket) handleRequest(r *repo.Repo, route Route, jsonBytes []byte, source string) ([]byte, error) { + start := time.Now() + + reply, err := h.executeRequest(r, route, jsonBytes, source) + result := "success" + if err != nil { + result = "error" + } + + metrics.ServiceRequestCounter.WithLabelValues(string(route), result, source).Inc() + metrics.ServiceRequestDuration.WithLabelValues(string(route), result, source).Observe(time.Since(start).Seconds()) + + return reply, err +} + +func (h *Socket) executeRequest(r *repo.Repo, route Route, jsonBytes []byte, source string) (replyBytes []byte, err error) { + + var ( + reply interface{} + apiErr error + jsonErr error + processIfJSONIsOk = func(err error, processingFunc func()) { + if err != nil { + jsonErr = err + return + } + processingFunc() + } + ) + metrics.ContentRequestCounter.WithLabelValues(source).Inc() + + // handle and process + switch route { + // case RouteGetRepo: // This case is handled prior to handleRequest being called. + // since the resulting bytes are written directly in to the http.ResponseWriter / net.Connection + case RouteGetURIs: + getURIRequest := &requests.URIs{} + processIfJSONIsOk(json.Unmarshal(jsonBytes, &getURIRequest), func() { + reply = r.GetURIs(getURIRequest.Dimension, getURIRequest.IDs) + }) + case RouteGetContent: + contentRequest := &requests.Content{} + processIfJSONIsOk(json.Unmarshal(jsonBytes, &contentRequest), func() { + reply, apiErr = r.GetContent(contentRequest) + }) + case RouteGetNodes: + nodesRequest := &requests.Nodes{} + processIfJSONIsOk(json.Unmarshal(jsonBytes, &nodesRequest), func() { + reply = r.GetNodes(nodesRequest) + }) + case RouteUpdate: + updateRequest := &requests.Update{} + processIfJSONIsOk(json.Unmarshal(jsonBytes, &updateRequest), func() { + reply = r.Update() + }) + + default: + reply = responses.NewError(1, "unknown handler: "+string(route)) + } + + // error handling + if jsonErr != nil { + h.l.Error("could not read incoming json", zap.Error(jsonErr)) + reply = responses.NewError(2, "could not read incoming json "+jsonErr.Error()) + } else if apiErr != nil { + h.l.Error("an API error occured", zap.Error(apiErr)) + reply = responses.NewError(3, "internal error "+apiErr.Error()) + } + + return h.encodeReply(reply) +} + +// encodeReply takes an interface and encodes it as JSON +// it returns the resulting JSON and a marshalling error +func (h *Socket) encodeReply(reply interface{}) (replyBytes []byte, err error) { + replyBytes, err = json.Marshal(map[string]interface{}{ + "reply": reply, + }) + if err != nil { + h.l.Error("could not encode reply", zap.Error(err)) + } + return +} diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go new file mode 100644 index 0000000..6822a48 --- /dev/null +++ b/pkg/metrics/metrics.go @@ -0,0 +1,100 @@ +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +const ( + namespace = "contentserver" + + metricLabelHandler = "handler" + metricLabelStatus = "status" + metricLabelSource = "source" + metricLabelRemote = "remote" +) + +// Metrics is the structure that holds all prometheus metrics +var ( + // InvalidNodeTreeRequests counts the number of invalid tree node requests + InvalidNodeTreeRequests = newCounterVec( + "invalid_node_tree_request_count", + "Counts the number of invalid tree nodes for a specific node", + ) + // ServiceRequestCounter count the number of requests for each service function + ServiceRequestCounter = newCounterVec( + "service_request_count", + "Count of requests for each handler", + metricLabelHandler, metricLabelStatus, metricLabelSource, + ) + // ServiceRequestDuration observe the duration of requests for each service function + ServiceRequestDuration = newSummaryVec( + "service_request_duration_seconds", + "Seconds to unmarshal requests, execute a service function and marshal its reponses", + metricLabelHandler, metricLabelStatus, metricLabelSource, + ) + // UpdatesCompletedCounter count the number of rejected updates + UpdatesCompletedCounter = newCounterVec( + "updates_completed_count", + "Number of updates that were successfully completed", + ) + // UpdatesFailedCounter count the number of updates that had an error + UpdatesFailedCounter = newCounterVec( + "updates_failed_count", + "Number of updates that failed due to an error", + ) + // UpdateDuration observe the duration of each repo.update() call + UpdateDuration = newSummaryVec( + "update_duration_seconds", + "Duration in seconds for each successful repo.update() call", + ) + // ContentRequestCounter count the total number of content requests + ContentRequestCounter = newCounterVec( + "content_request_count", + "Number of requests for content", + metricLabelSource, + ) + // NumSocketsGauge keep track of the total number of open sockets + NumSocketsGauge = newGaugeVec( + "num_sockets_total", + "Total number of currently open socket connections", + metricLabelRemote, + ) + // HistoryPersistFailedCounter count the number of failed attempts to persist the content history + HistoryPersistFailedCounter = newCounterVec( + "history_persist_failed_count", + "Number of failures to store the content history on the filesystem", + ) +) + +func newSummaryVec(name, help string, labels ...string) *prometheus.SummaryVec { + vec := prometheus.NewSummaryVec( + prometheus.SummaryOpts{ + Namespace: namespace, + Name: name, + Help: help, + }, labels) + prometheus.MustRegister(vec) + return vec +} + +func newCounterVec(name, help string, labels ...string) *prometheus.CounterVec { + vec := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Name: name, + Help: help, + }, labels) + prometheus.MustRegister(vec) + return vec +} + +func newGaugeVec(name, help string, labels ...string) *prometheus.GaugeVec { + vec := prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: namespace, + Name: name, + Help: help, + }, labels) + prometheus.MustRegister(vec) + return vec +} diff --git a/pkg/repo/dimension.go b/pkg/repo/dimension.go new file mode 100644 index 0000000..b0bb302 --- /dev/null +++ b/pkg/repo/dimension.go @@ -0,0 +1,12 @@ +package repo + +import ( + "github.com/foomo/contentserver/content" +) + +// Dimension dimension in a repo +type Dimension struct { + Directory map[string]*content.RepoNode + URIDirectory map[string]*content.RepoNode + Node *content.RepoNode +} diff --git a/pkg/repo/history.go b/pkg/repo/history.go new file mode 100644 index 0000000..a89dc60 --- /dev/null +++ b/pkg/repo/history.go @@ -0,0 +1,172 @@ +package repo + +import ( + "bytes" + "errors" + "fmt" + "io" + "os" + "path" + "sort" + "strings" + "time" + + "go.uber.org/zap" +) + +const ( + HistoryRepoJSONPrefix = "contentserver-repo-" + HistoryRepoJSONSuffix = ".json" +) + +type ( + History struct { + l *zap.Logger + max int + varDir string + } + HistoryOption func(*History) +) + +// ------------------------------------------------------------------------------------------------ +// ~ Options +// ------------------------------------------------------------------------------------------------ + +func HistoryWithMax(v int) HistoryOption { + return func(o *History) { + o.max = v + } +} + +func HistoryWithVarDir(v string) HistoryOption { + return func(o *History) { + o.varDir = v + } +} + +// ------------------------------------------------------------------------------------------------ +// ~ Constructor +// ------------------------------------------------------------------------------------------------ + +func NewHistory(l *zap.Logger, opts ...HistoryOption) *History { + inst := &History{ + l: l, + max: 2, + varDir: "/var/lib/contentserver", + } + + for _, opt := range opts { + opt(inst) + } + + return inst +} + +// ------------------------------------------------------------------------------------------------ +// ~ Public methods +// ------------------------------------------------------------------------------------------------ + +func (h *History) Add(jsonBytes []byte) error { + var ( + // historiy file name + filename = path.Join(h.varDir, HistoryRepoJSONPrefix+time.Now().Format(time.RFC3339Nano)+HistoryRepoJSONSuffix) + err = os.WriteFile(filename, jsonBytes, 0600) + ) + if err != nil { + return err + } + + h.l.Debug("adding content backup", zap.String("file", filename)) + + // current filename + err = os.WriteFile(h.GetCurrentFilename(), jsonBytes, 0600) + if err != nil { + return err + } + + err = h.cleanup() + if err != nil { + h.l.Error("an error occurred while cleaning up my history", zap.Error(err)) + return err + } + + return nil +} + +func (h *History) GetCurrentFilename() string { + return path.Join(h.varDir, HistoryRepoJSONPrefix+"current"+HistoryRepoJSONSuffix) +} + +func (h *History) GetCurrent(buf *bytes.Buffer) (err error) { + f, err := os.Open(h.GetCurrentFilename()) + if err != nil { + return err + } + defer f.Close() + _, err = io.Copy(buf, f) + return err +} + +// ------------------------------------------------------------------------------------------------ +// ~ Private methods +// ------------------------------------------------------------------------------------------------ + +func (h *History) getHistory() (files []string, err error) { + fileInfos, err := os.ReadDir(h.varDir) + if err != nil { + return + } + currentName := h.GetCurrentFilename() + for _, f := range fileInfos { + if !f.IsDir() { + filename := f.Name() + if filename != currentName && (strings.HasPrefix(filename, HistoryRepoJSONPrefix) && strings.HasSuffix(filename, HistoryRepoJSONSuffix)) { + files = append(files, path.Join(h.varDir, filename)) + } + } + } + sort.Sort(sort.Reverse(sort.StringSlice(files))) + return +} + +func (h *History) cleanup() error { + files, err := h.getFilesForCleanup(h.max) + if err != nil { + return err + } + + for _, f := range files { + h.l.Debug("removing outdated backup", zap.String("file", f)) + err := os.Remove(f) + if err != nil { + return fmt.Errorf("could not remove file %s : %s", f, err.Error()) + } + } + + return nil +} + +func (h *History) getFilesForCleanup(historyVersions int) (files []string, err error) { + contentFiles, err := h.getHistory() + if err != nil { + return nil, errors.New("could not generate file cleanup list: " + err.Error()) + } + + // fmt.Println("contentFiles:") + // for _, f := range contentFiles { + // fmt.Println(f) + // } + + // -1 to remove the current backup file from the number of items + // so that only files with a timestamp are compared + if len(contentFiles)-1 > historyVersions { + for i := historyVersions + 1; i < len(contentFiles); i++ { + // ignore current repository file to fall back on + if contentFiles[i] == h.GetCurrentFilename() { + continue + } + files = append(files, contentFiles[i]) + } + } + return files, nil +} diff --git a/pkg/repo/history_test.go b/pkg/repo/history_test.go new file mode 100644 index 0000000..007bc14 --- /dev/null +++ b/pkg/repo/history_test.go @@ -0,0 +1,75 @@ +package repo + +import ( + "bytes" + "fmt" + "os" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap/zaptest" +) + +func TestHistoryCurrent(t *testing.T) { + var ( + h = testHistory(t) + test = []byte("test") + b bytes.Buffer + ) + err := h.Add(test) + require.NoError(t, err) + err = h.GetCurrent(&b) + require.NoError(t, err) + if !bytes.Equal(b.Bytes(), test) { + t.Fatalf("expected %q, got %q", string(test), b.String()) + } +} + +func TestHistoryCleanup(t *testing.T) { + h := testHistory(t) + for i := 0; i < 50; i++ { + err := h.Add([]byte(fmt.Sprint(i))) + require.NoError(t, err) + time.Sleep(time.Millisecond * 5) + } + err := h.cleanup() + require.NoError(t, err) + files, err := h.getHistory() + require.NoError(t, err) + + // -1 for ignoring the current content backup file + if len(files)-1 != 2 { + t.Fatal("history too long", len(files), "instead of", 2) + } +} + +func TestHistoryOrder(t *testing.T) { + h := testHistory(t) + h.varDir = "testdata/order" + + files, err := h.getHistory() + require.NoError(t, err) + assert.Equal(t, "testdata/order/contentserver-repo-current.json", files[0]) + assert.Equal(t, "testdata/order/contentserver-repo-2017-10-23.json", files[1]) + assert.Equal(t, "testdata/order/contentserver-repo-2017-10-22.json", files[2]) + assert.Equal(t, "testdata/order/contentserver-repo-2017-10-21.json", files[3]) +} + +func TestGetFilesForCleanup(t *testing.T) { + h := testHistory(t) + h.varDir = "testdata/order" + + files, err := h.getFilesForCleanup(2) + require.NoError(t, err) + assert.Equal(t, "testdata/order/contentserver-repo-2017-10-21.json", files[0]) +} + +func testHistory(t *testing.T) *History { + t.Helper() + l := zaptest.NewLogger(t) + tempDir, err := os.MkdirTemp(os.TempDir(), "contentserver-history-test") + require.NoError(t, err) + return NewHistory(l, HistoryWithMax(2), HistoryWithVarDir(tempDir)) +} diff --git a/repo/loader.go b/pkg/repo/loader.go similarity index 51% rename from repo/loader.go rename to pkg/repo/loader.go index 10a8065..4999157 100644 --- a/repo/loader.go +++ b/pkg/repo/loader.go @@ -3,13 +3,11 @@ package repo import ( "context" "io" - "io/ioutil" "net/http" "time" "github.com/foomo/contentserver/content" - "github.com/foomo/contentserver/logger" - "github.com/foomo/contentserver/status" + "github.com/foomo/contentserver/pkg/metrics" "github.com/google/uuid" jsoniter "github.com/json-iterator/go" "github.com/pkg/errors" @@ -19,7 +17,7 @@ import ( var ( json = jsoniter.ConfigCompatibleWithStandardLibrary - errUpdateRejected = errors.New("update rejected: queue full") + ErrUpdateRejected = errors.New("update rejected: queue full") ) type updateResponse struct { @@ -27,56 +25,91 @@ type updateResponse struct { err error } -func (repo *Repo) updateRoutine() { - for resChan := range repo.updateInProgressChannel { - log := logger.Log.With(zap.String("updateRunID", uuid.New().String())) - - log.Info("Content update started") - start := time.Now() - - repoRuntime, err := repo.update(context.Background()) - if err != nil { - log.Error("Content update failed", zap.Error(err)) - status.M.UpdatesFailedCounter.WithLabelValues().Inc() - } else { - log.Info("Content update success") - status.M.UpdatesCompletedCounter.WithLabelValues().Inc() +func (r *Repo) PollRoutine(ctx context.Context) error { + l := r.l.Named("routine.poll") + ticker := time.NewTicker(10 * time.Second) + for { + select { + case <-ctx.Done(): + l.Debug("routine canceled", zap.Error(ctx.Err())) + return nil + case <-ticker.C: + chanReponse := make(chan updateResponse) + r.updateInProgressChannel <- chanReponse + response := <-chanReponse + if response.err == nil { + l.Info("update success", zap.String("revision", r.pollVersion)) + } else { + l.Error("update failed", zap.Error(response.err)) + } } - - resChan <- updateResponse{ - repoRuntime: repoRuntime, - err: err, - } - - status.M.UpdateDuration.WithLabelValues().Observe(time.Since(start).Seconds()) } } -func (repo *Repo) dimensionUpdateRoutine() { - for newDimension := range repo.dimensionUpdateChannel { - logger.Log.Info("dimensionUpdateRoutine received a new dimension", zap.String("dimension", newDimension.Dimension)) +func (r *Repo) UpdateRoutine(ctx context.Context) error { + l := r.l.Named("routine.update") + for { + select { + case <-ctx.Done(): + l.Debug("routine canceled", zap.Error(ctx.Err())) + return nil + case resChan := <-r.updateInProgressChannel: + start := time.Now() + l := l.With(zap.String("run_id", uuid.New().String())) - err := repo._updateDimension(newDimension.Dimension, newDimension.Node) - logger.Log.Info("dimensionUpdateRoutine received result") - if err != nil { - logger.Log.Debug("update dimension failed", zap.Error(err)) + l.Info("update started") + + repoRuntime, err := r.update(context.Background()) + if err != nil { + l.Error("update failed", zap.Error(err)) + metrics.UpdatesFailedCounter.WithLabelValues().Inc() + } else { + l.Info("update success") + metrics.UpdatesCompletedCounter.WithLabelValues().Inc() + } + + resChan <- updateResponse{ + repoRuntime: repoRuntime, + err: err, + } + + metrics.UpdateDuration.WithLabelValues().Observe(time.Since(start).Seconds()) } - repo.dimensionUpdateDoneChannel <- err } } -func (repo *Repo) updateDimension(dimension string, node *content.RepoNode) error { - logger.Log.Debug("trying to push dimension into update channel", zap.String("dimension", dimension), zap.String("nodeName", node.Name)) - repo.dimensionUpdateChannel <- &repoDimension{ +func (r *Repo) DimensionUpdateRoutine(ctx context.Context) error { + l := r.l.Named("routine.dimensionUpdate") + for { + select { + case <-ctx.Done(): + l.Debug("routine canceled", zap.Error(ctx.Err())) + return nil + case newDimension := <-r.dimensionUpdateChannel: + l.Debug("received a new dimension", zap.String("dimension", newDimension.Dimension)) + + err := r._updateDimension(newDimension.Dimension, newDimension.Node) + l.Info("received result") + if err != nil { + l.Debug("update failed", zap.Error(err)) + } + r.dimensionUpdateDoneChannel <- err + } + } +} + +func (r *Repo) updateDimension(dimension string, node *content.RepoNode) error { + r.l.Debug("trying to push dimension into update channel", zap.String("dimension", dimension), zap.String("nodeName", node.Name)) + r.dimensionUpdateChannel <- &RepoDimension{ Dimension: dimension, Node: node, } - logger.Log.Debug("waiting for done signal") - return <-repo.dimensionUpdateDoneChannel + r.l.Debug("waiting for done signal") + return <-r.dimensionUpdateDoneChannel } // do not call directly, but only through channel -func (repo *Repo) _updateDimension(dimension string, newNode *content.RepoNode) error { +func (r *Repo) _updateDimension(dimension string, newNode *content.RepoNode) error { newNode.WireParents() var ( @@ -97,7 +130,7 @@ func (repo *Repo) _updateDimension(dimension string, newNode *content.RepoNode) // copy old datastructure to prevent concurrent map access // collect other dimension in the Directory newRepoDirectory := map[string]*Dimension{} - for d, D := range repo.Directory { + for d, D := range r.Directory { if d != dimension { newRepoDirectory[d] = D } @@ -109,7 +142,7 @@ func (repo *Repo) _updateDimension(dimension string, newNode *content.RepoNode) Directory: newDirectory, URIDirectory: newURIDirectory, } - repo.Directory = newRepoDirectory + r.Directory = newRepoDirectory // --------------------------------------------- @@ -126,15 +159,12 @@ func (repo *Repo) _updateDimension(dimension string, newNode *content.RepoNode) } func buildDirectory(dirNode *content.RepoNode, directory map[string]*content.RepoNode, uRIDirectory map[string]*content.RepoNode) error { - - // Log.Debug("buildDirectory", zap.String("ID", dirNode.ID)) - existingNode, ok := directory[dirNode.ID] if ok { return errors.New("duplicate node with id:" + existingNode.ID) } directory[dirNode.ID] = dirNode - //todo handle duplicate uris + // todo handle duplicate uris if _, thereIsAnExistingURINode := uRIDirectory[dirNode.URI]; thereIsAnExistingURINode { return errors.New("duplicate uri: " + dirNode.URI + " (bad node id: " + dirNode.ID + ")") } @@ -161,26 +191,30 @@ func wireAliases(directory map[string]*content.RepoNode) error { return nil } -func (repo *Repo) loadNodesFromJSON() (nodes map[string]*content.RepoNode, err error) { +func (r *Repo) loadNodesFromJSON() (nodes map[string]*content.RepoNode, err error) { nodes = make(map[string]*content.RepoNode) - err = json.Unmarshal(repo.jsonBuf.Bytes(), &nodes) + err = json.Unmarshal(r.jsonBuf.Bytes(), &nodes) if err != nil { - logger.Log.Error("Failed to deserialize nodes", zap.Error(err)) + r.l.Error("Failed to deserialize nodes", zap.Error(err)) return nil, errors.New("failed to deserialize nodes") } return nodes, nil } -func (repo *Repo) tryToRestoreCurrent() (err error) { - err = repo.history.getCurrent(&repo.jsonBuf) +func (r *Repo) tryToRestoreCurrent() error { + err := r.history.GetCurrent(&r.jsonBuf) if err != nil { return err } - return repo.loadJSONBytes() + return r.loadJSONBytes() } -func (repo *Repo) get(URL string) error { - response, err := repo.httpClient.Get(URL) +func (r *Repo) get(ctx context.Context, url string) error { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) + if err != nil { + return errors.Wrap(err, "failed to create get repo request") + } + response, err := r.httpClient.Do(req) if err != nil { return errors.Wrap(err, "failed to get repo") } @@ -191,10 +225,10 @@ func (repo *Repo) get(URL string) error { } // Log.Info(ansi.Red + "RESETTING BUFFER" + ansi.Reset) - repo.jsonBuf.Reset() + r.jsonBuf.Reset() // Log.Info(ansi.Green + "LOADING DATA INTO BUFFER" + ansi.Reset) - _, err = io.Copy(&repo.jsonBuf, response.Body) + _, err = io.Copy(&r.jsonBuf, response.Body) if err != nil { return errors.Wrap(err, "failed to copy IO stream") } @@ -202,12 +236,16 @@ func (repo *Repo) get(URL string) error { return nil } -func (repo *Repo) update(ctx context.Context) (repoRuntime int64, err error) { +func (r *Repo) update(ctx context.Context) (repoRuntime int64, err error) { startTimeRepo := time.Now().UnixNano() - repoURL := repo.server - if repo.pollForUpdates { - resp, err := repo.httpClient.Get(repo.server) + repoURL := r.url + if r.poll { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, r.url, nil) + if err != nil { + return repoRuntime, err + } + resp, err := r.httpClient.Do(req) if err != nil { return repoRuntime, err } @@ -215,71 +253,71 @@ func (repo *Repo) update(ctx context.Context) (repoRuntime int64, err error) { if resp.StatusCode != http.StatusOK { return repoRuntime, errors.New("could not poll latest repo download url - non 200 response") } - responseBytes, err := ioutil.ReadAll(resp.Body) + responseBytes, err := io.ReadAll(resp.Body) if err != nil { return repoRuntime, errors.New("could not poll latest repo download url, could not read body") } repoURL = string(responseBytes) - if repoURL == repo.pollVersion { - logger.Log.Info( + if repoURL == r.pollVersion { + r.l.Info( "repo is up to date", - zap.String("pollVersion", repo.pollVersion), + zap.String("pollVersion", r.pollVersion), ) // already up to date return repoRuntime, nil } else { - logger.Log.Info( + r.l.Info( "new repo poll version", - zap.String("pollVersion", repo.pollVersion), + zap.String("pollVersion", r.pollVersion), ) } } - err = repo.get(repoURL) + err = r.get(ctx, repoURL) repoRuntime = time.Now().UnixNano() - startTimeRepo if err != nil { // we have no json to load - the repo server did not reply - logger.Log.Debug("Failed to load json", zap.Error(err)) + r.l.Debug("failed to load json", zap.Error(err)) return repoRuntime, err } - logger.Log.Debug("loading json", zap.String("server", repoURL), zap.Int("length", len(repo.jsonBuf.Bytes()))) - nodes, err := repo.loadNodesFromJSON() + r.l.Debug("loading json", zap.String("server", repoURL), zap.Int("length", len(r.jsonBuf.Bytes()))) + nodes, err := r.loadNodesFromJSON() if err != nil { // could not load nodes from json return repoRuntime, err } - err = repo.loadNodes(nodes) + err = r.loadNodes(nodes) if err != nil { // repo failed to load nodes return repoRuntime, err } - if repo.pollForUpdates { - repo.pollVersion = repoURL + if r.poll { + r.pollVersion = repoURL } return repoRuntime, nil } // limit ressources and allow only one update request at once -func (repo *Repo) tryUpdate() (repoRuntime int64, err error) { +func (r *Repo) tryUpdate() (repoRuntime int64, err error) { c := make(chan updateResponse) select { - case repo.updateInProgressChannel <- c: - logger.Log.Info("update request added to queue") + case r.updateInProgressChannel <- c: + r.l.Debug("update request added to queue") ur := <-c return ur.repoRuntime, ur.err default: - logger.Log.Info("update request accepted, will be processed after the previous update") - return 0, errUpdateRejected + r.l.Info("update request accepted, will be processed after the previous update") + return 0, ErrUpdateRejected } } -func (repo *Repo) loadJSONBytes() error { - nodes, err := repo.loadNodesFromJSON() +func (r *Repo) loadJSONBytes() error { + nodes, err := r.loadNodesFromJSON() if err != nil { - data := repo.jsonBuf.Bytes() + data := r.jsonBuf.Bytes() if len(data) > 10 { - logger.Log.Debug("could not parse json", + r.l.Debug("could not parse json", zap.String("jsonStart", string(data[:10])), zap.String("jsonStart", string(data[len(data)-10:])), ) @@ -287,26 +325,26 @@ func (repo *Repo) loadJSONBytes() error { return err } - err = repo.loadNodes(nodes) + err = r.loadNodes(nodes) if err == nil { - errHistory := repo.history.add(repo.jsonBuf.Bytes()) + errHistory := r.history.Add(r.jsonBuf.Bytes()) if errHistory != nil { - logger.Log.Error("Could not add valid JSON to history", zap.Error(errHistory)) - status.M.HistoryPersistFailedCounter.WithLabelValues().Inc() + r.l.Error("Could not add valid JSON to history", zap.Error(errHistory)) + metrics.HistoryPersistFailedCounter.WithLabelValues().Inc() } else { - logger.Log.Info("Added valid JSON to history") + r.l.Info("added valid JSON to history") } } return err } -func (repo *Repo) loadNodes(newNodes map[string]*content.RepoNode) error { - var newDimensions []string +func (r *Repo) loadNodes(newNodes map[string]*content.RepoNode) error { var err error + newDimensions := make([]string, 0, len(newNodes)) for dimension, newNode := range newNodes { newDimensions = append(newDimensions, dimension) - logger.Log.Debug("Loading nodes for dimension", zap.String("dimension", dimension)) - errLoad := repo.updateDimension(dimension, newNode) + r.l.Debug("loading nodes for dimension", zap.String("dimension", dimension)) + errLoad := r.updateDimension(dimension, newNode) if errLoad != nil { err = multierr.Append(err, errLoad) } @@ -323,10 +361,10 @@ func (repo *Repo) loadNodes(newNodes map[string]*content.RepoNode) error { return false } // we need to throw away orphaned dimensions - for dimension := range repo.Directory { + for dimension := range r.Directory { if !dimensionIsValid(dimension) { - logger.Log.Info("Removing orphaned dimension", zap.String("dimension", dimension)) - delete(repo.Directory, dimension) + r.l.Info("removing orphaned dimension", zap.String("dimension", dimension)) + delete(r.Directory, dimension) } } return nil diff --git a/repo/mock/mock.go b/pkg/repo/mock/mock.go similarity index 81% rename from repo/mock/mock.go rename to pkg/repo/mock/mock.go index 2b1ae02..42edd97 100644 --- a/repo/mock/mock.go +++ b/pkg/repo/mock/mock.go @@ -1,32 +1,33 @@ package mock import ( - "io/ioutil" "net/http" "net/http/httptest" + "os" "path" "runtime" "testing" "time" "github.com/foomo/contentserver/requests" + "github.com/stretchr/testify/require" ) // GetMockData mock data to run a repo -func GetMockData(t testing.TB) (server *httptest.Server, varDir string) { - +func GetMockData(tb testing.TB) (*httptest.Server, string) { + tb.Helper() _, filename, _, _ := runtime.Caller(0) mockDir := path.Dir(filename) - server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { time.Sleep(time.Millisecond * 50) mockFilename := path.Join(mockDir, req.URL.Path[1:]) http.ServeFile(w, req, mockFilename) })) - varDir, err := ioutil.TempDir("", "content-server-test") - if err != nil { - panic(err) - } + + varDir, err := os.MkdirTemp("", "content-server-test") + require.NoError(tb, err) + return server, varDir } @@ -37,7 +38,7 @@ func MakeNodesRequest() *requests.Nodes { Dimensions: []string{"dimension_foo"}, }, Nodes: map[string]*requests.Node{ - "test": &requests.Node{ + "test": { ID: "id-root", Dimension: "dimension_foo", MimeTypes: []string{}, @@ -66,7 +67,7 @@ func MakeValidContentRequest() *requests.Content { Groups: []string{}, }, Nodes: map[string]*requests.Node{ - "id-root": &requests.Node{ + "id-root": { ID: "id-root", Dimension: dimensions[0], MimeTypes: []string{"application/x-node"}, @@ -75,5 +76,4 @@ func MakeValidContentRequest() *requests.Content { }, }, } - } diff --git a/repo/mock/repo-broken-json.json b/pkg/repo/mock/repo-broken-json.json similarity index 100% rename from repo/mock/repo-broken-json.json rename to pkg/repo/mock/repo-broken-json.json diff --git a/repo/mock/repo-duplicate-uris.json b/pkg/repo/mock/repo-duplicate-uris.json similarity index 100% rename from repo/mock/repo-duplicate-uris.json rename to pkg/repo/mock/repo-duplicate-uris.json diff --git a/repo/mock/repo-link-broken.json b/pkg/repo/mock/repo-link-broken.json similarity index 100% rename from repo/mock/repo-link-broken.json rename to pkg/repo/mock/repo-link-broken.json diff --git a/repo/mock/repo-link-ok.json b/pkg/repo/mock/repo-link-ok.json similarity index 100% rename from repo/mock/repo-link-ok.json rename to pkg/repo/mock/repo-link-ok.json diff --git a/repo/mock/repo-ok-exposehidden.json b/pkg/repo/mock/repo-ok-exposehidden.json similarity index 100% rename from repo/mock/repo-ok-exposehidden.json rename to pkg/repo/mock/repo-ok-exposehidden.json diff --git a/repo/mock/repo-ok.json b/pkg/repo/mock/repo-ok.json similarity index 100% rename from repo/mock/repo-ok.json rename to pkg/repo/mock/repo-ok.json diff --git a/repo/mock/repo-two-dimensions.json b/pkg/repo/mock/repo-two-dimensions.json similarity index 100% rename from repo/mock/repo-two-dimensions.json rename to pkg/repo/mock/repo-two-dimensions.json diff --git a/pkg/repo/repo.go b/pkg/repo/repo.go new file mode 100644 index 0000000..b0b6e64 --- /dev/null +++ b/pkg/repo/repo.go @@ -0,0 +1,483 @@ +package repo + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http" + "os" + "strings" + "sync/atomic" + "time" + + "github.com/foomo/contentserver/content" + "github.com/foomo/contentserver/pkg/metrics" + "github.com/foomo/contentserver/requests" + "github.com/foomo/contentserver/responses" + "github.com/pkg/errors" + "go.uber.org/zap" + "golang.org/x/sync/errgroup" +) + +const maxGetURIForNodeRecursionLevel = 1000 + +// Repo content repository +type ( + Repo struct { + l *zap.Logger + url string + poll bool + pollVersion string + onStart func() + loaded *atomic.Bool + history *History + httpClient *http.Client + Directory map[string]*Dimension + // updateLock sync.Mutex + dimensionUpdateChannel chan *RepoDimension + dimensionUpdateDoneChannel chan error + updateInProgressChannel chan chan updateResponse + // jsonBytes []byte + jsonBuf bytes.Buffer + } + Option func(*Repo) +) + +// ------------------------------------------------------------------------------------------------ +// ~ Constructor +// ------------------------------------------------------------------------------------------------ + +func New(l *zap.Logger, url string, history *History, opts ...Option) *Repo { + inst := &Repo{ + l: l.Named("repo"), + url: url, + poll: false, + loaded: &atomic.Bool{}, + history: history, + httpClient: http.DefaultClient, + Directory: map[string]*Dimension{}, + dimensionUpdateChannel: make(chan *RepoDimension), + dimensionUpdateDoneChannel: make(chan error), + updateInProgressChannel: make(chan chan updateResponse), + } + + for _, opt := range opts { + opt(inst) + } + + return inst +} + +// ------------------------------------------------------------------------------------------------ +// ~ Options +// ------------------------------------------------------------------------------------------------ + +func WithHTTPClient(v *http.Client) Option { + return func(o *Repo) { + o.httpClient = v + } +} + +func WithPollForUpdates(v bool) Option { + return func(o *Repo) { + o.poll = v + } +} + +// ------------------------------------------------------------------------------------------------ +// ~ Getter +// ------------------------------------------------------------------------------------------------ + +func (r *Repo) Loaded() bool { + return r.loaded.Load() +} + +func (r *Repo) OnStart(fn func()) { + r.onStart = fn +} + +// ------------------------------------------------------------------------------------------------ +// ~ Public methods +// ------------------------------------------------------------------------------------------------ + +// GetURIs get many uris at once +func (r *Repo) GetURIs(dimension string, ids []string) map[string]string { + uris := map[string]string{} + for _, id := range ids { + uris[id] = r.getURI(dimension, id) + } + return uris +} + +// GetNodes get nodes +func (r *Repo) GetNodes(nodes *requests.Nodes) map[string]*content.Node { + return r.getNodes(nodes.Nodes, nodes.Env) +} + +// GetContent resolves content and fetches nodes in one call. It combines those +// two tasks for performance reasons. +// +// In the first step it uses r.URI to look up content in all given +// r.Env.Dimensions of repo.Directory. +// +// In the second step it collects the requested nodes. +// +// those two steps are independent. +func (r *Repo) GetContent(req *requests.Content) (*content.SiteContent, error) { + // add more input validation + err := r.validateContentRequest(req) + if err != nil { + return nil, errors.Wrap(err, "repo.GetContent invalid request") + } + r.l.Debug("repo.GetContent", zap.String("URI", req.URI)) + c := content.NewSiteContent() + resolved, resolvedURI, resolvedDimension, node := r.resolveContent(req.Env.Dimensions, req.URI) + if resolved { + if !node.CanBeAccessedByGroups(req.Env.Groups) { + r.l.Warn("Resolved content cannot be accessed by specified group", zap.String("uri", req.URI)) + c.Status = content.StatusForbidden + } else { + r.l.Info("Content resolved", zap.String("uri", req.URI)) + c.Status = content.StatusOk + c.Data = node.Data + } + c.MimeType = node.MimeType + c.Dimension = resolvedDimension + c.URI = resolvedURI + c.Item = node.ToItem(req.DataFields) + c.Path = node.GetPath(req.PathDataFields) + // fetch URIs for all dimensions + uris := make(map[string]string) + for dimensionName := range r.Directory { + uris[dimensionName] = r.getURI(dimensionName, node.ID) + } + c.URIs = uris + } else { + r.l.Info("Content not found", zap.String("URI", req.URI)) + c.Status = content.StatusNotFound + c.Dimension = req.Env.Dimensions[0] + + r.l.Debug("Failed to resolve, falling back to default dimension", + zap.String("uri", req.URI), + zap.String("default_dimension", req.Env.Dimensions[0]), + ) + // r.Env.Dimensions is validated => we can access it + resolvedDimension = req.Env.Dimensions[0] + } + + // add navigation trees + for _, node := range req.Nodes { + if node.Dimension == "" { + node.Dimension = resolvedDimension + } + } + c.Nodes = r.getNodes(req.Nodes, req.Env) + return c, nil +} + +// GetRepo get the whole repo in all dimensions +func (r *Repo) GetRepo() map[string]*content.RepoNode { + response := make(map[string]*content.RepoNode) + for dimensionName, dimension := range r.Directory { + response[dimensionName] = dimension.Node + } + return response +} + +// WriteRepoBytes get the whole repo in all dimensions +// reads the JSON history file from the Filesystem and copies it directly in to the supplied buffer +// the result is wrapped as service response, e.g: {"reply": } +func (r *Repo) WriteRepoBytes(w io.Writer) { + f, err := os.Open(r.history.GetCurrentFilename()) + if err != nil { + r.l.Error("Failed to open Repo JSON", zap.Error(err)) + } + + _, _ = w.Write([]byte("{\"reply\":")) + _, err = io.Copy(w, f) + if err != nil { + r.l.Error("Failed to serve Repo JSON", zap.Error(err)) + } + _, _ = w.Write([]byte("}")) +} + +func (r *Repo) Update() (updateResponse *responses.Update) { + floatSeconds := func(nanoSeconds int64) float64 { + return float64(nanoSeconds) / float64(1000000000) + } + + r.l.Info("Update triggered") + // Log.Info(ansi.Yellow + "BUFFER LENGTH BEFORE tryUpdate(): " + strconv.Itoa(len(repo.jsonBuf.Bytes())) + ansi.Reset) + + start := time.Now() + updateRepotime, err := r.tryUpdate() + updateResponse = &responses.Update{} + updateResponse.Stats.RepoRuntime = floatSeconds(updateRepotime) + + if err != nil { + updateResponse.Success = false + updateResponse.Stats.NumberOfNodes = -1 + updateResponse.Stats.NumberOfURIs = -1 + + // let us try to restore the world from a file + // Log.Info(ansi.Yellow + "BUFFER LENGTH AFTER ERROR: " + strconv.Itoa(len(r.jsonBuf.Bytes())) + ansi.Reset) + // only try to restore if the update failed during processing + + if !errors.Is(err, ErrUpdateRejected) { + updateResponse.ErrorMessage = err.Error() + r.l.Error("Failed to update repository", zap.Error(err)) + + restoreErr := r.tryToRestoreCurrent() + if restoreErr != nil { + r.l.Error("Failed to restore preceding repository version", zap.Error(restoreErr)) + } else { + r.l.Info("Successfully restored current repository from local history") + } + } + } else { + updateResponse.Success = true + // persist the currently loaded one + historyErr := r.history.Add(r.jsonBuf.Bytes()) + if historyErr != nil { + r.l.Error("Could not persist current repo in history", zap.Error(historyErr)) + metrics.HistoryPersistFailedCounter.WithLabelValues().Inc() + } + // add some stats + for dimension := range r.Directory { + updateResponse.Stats.NumberOfNodes += len(r.Directory[dimension].Directory) + updateResponse.Stats.NumberOfURIs += len(r.Directory[dimension].URIDirectory) + } + r.loaded.Store(true) + } + updateResponse.Stats.OwnRuntime = floatSeconds(time.Since(start).Nanoseconds()) - updateResponse.Stats.RepoRuntime + return updateResponse +} + +func (r *Repo) Start(ctx context.Context) error { + g, gCtx := errgroup.WithContext(ctx) + + l := r.l.Named("start") + + up := make(chan bool, 1) + g.Go(func() error { + l.Debug("starting update routine") + up <- true + return r.UpdateRoutine(gCtx) + }) + l.Debug("waiting for UpdateRoutine") + <-up + + g.Go(func() error { + l.Debug("starting dimension update routine") + up <- true + return r.DimensionUpdateRoutine(gCtx) + }) + l.Debug("waiting for DimensionUpdateRoutine") + <-up + + l.Debug("trying to restore previous repo") + if err := r.tryToRestoreCurrent(); errors.Is(err, os.ErrNotExist) { + l.Info("previous repo content file does not exist") + } else if err != nil { + l.Warn("could not restore previous repo content", zap.Error(err)) + } else { + l.Info("restored previous repo") + r.loaded.Store(true) + } + + if r.poll { + g.Go(func() error { + l.Debug("starting poll routine") + return r.PollRoutine(gCtx) + }) + } else if !r.Loaded() { + l.Debug("trying to update initial state") + if resp := r.Update(); !resp.Success { + l.Fatal("failed to update", + zap.String("error", resp.ErrorMessage), + zap.Int("num_modes", resp.Stats.NumberOfNodes), + zap.Int("num_uris", resp.Stats.NumberOfURIs), + zap.Float64("own_runtime", resp.Stats.OwnRuntime), + zap.Float64("repo_runtime", resp.Stats.RepoRuntime), + ) + } + } + + if r.onStart != nil { + r.onStart() + } + + return g.Wait() +} + +// ------------------------------------------------------------------------------------------------ +// ~ Private methods +// ------------------------------------------------------------------------------------------------ + +func (r *Repo) getNodes(nodeRequests map[string]*requests.Node, env *requests.Env) map[string]*content.Node { + var ( + path []*content.Item + nodes = map[string]*content.Node{} + ) + for nodeName, nodeRequest := range nodeRequests { + if nodeName == "" || nodeRequest.ID == "" { + r.l.Warn("invalid node request", zap.Error(errors.New("nodeName or nodeRequest.ID empty"))) + continue + } + r.l.Debug("adding node", zap.String("name", nodeName), zap.String("requestID", nodeRequest.ID)) + + groups := env.Groups + if len(nodeRequest.Groups) > 0 { + groups = nodeRequest.Groups + } + + dimensionNode, ok := r.Directory[nodeRequest.Dimension] + nodes[nodeName] = nil + + if !ok && nodeRequest.Dimension == "" { + r.l.Debug("Could not get dimension root node", zap.String("dimension", nodeRequest.Dimension)) + for _, dimension := range env.Dimensions { + dimensionNode, ok = r.Directory[dimension] + if ok { + r.l.Debug("Found root node in env.Dimensions", zap.String("dimension", dimension)) + break + } + r.l.Debug("Could NOT find root node in env.Dimensions", zap.String("dimension", dimension)) + } + } + + if !ok { + r.l.Error("could not get dimension root node", zap.String("nodeRequest.Dimension", nodeRequest.Dimension)) + continue + } + + treeNode, ok := dimensionNode.Directory[nodeRequest.ID] + if !ok { + r.l.Error("Invalid tree node requested", + zap.String("nodeName", nodeName), + zap.String("nodeID", nodeRequest.ID), + ) + metrics.InvalidNodeTreeRequests.WithLabelValues().Inc() + continue + } + nodes[nodeName] = r.getNode(treeNode, nodeRequest.Expand, nodeRequest.MimeTypes, path, 0, groups, nodeRequest.DataFields, nodeRequest.ExposeHiddenNodes) + } + return nodes +} + +// resolveContent find content in a repository +func (r *Repo) resolveContent(dimensions []string, uri string) (resolved bool, resolvedURI string, resolvedDimension string, repoNode *content.RepoNode) { + parts := strings.Split(uri, content.PathSeparator) + r.l.Debug("repo.ResolveContent", zap.String("URI", uri)) + for i := len(parts); i > 0; i-- { + testURI := strings.Join(parts[0:i], content.PathSeparator) + if testURI == "" { + testURI = content.PathSeparator + } + for _, dimension := range dimensions { + if d, ok := r.Directory[dimension]; ok { + r.l.Debug("Checking node", + zap.String("dimension", dimension), + zap.String("URI", testURI), + ) + if repoNode, ok := d.URIDirectory[testURI]; ok { + resolved = true + r.l.Debug("Node found", zap.String("URI", testURI), zap.String("destination", repoNode.DestinationID)) + if len(repoNode.DestinationID) > 0 { + if destionationNode, destinationNodeOk := d.Directory[repoNode.DestinationID]; destinationNodeOk { + repoNode = destionationNode + } + } + return resolved, testURI, dimension, repoNode + } + } + } + } + return +} + +func (r *Repo) getURIForNode(dimension string, repoNode *content.RepoNode, recursionLevel int64) (uri string) { + if len(repoNode.LinkID) == 0 { + uri = repoNode.URI + return + } + linkedNode, ok := r.Directory[dimension].Directory[repoNode.LinkID] + if ok { + if recursionLevel > maxGetURIForNodeRecursionLevel { + r.l.Error("maxGetURIForNodeRecursionLevel reached", zap.String("repoNode.ID", repoNode.ID), zap.String("linkID", repoNode.LinkID), zap.String("dimension", dimension)) + return "" + } + return r.getURIForNode(dimension, linkedNode, recursionLevel+1) + } + return +} + +func (r *Repo) getURI(dimension string, id string) string { + directory, ok := r.Directory[dimension] + if !ok { + return "" + } + repoNode, ok := directory.Directory[id] + if !ok { + return "" + } + return r.getURIForNode(dimension, repoNode, 0) +} + +func (r *Repo) getNode( + repoNode *content.RepoNode, + expanded bool, + mimeTypes []string, + path []*content.Item, + level int, + groups []string, + dataFields []string, + exposeHiddenNodes bool, +) *content.Node { + node := content.NewNode() + node.Item = repoNode.ToItem(dataFields) + r.l.Debug("getNode", zap.String("ID", repoNode.ID)) + for _, childID := range repoNode.Index { + childNode := repoNode.Nodes[childID] + if (level == 0 || expanded || !expanded && childNode.InPath(path)) && (!childNode.Hidden || exposeHiddenNodes) && childNode.CanBeAccessedByGroups(groups) && childNode.IsOneOfTheseMimeTypes(mimeTypes) { + node.Nodes[childID] = r.getNode(childNode, expanded, mimeTypes, path, level+1, groups, dataFields, exposeHiddenNodes) + node.Index = append(node.Index, childID) + } + } + return node +} + +func (r *Repo) validateContentRequest(req *requests.Content) (err error) { + if req == nil { + return errors.New("request must not be nil") + } + if len(req.URI) == 0 { + return errors.New("request URI must not be empty") + } + if req.Env == nil { + return errors.New("request.Env must not be nil") + } + if len(req.Env.Dimensions) == 0 { + return errors.New("request.Env.Dimensions must not be empty") + } + for _, envDimension := range req.Env.Dimensions { + if !r.hasDimension(envDimension) { + availableDimensions := make([]string, 0, len(r.Directory)) + for availableDimension := range r.Directory { + availableDimensions = append(availableDimensions, availableDimension) + } + return errors.New(fmt.Sprint( + "unknown dimension ", envDimension, + " in r.Env must be one of ", availableDimensions, + " repo has ", len(r.Directory), " dimensions", + )) + } + } + return nil +} + +func (r *Repo) hasDimension(d string) bool { + _, hasDimension := r.Directory[d] + return hasDimension +} diff --git a/repo/repo_test.go b/pkg/repo/repo_test.go similarity index 54% rename from repo/repo_test.go rename to pkg/repo/repo_test.go index 4f450d2..0af28fd 100644 --- a/repo/repo_test.go +++ b/pkg/repo/repo_test.go @@ -1,33 +1,29 @@ package repo import ( + "context" "strings" "testing" "time" - . "github.com/foomo/contentserver/logger" - _ "github.com/foomo/contentserver/logger" - "github.com/foomo/contentserver/repo/mock" + "github.com/foomo/contentserver/pkg/repo/mock" "github.com/foomo/contentserver/requests" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "go.uber.org/zap" + "go.uber.org/zap/zaptest" ) -func init() { - SetupLogging(true, "contentserver_repo_test.log") -} - -func NewTestRepo(server, varDir string) *Repo { - - r := NewRepo(server, varDir, 2*time.Minute, false) - - // because the travis CI VMs are very slow, - // we need to add some delay to allow the server to startup - time.Sleep(1 * time.Second) - +func NewTestRepo(l *zap.Logger, server, varDir string) *Repo { + h := NewHistory(l, HistoryWithMax(2), HistoryWithVarDir(varDir)) + r := New(l, server, h) + go r.Start(context.Background()) //nolint:errcheck + time.Sleep(100 * time.Millisecond) return r } func assertRepoIsEmpty(t *testing.T, r *Repo, empty bool) { + t.Helper() if empty { if len(r.Directory) > 0 { t.Fatal("directory should have been empty, but is not") @@ -40,10 +36,12 @@ func assertRepoIsEmpty(t *testing.T, r *Repo, empty bool) { } func TestLoad404(t *testing.T) { + l := zaptest.NewLogger(t) + var ( mockServer, varDir = mock.GetMockData(t) server = mockServer.URL + "/repo-no-have" - r = NewTestRepo(server, varDir) + r = NewTestRepo(l, server, varDir) ) response := r.Update() @@ -53,10 +51,12 @@ func TestLoad404(t *testing.T) { } func TestLoadBrokenRepo(t *testing.T) { + l := zaptest.NewLogger(t) + var ( mockServer, varDir = mock.GetMockData(t) server = mockServer.URL + "/repo-broken-json.json" - r = NewTestRepo(server, varDir) + r = NewTestRepo(l, server, varDir) ) response := r.Update() @@ -66,11 +66,12 @@ func TestLoadBrokenRepo(t *testing.T) { } func TestLoadRepo(t *testing.T) { + l := zaptest.NewLogger(t) var ( mockServer, varDir = mock.GetMockData(t) server = mockServer.URL + "/repo-ok.json" - r = NewTestRepo(server, varDir) + r = NewTestRepo(l, server, varDir) ) assertRepoIsEmpty(t, r, true) @@ -83,22 +84,23 @@ func TestLoadRepo(t *testing.T) { if response.Stats.OwnRuntime > response.Stats.RepoRuntime { t.Fatal("how could all take less time, than me alone") } - if response.Stats.RepoRuntime < float64(0.05) { + if response.Stats.RepoRuntime < 0.05 { t.Fatal("the server was too fast") } // see what happens if we try to start it up again - nr := NewTestRepo(server, varDir) - assertRepoIsEmpty(t, nr, false) + // nr := NewTestRepo(l, server, varDir) + // assertRepoIsEmpty(t, nr, false) } func BenchmarkLoadRepo(b *testing.B) { + l := zaptest.NewLogger(b) var ( t = &testing.T{} mockServer, varDir = mock.GetMockData(t) server = mockServer.URL + "/repo-ok.json" - r = NewTestRepo(server, varDir) + r = NewTestRepo(l, server, varDir) ) if len(r.Directory) > 0 { b.Fatal("directory should have been empty, but is not") @@ -119,114 +121,93 @@ func BenchmarkLoadRepo(b *testing.B) { } func TestLoadRepoDuplicateUris(t *testing.T) { + l := zaptest.NewLogger(t) - var ( - mockServer, varDir = mock.GetMockData(t) - server = mockServer.URL + "/repo-duplicate-uris.json" - r = NewTestRepo(server, varDir) - ) + mockServer, varDir := mock.GetMockData(t) + server := mockServer.URL + "/repo-duplicate-uris.json" + r := NewTestRepo(l, server, varDir) response := r.Update() - if response.Success { - t.Fatal("there are duplicates, this repo update should have failed") - } - if !strings.Contains(response.ErrorMessage, "update dimension") { - t.Fatal("error message not as expected: " + response.ErrorMessage) - } + require.False(t, response.Success, "there are duplicates, this repo update should have failed") + + assert.True(t, strings.Contains(response.ErrorMessage, "update dimension"), "error message not as expected: "+response.ErrorMessage) } func TestDimensionHygiene(t *testing.T) { + l := zaptest.NewLogger(t) - var ( - mockServer, varDir = mock.GetMockData(t) - server = mockServer.URL + "/repo-two-dimensions.json" - r = NewTestRepo(server, varDir) - ) + mockServer, varDir := mock.GetMockData(t) + server := mockServer.URL + "/repo-two-dimensions.json" + r := NewTestRepo(l, server, varDir) response := r.Update() - if !response.Success { - t.Fatal("well those two dimension should be fine") - } - r.server = mockServer.URL + "/repo-ok.json" + require.True(t, response.Success, "well those two dimension should be fine") + + r.url = mockServer.URL + "/repo-ok.json" response = r.Update() - if !response.Success { - t.Fatal("wtf it is called repo ok") - } - if len(r.Directory) != 1 { - t.Fatal("directory hygiene failed") - } + require.True(t, response.Success, "it is called repo ok") + + assert.Lenf(t, r.Directory, 1, "directory hygiene failed") } -func getTestRepo(path string, t *testing.T) *Repo { +func getTestRepo(t *testing.T, path string) *Repo { + t.Helper() + l := zaptest.NewLogger(t) + + mockServer, varDir := mock.GetMockData(t) + server := mockServer.URL + path + r := NewTestRepo(l, server, varDir) + response := r.Update() + + require.True(t, response.Success, "well those two dimension should be fine") - var ( - mockServer, varDir = mock.GetMockData(t) - server = mockServer.URL + path - r = NewTestRepo(server, varDir) - response = r.Update() - ) - if !response.Success { - t.Fatal("well those two dimension should be fine") - } return r } func TestGetNodes(t *testing.T) { - var ( - r = getTestRepo("/repo-two-dimensions.json", t) - nodesRequest = mock.MakeNodesRequest() - nodes = r.GetNodes(nodesRequest) - testNode, ok = nodes["test"] - ) - if !ok { - t.Fatal("wtf that should be a node") - } + r := getTestRepo(t, "/repo-two-dimensions.json") + nodesRequest := mock.MakeNodesRequest() + nodes := r.GetNodes(nodesRequest) + testNode, ok := nodes["test"] + + require.True(t, ok, "should be a node") + testData, ok := testNode.Item.Data["foo"] + require.True(t, ok, "failed to fetch test data") + t.Log("testData", testData) - if !ok { - t.Fatal("failed to fetch test data") - } } func TestGetNodesExposeHidden(t *testing.T) { - var ( - r = getTestRepo("/repo-ok-exposehidden.json", t) - nodesRequest = mock.MakeNodesRequest() - ) + r := getTestRepo(t, "/repo-ok-exposehidden.json") + nodesRequest := mock.MakeNodesRequest() nodesRequest.Nodes["test"].ExposeHiddenNodes = true nodes := r.GetNodes(nodesRequest) + testNode, ok := nodes["test"] - if !ok { - t.Fatal("wtf that should be a node") - } + require.True(t, ok, "should be a node") + _, ok = testNode.Item.Data["foo"] - if !ok { - t.Fatal("failed to fetch test data") - } + require.True(t, ok, "failed to fetch test data") + require.Equal(t, 2, len(testNode.Nodes)) } + func TestResolveContent(t *testing.T) { - - var ( - r = getTestRepo("/repo-two-dimensions.json", t) - contentRequest = mock.MakeValidContentRequest() - siteContent, err = r.GetContent(contentRequest) - ) - - if siteContent.URI != contentRequest.URI { - t.Fatal("failed to resolve uri") - } - if err != nil { - t.Fatal(err) - } + r := getTestRepo(t, "/repo-two-dimensions.json") + contentRequest := mock.MakeValidContentRequest() + siteContent, err := r.GetContent(contentRequest) + require.NoError(t, err) + assert.Equal(t, contentRequest.URI, siteContent.URI, "failed to resolve uri") } func TestLinkIds(t *testing.T) { + l := zaptest.NewLogger(t) var ( mockServer, varDir = mock.GetMockData(t) server = mockServer.URL + "/repo-link-ok.json" - r = NewTestRepo(server, varDir) + r = NewTestRepo(l, server, varDir) response = r.Update() ) @@ -234,18 +215,16 @@ func TestLinkIds(t *testing.T) { t.Fatal("those links should have been fine") } - r.server = mockServer.URL + "/repo-link-broken.json" + r.url = mockServer.URL + "/repo-link-broken.json" response = r.Update() if response.Success { t.Fatal("I do not think so") } - } func TestInvalidRequest(t *testing.T) { - - r := getTestRepo("/repo-two-dimensions.json", t) + r := getTestRepo(t, "/repo-two-dimensions.json") if r.validateContentRequest(mock.MakeValidContentRequest()) != nil { t.Fatal("failed validation a valid request") @@ -265,9 +244,9 @@ func TestInvalidRequest(t *testing.T) { rEmptyEnvDimensions.Env.Dimensions = []string{} tests["empty env dimensions"] = rEmptyEnvDimensions - //rNodesValidID := mock.MakeValidContentRequest() - //rNodesValidID.Nodes["id-root"].Id = "" - //tests["nodes must have a valid id"] = rNodesValidID + // rNodesValidID := mock.MakeValidContentRequest() + // rNodesValidID.Nodes["id-root"].Id = "" + // tests["nodes must have a valid id"] = rNodesValidID for comment, req := range tests { if r.validateContentRequest(req) == nil { diff --git a/pkg/repo/repodimension.go b/pkg/repo/repodimension.go new file mode 100644 index 0000000..2e3c8d5 --- /dev/null +++ b/pkg/repo/repodimension.go @@ -0,0 +1,10 @@ +package repo + +import ( + "github.com/foomo/contentserver/content" +) + +type RepoDimension struct { + Dimension string + Node *content.RepoNode +} diff --git a/repo/testdata/order/contentserver-repo-2017-10-21.json b/pkg/repo/testdata/order/contentserver-repo-2017-10-21.json similarity index 100% rename from repo/testdata/order/contentserver-repo-2017-10-21.json rename to pkg/repo/testdata/order/contentserver-repo-2017-10-21.json diff --git a/repo/testdata/order/contentserver-repo-2017-10-22.json b/pkg/repo/testdata/order/contentserver-repo-2017-10-22.json similarity index 100% rename from repo/testdata/order/contentserver-repo-2017-10-22.json rename to pkg/repo/testdata/order/contentserver-repo-2017-10-22.json diff --git a/repo/testdata/order/contentserver-repo-2017-10-23.json b/pkg/repo/testdata/order/contentserver-repo-2017-10-23.json similarity index 100% rename from repo/testdata/order/contentserver-repo-2017-10-23.json rename to pkg/repo/testdata/order/contentserver-repo-2017-10-23.json diff --git a/repo/testdata/order/contentserver-repo-current.json b/pkg/repo/testdata/order/contentserver-repo-current.json similarity index 100% rename from repo/testdata/order/contentserver-repo-current.json rename to pkg/repo/testdata/order/contentserver-repo-current.json diff --git a/pkg/utils/url.go b/pkg/utils/url.go new file mode 100644 index 0000000..c4568b8 --- /dev/null +++ b/pkg/utils/url.go @@ -0,0 +1,15 @@ +package utils + +import ( + "net/url" +) + +func IsValidUrl(str string) bool { + u, err := url.Parse(str) + + if u.Scheme != "http" && u.Scheme != "https" { + return false + } + + return err == nil && u.Scheme != "" && u.Host != "" +} diff --git a/prometheus/prometheus.yml b/prometheus/prometheus.yml deleted file mode 100644 index f6accda..0000000 --- a/prometheus/prometheus.yml +++ /dev/null @@ -1,14 +0,0 @@ -# reference: https://prometheus.io/docs/prometheus/latest/configuration/configuration/ - -global: - scrape_interval: 15s - scrape_timeout: 15s - #evaluation_interval: 15s - -scrape_configs: - - job_name: contentserver - metrics_path: /metrics - scheme: http - static_configs: - - targets: - - 127.0.0.1:9111 diff --git a/repo/history.go b/repo/history.go deleted file mode 100644 index 81f9422..0000000 --- a/repo/history.go +++ /dev/null @@ -1,137 +0,0 @@ -package repo - -import ( - "bytes" - "errors" - "flag" - "fmt" - "io" - "io/ioutil" - "os" - "path" - "sort" - "strings" - "time" - - . "github.com/foomo/contentserver/logger" - "go.uber.org/zap" -) - -const ( - historyRepoJSONPrefix = "contentserver-repo-" - historyRepoJSONSuffix = ".json" -) - -var flagMaxHistoryVersions = flag.Int("max-history", 2, "set the maximum number of content backup files") - -type history struct { - varDir string -} - -func newHistory(varDir string) *history { - return &history{ - varDir: varDir, - } -} - -func (h *history) add(jsonBytes []byte) error { - - var ( - // historiy file name - filename = path.Join(h.varDir, historyRepoJSONPrefix+time.Now().Format(time.RFC3339Nano)+historyRepoJSONSuffix) - err = ioutil.WriteFile(filename, jsonBytes, 0644) - ) - if err != nil { - return err - } - - Log.Info("adding content backup", zap.String("file", filename)) - - // current filename - err = ioutil.WriteFile(h.getCurrentFilename(), jsonBytes, 0644) - if err != nil { - return err - } - - err = h.cleanup() - if err != nil { - Log.Error("an error occured while cleaning up my history", zap.Error(err)) - return err - } - - return nil -} - -func (h *history) getHistory() (files []string, err error) { - fileInfos, err := ioutil.ReadDir(h.varDir) - if err != nil { - return - } - currentName := h.getCurrentFilename() - for _, f := range fileInfos { - if !f.IsDir() { - filename := f.Name() - if filename != currentName && (strings.HasPrefix(filename, historyRepoJSONPrefix) && strings.HasSuffix(filename, historyRepoJSONSuffix)) { - files = append(files, path.Join(h.varDir, filename)) - } - } - } - sort.Sort(sort.Reverse(sort.StringSlice(files))) - return -} - -func (h *history) cleanup() error { - files, err := h.getFilesForCleanup(*flagMaxHistoryVersions) - if err != nil { - return err - } - - for _, f := range files { - Log.Info("removing outdated backup", zap.String("file", f)) - err := os.Remove(f) - if err != nil { - return fmt.Errorf("could not remove file %s : %s", f, err.Error()) - } - } - - return nil -} - -func (h *history) getFilesForCleanup(historyVersions int) (files []string, err error) { - contentFiles, err := h.getHistory() - if err != nil { - return nil, errors.New("could not generate file cleanup list: " + err.Error()) - } - - // fmt.Println("contentFiles:") - // for _, f := range contentFiles { - // fmt.Println(f) - // } - - // -1 to remove the current backup file from the number of items - // so that only files with a timestamp are compared - if len(contentFiles)-1 > historyVersions { - for i := historyVersions + 1; i < len(contentFiles); i++ { - // ignore current repository file to fall back on - if contentFiles[i] == h.getCurrentFilename() { - continue - } - files = append(files, contentFiles[i]) - } - } - return files, nil -} - -func (h *history) getCurrentFilename() string { - return path.Join(h.varDir, historyRepoJSONPrefix+"current"+historyRepoJSONSuffix) -} - -func (h *history) getCurrent(buf *bytes.Buffer) (err error) { - f, err := os.Open(h.getCurrentFilename()) - if err != nil { - return err - } - defer f.Close() - _, err = io.Copy(buf, f) - return err -} diff --git a/repo/history_test.go b/repo/history_test.go deleted file mode 100644 index 4af83ce..0000000 --- a/repo/history_test.go +++ /dev/null @@ -1,92 +0,0 @@ -package repo - -import ( - "bytes" - "fmt" - "io/ioutil" - "os" - "testing" - "time" -) - -func testHistory() *history { - tempDir, err := ioutil.TempDir(os.TempDir(), "contentserver-history-test") - if err != nil { - panic(err) - } - return newHistory(tempDir) -} - -func TestHistoryCurrent(t *testing.T) { - var ( - h = testHistory() - test = []byte("test") - b bytes.Buffer - ) - err := h.add(test) - if err != nil { - t.Fatal("failed to add: ", err) - } - err = h.getCurrent(&b) - if err != nil { - t.Fatal(err) - } - if !bytes.Equal(b.Bytes(), test) { - t.Fatalf("expected %q, got %q", string(test), b.String()) - } -} - -func TestHistoryCleanup(t *testing.T) { - h := testHistory() - for i := 0; i < 50; i++ { - err := h.add([]byte(fmt.Sprint(i))) - if err != nil { - t.Fatal("failed to add: ", err) - } - time.Sleep(time.Millisecond * 5) - } - err := h.cleanup() - if err != nil { - t.Fatal("failed to run cleanup: ", err) - } - files, err := h.getHistory() - if err != nil { - t.Fatal(err) - } - - // -1 for ignoring the current content backup file - if len(files)-1 != *flagMaxHistoryVersions { - t.Fatal("history too long", len(files), "instead of", *flagMaxHistoryVersions) - } -} - -func TestHistoryOrder(t *testing.T) { - h := testHistory() - h.varDir = "testdata/order" - - files, err := h.getHistory() - if err != nil { - t.Fatal("error not expected") - } - assertStringEqual(t, "testdata/order/contentserver-repo-current.json", files[0]) - assertStringEqual(t, "testdata/order/contentserver-repo-2017-10-23.json", files[1]) - assertStringEqual(t, "testdata/order/contentserver-repo-2017-10-22.json", files[2]) - assertStringEqual(t, "testdata/order/contentserver-repo-2017-10-21.json", files[3]) -} - -func TestGetFilesForCleanup(t *testing.T) { - h := testHistory() - h.varDir = "testdata/order" - - files, err := h.getFilesForCleanup(2) - if err != nil { - t.Fatal("error not expected") - } - assertStringEqual(t, "testdata/order/contentserver-repo-2017-10-21.json", files[0]) -} - -func assertStringEqual(t *testing.T, expected, actual string) { - if expected != actual { - t.Errorf("expected string %s differs from the actual %s", expected, actual) - } -} diff --git a/repo/repo.go b/repo/repo.go deleted file mode 100644 index 6cfabae..0000000 --- a/repo/repo.go +++ /dev/null @@ -1,440 +0,0 @@ -package repo - -import ( - "bytes" - "crypto/tls" - "errors" - "fmt" - "io" - "net/http" - "os" - "strings" - "time" - - "github.com/foomo/contentserver/status" - - "go.uber.org/zap" - - "github.com/foomo/contentserver/content" - "github.com/foomo/contentserver/logger" - "github.com/foomo/contentserver/requests" - "github.com/foomo/contentserver/responses" -) - -const maxGetURIForNodeRecursionLevel = 1000 - -// Dimension dimension in a repo -type Dimension struct { - Directory map[string]*content.RepoNode - URIDirectory map[string]*content.RepoNode - Node *content.RepoNode -} - -// Repo content repositiory -type Repo struct { - pollForUpdates bool - pollVersion string - server string - recovered bool - Directory map[string]*Dimension - // updateLock sync.Mutex - dimensionUpdateChannel chan *repoDimension - dimensionUpdateDoneChannel chan error - - history *history - updateInProgressChannel chan chan updateResponse - - // jsonBytes []byte - jsonBuf bytes.Buffer - - httpClient *http.Client -} - -type repoDimension struct { - Dimension string - Node *content.RepoNode -} - -// NewRepo constructor -func NewRepo(server string, varDir string, repositoryTimeout time.Duration, pollForUpdates bool) *Repo { - - logger.Log.Info("creating new repo", - zap.String("server", server), - zap.String("varDir", varDir), - ) - repo := &Repo{ - pollForUpdates: pollForUpdates, - recovered: false, - server: server, - Directory: map[string]*Dimension{}, - history: newHistory(varDir), - dimensionUpdateChannel: make(chan *repoDimension), - dimensionUpdateDoneChannel: make(chan error), - httpClient: getDefaultHTTPClient(repositoryTimeout), - updateInProgressChannel: make(chan chan updateResponse), - } - - go repo.updateRoutine() - go repo.dimensionUpdateRoutine() - if pollForUpdates { - go func() { - ticker := time.NewTicker(10 * time.Second) - for range ticker.C { - chanReponse := make(chan updateResponse) - repo.updateInProgressChannel <- chanReponse - response := <-chanReponse - if response.err == nil { - logger.Log.Info("poll update success", zap.String("revision", repo.pollVersion)) - } else { - logger.Log.Info("poll error", zap.Error(response.err)) - } - } - }() - } - - logger.Log.Info("trying to restore previous state") - restoreErr := repo.tryToRestoreCurrent() - if restoreErr != nil { - logger.Log.Error(" could not restore previous repo content", zap.Error(restoreErr)) - } else { - repo.recovered = true - logger.Log.Info("restored previous repo content") - } - - return repo -} - -func getDefaultHTTPClient(timeout time.Duration) *http.Client { - client := &http.Client{ - Transport: &http.Transport{ - DisableKeepAlives: true, - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - TLSHandshakeTimeout: 5 * time.Second, - }, - Timeout: timeout, - } - return client -} - -func (repo *Repo) Recovered() bool { - return repo.recovered -} - -// GetURIs get many uris at once -func (repo *Repo) GetURIs(dimension string, ids []string) map[string]string { - uris := map[string]string{} - for _, id := range ids { - uris[id] = repo.getURI(dimension, id) - } - return uris -} - -// GetNodes get nodes -func (repo *Repo) GetNodes(r *requests.Nodes) map[string]*content.Node { - return repo.getNodes(r.Nodes, r.Env) -} - -func (repo *Repo) getNodes(nodeRequests map[string]*requests.Node, env *requests.Env) map[string]*content.Node { - - var ( - nodes = map[string]*content.Node{} - path = []*content.Item{} - ) - for nodeName, nodeRequest := range nodeRequests { - if nodeName == "" || nodeRequest.ID == "" { - logger.Log.Info("invalid node request", zap.Error(errors.New("nodeName or nodeRequest.ID empty"))) - continue - } - logger.Log.Debug("adding node", zap.String("name", nodeName), zap.String("requestID", nodeRequest.ID)) - - groups := env.Groups - if len(nodeRequest.Groups) > 0 { - groups = nodeRequest.Groups - } - - dimensionNode, ok := repo.Directory[nodeRequest.Dimension] - nodes[nodeName] = nil - - if !ok && nodeRequest.Dimension == "" { - logger.Log.Debug("Could not get dimension root node", zap.String("dimension", nodeRequest.Dimension)) - for _, dimension := range env.Dimensions { - dimensionNode, ok = repo.Directory[dimension] - if ok { - logger.Log.Debug("Found root node in env.Dimensions", zap.String("dimension", dimension)) - break - } - logger.Log.Debug("Could NOT find root node in env.Dimensions", zap.String("dimension", dimension)) - } - } - - if !ok { - logger.Log.Error("could not get dimension root node", zap.String("nodeRequest.Dimension", nodeRequest.Dimension)) - continue - } - - treeNode, ok := dimensionNode.Directory[nodeRequest.ID] - if !ok { - logger.Log.Error("Invalid tree node requested", - zap.String("nodeName", nodeName), - zap.String("nodeID", nodeRequest.ID), - ) - status.M.InvalidNodeTreeRequests.WithLabelValues().Inc() - continue - } - nodes[nodeName] = repo.getNode(treeNode, nodeRequest.Expand, nodeRequest.MimeTypes, path, 0, groups, nodeRequest.DataFields, nodeRequest.ExposeHiddenNodes) - } - return nodes -} - -// GetContent resolves content and fetches nodes in one call. It combines those -// two tasks for performance reasons. -// -// In the first step it uses r.URI to look up content in all given -// r.Env.Dimensions of repo.Directory. -// -// In the second step it collects the requested nodes. -// -// those two steps are independent. -func (repo *Repo) GetContent(r *requests.Content) (c *content.SiteContent, err error) { - // add more input validation - err = repo.validateContentRequest(r) - if err != nil { - logger.Log.Error("repo.GetContent invalid request", zap.Error(err)) - return - } - logger.Log.Debug("repo.GetContent", zap.String("URI", r.URI)) - c = content.NewSiteContent() - resolved, resolvedURI, resolvedDimension, node := repo.resolveContent(r.Env.Dimensions, r.URI) - if resolved { - if !node.CanBeAccessedByGroups(r.Env.Groups) { - logger.Log.Warn("Resolved content cannot be accessed by specified group", zap.String("URI", r.URI)) - c.Status = content.StatusForbidden - } else { - logger.Log.Info("Content resolved", zap.String("URI", r.URI)) - c.Status = content.StatusOk - c.Data = node.Data - } - c.MimeType = node.MimeType - c.Dimension = resolvedDimension - c.URI = resolvedURI - c.Item = node.ToItem(r.DataFields) - c.Path = node.GetPath(r.PathDataFields) - // fetch URIs for all dimensions - uris := make(map[string]string) - for dimensionName := range repo.Directory { - uris[dimensionName] = repo.getURI(dimensionName, node.ID) - } - c.URIs = uris - } else { - logger.Log.Info("Content not found", zap.String("URI", r.URI)) - c.Status = content.StatusNotFound - c.Dimension = r.Env.Dimensions[0] - - logger.Log.Debug("Failed to resolve, falling back to default dimension", - zap.String("URI", r.URI), - zap.String("defaultDimension", r.Env.Dimensions[0]), - ) - // r.Env.Dimensions is validated => we can access it - resolvedDimension = r.Env.Dimensions[0] - } - - // add navigation trees - for _, node := range r.Nodes { - if node.Dimension == "" { - node.Dimension = resolvedDimension - } - } - c.Nodes = repo.getNodes(r.Nodes, r.Env) - return c, nil -} - -// GetRepo get the whole repo in all dimensions -func (repo *Repo) GetRepo() map[string]*content.RepoNode { - response := make(map[string]*content.RepoNode) - for dimensionName, dimension := range repo.Directory { - response[dimensionName] = dimension.Node - } - return response -} - -// WriteRepoBytes get the whole repo in all dimensions -// reads the JSON history file from the Filesystem and copies it directly in to the supplied buffer -// the result is wrapped as service response, e.g: {"reply": } -func (repo *Repo) WriteRepoBytes(w io.Writer) { - - f, err := os.Open(repo.history.getCurrentFilename()) - if err != nil { - logger.Log.Error("Failed to serve Repo JSON", zap.Error(err)) - } - - _, _ = w.Write([]byte("{\"reply\":")) - _, err = io.Copy(w, f) - if err != nil { - logger.Log.Error("Failed to serve Repo JSON", zap.Error(err)) - } - _, _ = w.Write([]byte("}")) -} - -// Update - reload contents of repository with json from repo.server -func (repo *Repo) Update() (updateResponse *responses.Update) { - floatSeconds := func(nanoSeconds int64) float64 { - return float64(float64(nanoSeconds) / float64(1000000000.0)) - } - - logger.Log.Info("Update triggered") - // Log.Info(ansi.Yellow + "BUFFER LENGTH BEFORE tryUpdate(): " + strconv.Itoa(len(repo.jsonBuf.Bytes())) + ansi.Reset) - - startTime := time.Now().UnixNano() - updateRepotime, updateErr := repo.tryUpdate() - updateResponse = &responses.Update{} - updateResponse.Stats.RepoRuntime = floatSeconds(updateRepotime) - - if updateErr != nil { - updateResponse.Success = false - updateResponse.Stats.NumberOfNodes = -1 - updateResponse.Stats.NumberOfURIs = -1 - - // let us try to restore the world from a file - // Log.Info(ansi.Yellow + "BUFFER LENGTH AFTER ERROR: " + strconv.Itoa(len(repo.jsonBuf.Bytes())) + ansi.Reset) - // only try to restore if the update failed during processing - - if updateErr != errUpdateRejected { - updateResponse.ErrorMessage = updateErr.Error() - logger.Log.Error("Failed to update repository", zap.Error(updateErr)) - - restoreErr := repo.tryToRestoreCurrent() - if restoreErr != nil { - logger.Log.Error("Failed to restore preceding repository version", zap.Error(restoreErr)) - } else { - logger.Log.Info("Successfully restored current repository from local history") - } - } - } else { - updateResponse.Success = true - // persist the currently loaded one - historyErr := repo.history.add(repo.jsonBuf.Bytes()) - if historyErr != nil { - logger.Log.Error("Could not persist current repo in history", zap.Error(historyErr)) - status.M.HistoryPersistFailedCounter.WithLabelValues().Inc() - } - // add some stats - for dimension := range repo.Directory { - updateResponse.Stats.NumberOfNodes += len(repo.Directory[dimension].Directory) - updateResponse.Stats.NumberOfURIs += len(repo.Directory[dimension].URIDirectory) - } - } - updateResponse.Stats.OwnRuntime = floatSeconds(time.Now().UnixNano()-startTime) - updateResponse.Stats.RepoRuntime - return updateResponse -} - -// resolveContent find content in a repository -func (repo *Repo) resolveContent(dimensions []string, URI string) (resolved bool, resolvedURI string, resolvedDimension string, repoNode *content.RepoNode) { - parts := strings.Split(URI, content.PathSeparator) - logger.Log.Debug("repo.ResolveContent", zap.String("URI", URI)) - for i := len(parts); i > 0; i-- { - testURI := strings.Join(parts[0:i], content.PathSeparator) - if testURI == "" { - testURI = content.PathSeparator - } - for _, dimension := range dimensions { - if d, ok := repo.Directory[dimension]; ok { - logger.Log.Debug("Checking node", - zap.String("dimension", dimension), - zap.String("URI", testURI), - ) - if repoNode, ok := d.URIDirectory[testURI]; ok { - resolved = true - logger.Log.Debug("Node found", zap.String("URI", testURI), zap.String("destination", repoNode.DestinationID)) - if len(repoNode.DestinationID) > 0 { - if destionationNode, destinationNodeOk := d.Directory[repoNode.DestinationID]; destinationNodeOk { - repoNode = destionationNode - } - } - return true, testURI, dimension, repoNode - } - } - } - } - return -} - -func (repo *Repo) getURIForNode(dimension string, repoNode *content.RepoNode, recursionLevel int64) (uri string) { - if len(repoNode.LinkID) == 0 { - uri = repoNode.URI - return - } - linkedNode, ok := repo.Directory[dimension].Directory[repoNode.LinkID] - if ok { - if recursionLevel > maxGetURIForNodeRecursionLevel { - logger.Log.Error("maxGetURIForNodeRecursionLevel reached", zap.String("repoNode.ID", repoNode.ID), zap.String("linkID", repoNode.LinkID), zap.String("dimension", dimension)) - return "" - } - return repo.getURIForNode(dimension, linkedNode, recursionLevel+1) - } - return -} - -func (repo *Repo) getURI(dimension string, id string) string { - repoNode, ok := repo.Directory[dimension].Directory[id] - if ok { - return repo.getURIForNode(dimension, repoNode, 0) - } - return "" -} - -func (repo *Repo) getNode( - repoNode *content.RepoNode, - expanded bool, - mimeTypes []string, - path []*content.Item, - level int, - groups []string, - dataFields []string, - exposeHiddenNodes bool, -) *content.Node { - node := content.NewNode() - node.Item = repoNode.ToItem(dataFields) - logger.Log.Debug("getNode", zap.String("ID", repoNode.ID)) - for _, childID := range repoNode.Index { - childNode := repoNode.Nodes[childID] - if (level == 0 || expanded || !expanded && childNode.InPath(path)) && (!childNode.Hidden || exposeHiddenNodes) && childNode.CanBeAccessedByGroups(groups) && childNode.IsOneOfTheseMimeTypes(mimeTypes) { - node.Nodes[childID] = repo.getNode(childNode, expanded, mimeTypes, path, level+1, groups, dataFields, exposeHiddenNodes) - node.Index = append(node.Index, childID) - } - } - return node -} - -func (repo *Repo) validateContentRequest(req *requests.Content) (err error) { - if req == nil { - return errors.New("request must not be nil") - } - if len(req.URI) == 0 { - return errors.New("request URI must not be empty") - } - if req.Env == nil { - return errors.New("request.Env must not be nil") - } - if len(req.Env.Dimensions) == 0 { - return errors.New("request.Env.Dimensions must not be empty") - } - for _, envDimension := range req.Env.Dimensions { - if !repo.hasDimension(envDimension) { - availableDimensions := []string{} - for availableDimension := range repo.Directory { - availableDimensions = append(availableDimensions, availableDimension) - } - return errors.New(fmt.Sprint( - "unknown dimension ", envDimension, - " in r.Env must be one of ", availableDimensions, - " repo has ", len(repo.Directory), " dimensions", - )) - } - } - return nil -} - -func (repo *Repo) hasDimension(d string) bool { - _, hasDimension := repo.Directory[d] - return hasDimension -} diff --git a/requests/content.go b/requests/content.go new file mode 100644 index 0000000..e9f2006 --- /dev/null +++ b/requests/content.go @@ -0,0 +1,10 @@ +package requests + +// Content - the standard request to contentserver +type Content struct { + Env *Env `json:"env"` + URI string `json:"URI"` + Nodes map[string]*Node `json:"nodes"` + DataFields []string `json:"dataFields"` + PathDataFields []string `json:"pathDataFields"` +} diff --git a/requests/env.go b/requests/env.go new file mode 100644 index 0000000..03b81f5 --- /dev/null +++ b/requests/env.go @@ -0,0 +1,9 @@ +package requests + +// Env - abstract your server state +type Env struct { + // when resolving conten these are processed in their order + Dimensions []string `json:"dimensions"` + // who is it for + Groups []string `json:"groups"` +} diff --git a/requests/itemmap.go b/requests/itemmap.go new file mode 100644 index 0000000..1b3276f --- /dev/null +++ b/requests/itemmap.go @@ -0,0 +1,7 @@ +package requests + +// ItemMap - map of items +type ItemMap struct { + ID string `json:"id"` + DataFields []string `json:"dataFields"` +} diff --git a/requests/node.go b/requests/node.go new file mode 100644 index 0000000..8a95f04 --- /dev/null +++ b/requests/node.go @@ -0,0 +1,19 @@ +package requests + +// Node - an abstract node request, use this one to request navigations +type Node struct { + // this one should be obvious + ID string `json:"id"` + // from which dimension + Dimension string `json:"dimension"` + // allowed access groups + Groups []string `json:"groups"` + // what do you want to see in your navigations, folders, images or unicorns + MimeTypes []string `json:"mimeTypes"` + // expand the navigation tree or just the path to the resolved content + Expand bool `json:"expand"` + // Expose hidden nodes + ExposeHiddenNodes bool `json:"exposeHiddenNodes,omitempty"` + // filter with these + DataFields []string `json:"dataFields"` +} diff --git a/requests/nodes.go b/requests/nodes.go new file mode 100644 index 0000000..cd2ed06 --- /dev/null +++ b/requests/nodes.go @@ -0,0 +1,8 @@ +package requests + +// Nodes - which nodes in which dimensions +type Nodes struct { + // map[dimension]*node + Nodes map[string]*Node `json:"nodes"` + Env *Env `json:"env"` +} diff --git a/requests/repo.go b/requests/repo.go new file mode 100644 index 0000000..c117084 --- /dev/null +++ b/requests/repo.go @@ -0,0 +1,4 @@ +package requests + +// Repo - query repo +type Repo struct{} diff --git a/requests/requests.go b/requests/requests.go deleted file mode 100644 index 132cc93..0000000 --- a/requests/requests.go +++ /dev/null @@ -1,63 +0,0 @@ -package requests - -// Env - abstract your server state -type Env struct { - // when resolving conten these are processed in their order - Dimensions []string `json:"dimensions"` - // who is it for - Groups []string `json:"groups"` -} - -// Node - an abstract node request, use this one to request navigations -type Node struct { - // this one should be obvious - ID string `json:"id"` - // from which dimension - Dimension string `json:"dimension"` - // allowed access groups - Groups []string `json:"groups"` - // what do you want to see in your navigations, folders, images or unicorns - MimeTypes []string `json:"mimeTypes"` - // expand the navigation tree or just the path to the resolved content - Expand bool `json:"expand"` - // Expose hidden nodes - ExposeHiddenNodes bool `json:"exposeHiddenNodes,omitempty"` - // filter with these - DataFields []string `json:"dataFields"` -} - -// Nodes - which nodes in which dimensions -type Nodes struct { - // map[dimension]*node - Nodes map[string]*Node `json:"nodes"` - - Env *Env `json:"env"` -} - -// Content - the standard request to contentserver -type Content struct { - Env *Env `json:"env"` - URI string `json:"URI"` - Nodes map[string]*Node `json:"nodes"` - DataFields []string `json:"dataFields"` - PathDataFields []string `json:"pathDataFields"` -} - -// Update - request an update -type Update struct{} - -// Repo - query repo -type Repo struct{} - -// ItemMap - map of items -type ItemMap struct { - ID string `json:"id"` - DataFields []string `json:"dataFields"` -} - -// URIs - request multiple URIs for a dimension use this resolve uris for links -// in a document -type URIs struct { - IDs []string `json:"ids"` - Dimension string `json:"dimension"` -} diff --git a/requests/update.go b/requests/update.go new file mode 100644 index 0000000..781f8f4 --- /dev/null +++ b/requests/update.go @@ -0,0 +1,4 @@ +package requests + +// Update - request an update +type Update struct{} diff --git a/requests/uris.go b/requests/uris.go new file mode 100644 index 0000000..d20b35c --- /dev/null +++ b/requests/uris.go @@ -0,0 +1,8 @@ +package requests + +// URIs - request multiple URIs for a dimension use this resolve uris for links +// in a document +type URIs struct { + IDs []string `json:"ids"` + Dimension string `json:"dimension"` +} diff --git a/responses/responses.go b/responses/error.go similarity index 51% rename from responses/responses.go rename to responses/error.go index ceddbb0..9e14b54 100644 --- a/responses/responses.go +++ b/responses/error.go @@ -1,6 +1,8 @@ package responses -import "fmt" +import ( + "fmt" +) // Error describes an error for humans and machines type Error struct { @@ -21,19 +23,3 @@ func NewError(code int, message string) *Error { Message: message, } } - -// Update - information about an update -type Update struct { - // did it work or not - Success bool `json:"success"` - // this is for humand - ErrorMessage string `json:"errorMessage"` - Stats struct { - NumberOfNodes int `json:"numberOfNodes"` - NumberOfURIs int `json:"numberOfURIs"` - // seconds - RepoRuntime float64 `json:"repoRuntime"` - // seconds - OwnRuntime float64 `json:"ownRuntime"` - } `json:"stats"` -} diff --git a/responses/stats.go b/responses/stats.go new file mode 100644 index 0000000..a89f9bb --- /dev/null +++ b/responses/stats.go @@ -0,0 +1,10 @@ +package responses + +type Stats struct { + NumberOfNodes int `json:"numberOfNodes"` + NumberOfURIs int `json:"numberOfURIs"` + // seconds + RepoRuntime float64 `json:"repoRuntime"` + // seconds + OwnRuntime float64 `json:"ownRuntime"` +} diff --git a/responses/update.go b/responses/update.go new file mode 100644 index 0000000..dfbabc3 --- /dev/null +++ b/responses/update.go @@ -0,0 +1,10 @@ +package responses + +// Update - information about an update +type Update struct { + // did it work or not + Success bool `json:"success"` + // this is for humans + ErrorMessage string `json:"errorMessage"` + Stats Stats `json:"stats"` +} diff --git a/server/handlerequest.go b/server/handlerequest.go deleted file mode 100644 index 2c72e68..0000000 --- a/server/handlerequest.go +++ /dev/null @@ -1,97 +0,0 @@ -package server - -import ( - "time" - - "go.uber.org/zap" - - . "github.com/foomo/contentserver/logger" - "github.com/foomo/contentserver/repo" - "github.com/foomo/contentserver/requests" - "github.com/foomo/contentserver/responses" - "github.com/foomo/contentserver/status" -) - -func handleRequest(r *repo.Repo, handler Handler, jsonBytes []byte, source string) ([]byte, error) { - start := time.Now() - - reply, err := executeRequest(r, handler, jsonBytes, source) - result := "success" - if err != nil { - result = "error" - } - - status.M.ServiceRequestCounter.WithLabelValues(string(handler), result, source).Inc() - status.M.ServiceRequestDuration.WithLabelValues(string(handler), result, source).Observe(time.Since(start).Seconds()) - - return reply, err -} - -func executeRequest(r *repo.Repo, handler Handler, jsonBytes []byte, source string) (replyBytes []byte, err error) { - - var ( - reply interface{} - apiErr error - jsonErr error - processIfJSONIsOk = func(err error, processingFunc func()) { - if err != nil { - jsonErr = err - return - } - processingFunc() - } - ) - status.M.ContentRequestCounter.WithLabelValues(source).Inc() - - // handle and process - switch handler { - // case HandlerGetRepo: // This case is handled prior to handleRequest being called. - // since the resulting bytes are written directly in to the http.ResponseWriter / net.Connection - case HandlerGetURIs: - getURIRequest := &requests.URIs{} - processIfJSONIsOk(json.Unmarshal(jsonBytes, &getURIRequest), func() { - reply = r.GetURIs(getURIRequest.Dimension, getURIRequest.IDs) - }) - case HandlerGetContent: - contentRequest := &requests.Content{} - processIfJSONIsOk(json.Unmarshal(jsonBytes, &contentRequest), func() { - reply, apiErr = r.GetContent(contentRequest) - }) - case HandlerGetNodes: - nodesRequest := &requests.Nodes{} - processIfJSONIsOk(json.Unmarshal(jsonBytes, &nodesRequest), func() { - reply = r.GetNodes(nodesRequest) - }) - case HandlerUpdate: - updateRequest := &requests.Update{} - processIfJSONIsOk(json.Unmarshal(jsonBytes, &updateRequest), func() { - reply = r.Update() - }) - - default: - reply = responses.NewError(1, "unknown handler: "+string(handler)) - } - - // error handling - if jsonErr != nil { - Log.Error("could not read incoming json", zap.Error(jsonErr)) - reply = responses.NewError(2, "could not read incoming json "+jsonErr.Error()) - } else if apiErr != nil { - Log.Error("an API error occured", zap.Error(apiErr)) - reply = responses.NewError(3, "internal error "+apiErr.Error()) - } - - return encodeReply(reply) -} - -// encodeReply takes an interface and encodes it as JSON -// it returns the resulting JSON and a marshalling error -func encodeReply(reply interface{}) (replyBytes []byte, err error) { - replyBytes, err = json.Marshal(map[string]interface{}{ - "reply": reply, - }) - if err != nil { - Log.Error("could not encode reply", zap.Error(err)) - } - return -} diff --git a/server/server.go b/server/server.go deleted file mode 100644 index dde526e..0000000 --- a/server/server.go +++ /dev/null @@ -1,142 +0,0 @@ -package server - -import ( - "errors" - "fmt" - "net" - "net/http" - "os" - "time" - - . "github.com/foomo/contentserver/logger" - "github.com/foomo/contentserver/repo" - jsoniter "github.com/json-iterator/go" - "go.uber.org/zap" -) - -var ( - json = jsoniter.ConfigCompatibleWithStandardLibrary -) - -// Handler type -type Handler string - -const ( - // HandlerGetURIs get uris, many at once, to keep it fast - HandlerGetURIs Handler = "getURIs" - // HandlerGetContent get (site) content - HandlerGetContent = "getContent" - // HandlerGetNodes get nodes - HandlerGetNodes = "getNodes" - // HandlerUpdate update repo - HandlerUpdate = "update" - // HandlerGetRepo get the whole repo - HandlerGetRepo = "getRepo" - - // DefaultRepositoryTimeout for the HTTP client towards the repo - DefaultRepositoryTimeout = 2 * time.Minute -) - -// Run - let it run and enjoy on a socket near you -func Run(server string, address string, varDir string, pollUpdates bool) error { - return RunServerSocketAndWebServer(server, address, "", "", varDir, DefaultRepositoryTimeout, pollUpdates) -} - -func RunServerSocketAndWebServer( - server string, - address string, - webserverAddress string, - webserverPath string, - varDir string, - repositoryTimeout time.Duration, - pollForUpdates bool, -) error { - if address == "" && webserverAddress == "" { - return errors.New("one of the addresses needs to be set") - } - Log.Info("building repo with content", zap.String("server", server)) - - r := repo.NewRepo(server, varDir, repositoryTimeout, pollForUpdates) - - // start initial update and handle error - if !pollForUpdates { - go func() { - resp := r.Update() - if !resp.Success { - Log.Error("failed to update", - zap.String("error", resp.ErrorMessage), - zap.Int("NumberOfNodes", resp.Stats.NumberOfNodes), - zap.Int("NumberOfURIs", resp.Stats.NumberOfURIs), - zap.Float64("OwnRuntime", resp.Stats.OwnRuntime), - zap.Float64("RepoRuntime", resp.Stats.RepoRuntime), - ) - - //Exit only if it hasn't recovered - if !r.Recovered() { - os.Exit(1) - } - } - - }() - } - - // update can run in bg - chanErr := make(chan error) - - if address != "" { - Log.Info("starting socketserver", zap.String("address", address)) - go runSocketServer(r, address, chanErr) - } - if webserverAddress != "" { - Log.Info("starting webserver", zap.String("webserverAddress", webserverAddress)) - go runWebserver(r, webserverAddress, webserverPath, chanErr) - } - return <-chanErr -} - -func runWebserver( - r *repo.Repo, - address string, - path string, - chanErr chan error, -) { - chanErr <- http.ListenAndServe(address, NewWebServer(path, r)) -} - -func runSocketServer( - repo *repo.Repo, - address string, - chanErr chan error, -) { - // create socket server - s := newSocketServer(repo) - - // listen on socket - ln, errListen := net.Listen("tcp", address) - if errListen != nil { - Log.Error("runSocketServer: could not start", - zap.String("address", address), - zap.Error(errListen), - ) - chanErr <- errors.New("runSocketServer: could not start the on \"" + address + "\" - error: " + fmt.Sprint(errListen)) - return - } - - Log.Info("runSocketServer: started listening", zap.String("address", address)) - for { - // this blocks until connection or error - conn, err := ln.Accept() - if err != nil { - Log.Error("runSocketServer: could not accept connection", zap.Error(err)) - continue - } - - // a goroutine handles conn so that the loop can accept other connections - go func() { - Log.Debug("accepted connection", zap.String("source", conn.RemoteAddr().String())) - s.handleConnection(conn) - conn.Close() - // log.Debug("connection closed") - }() - } -} diff --git a/server/socketserver.go b/server/socketserver.go deleted file mode 100644 index bbc7b16..0000000 --- a/server/socketserver.go +++ /dev/null @@ -1,167 +0,0 @@ -package server - -import ( - "bytes" - "errors" - "fmt" - "net" - "strconv" - "strings" - - "go.uber.org/zap" - - . "github.com/foomo/contentserver/logger" - "github.com/foomo/contentserver/repo" - "github.com/foomo/contentserver/responses" - "github.com/foomo/contentserver/status" -) - -const sourceSocketServer = "socketserver" - -type socketServer struct { - repo *repo.Repo -} - -// newSocketServer returns a shiny new socket server -func newSocketServer(repo *repo.Repo) *socketServer { - return &socketServer{ - repo: repo, - } -} - -func extractHandlerAndJSONLentgh(header string) (handler Handler, jsonLength int, err error) { - headerParts := strings.Split(header, ":") - if len(headerParts) != 2 { - return "", 0, errors.New("invalid header") - } - jsonLength, err = strconv.Atoi(headerParts[1]) - if err != nil { - err = fmt.Errorf("could not parse length in header: %q", header) - } - return Handler(headerParts[0]), jsonLength, err -} - -func (s *socketServer) execute(handler Handler, jsonBytes []byte) (reply []byte) { - Log.Debug("incoming json buffer", zap.Int("length", len(jsonBytes))) - - if handler == HandlerGetRepo { - var ( - b bytes.Buffer - ) - s.repo.WriteRepoBytes(&b) - return b.Bytes() - } - - reply, handlingError := handleRequest(s.repo, handler, jsonBytes, sourceSocketServer) - if handlingError != nil { - Log.Error("socketServer.execute failed", zap.Error(handlingError)) - } - return reply -} - -func (s *socketServer) writeResponse(conn net.Conn, reply []byte) { - headerBytes := []byte(strconv.Itoa(len(reply))) - reply = append(headerBytes, reply...) - Log.Debug("replying", zap.String("reply", string(reply))) - n, writeError := conn.Write(reply) - if writeError != nil { - Log.Error("socketServer.writeResponse: could not write reply", zap.Error(writeError)) - return - } - if n < len(reply) { - Log.Error("socketServer.writeResponse: write too short", - zap.Int("got", n), - zap.Int("expected", len(reply)), - ) - return - } - Log.Debug("replied. waiting for next request on open connection") -} - -func (s *socketServer) handleConnection(conn net.Conn) { - defer func() { - if r := recover(); r != nil { - Log.Error("panic in handle connection", zap.String("error", fmt.Sprint(r))) - } - }() - - Log.Debug("socketServer.handleConnection") - status.M.NumSocketsGauge.WithLabelValues(conn.RemoteAddr().String()).Inc() - - var ( - headerBuffer [1]byte - header = "" - i = 0 - ) - for { - i++ - // fmt.Println("---->", i) - // let us read with 1 byte steps on conn until we find "{" - _, readErr := conn.Read(headerBuffer[0:]) - if readErr != nil { - Log.Debug("looks like the client closed the connection", zap.Error(readErr)) - status.M.NumSocketsGauge.WithLabelValues(conn.RemoteAddr().String()).Dec() - return - } - // read next byte - current := headerBuffer[0:] - if string(current) == "{" { - // json has started - handler, jsonLength, headerErr := extractHandlerAndJSONLentgh(header) - // reset header - header = "" - if headerErr != nil { - Log.Error("invalid request could not read header", zap.Error(headerErr)) - encodedErr, encodingErr := encodeReply(responses.NewError(4, "invalid header "+headerErr.Error())) - if encodingErr == nil { - s.writeResponse(conn, encodedErr) - } else { - Log.Error("could not respond to invalid request", zap.Error(encodingErr)) - } - return - } - Log.Debug("found json", zap.Int("length", jsonLength)) - if jsonLength > 0 { - - var ( - // let us try to read some json - jsonBytes = make([]byte, jsonLength) - jsonLengthCurrent = 1 - readRound = 0 - ) - - // that is "{" - jsonBytes[0] = 123 - - for jsonLengthCurrent < jsonLength { - readRound++ - readLength, jsonReadErr := conn.Read(jsonBytes[jsonLengthCurrent:jsonLength]) - if jsonReadErr != nil { - //@fixme we need to force a read timeout (SetReadDeadline?), if expected jsonLength is lower than really sent bytes (e.g. if client implements protocol wrong) - //@todo should we check for io.EOF here - Log.Error("could not read json - giving up with this client connection", zap.Error(jsonReadErr)) - status.M.NumSocketsGauge.WithLabelValues(conn.RemoteAddr().String()).Dec() - return - } - jsonLengthCurrent += readLength - Log.Debug("read cycle status", - zap.Int("jsonLengthCurrent", jsonLengthCurrent), - zap.Int("jsonLength", jsonLength), - zap.Int("readRound", readRound), - ) - } - - Log.Debug("read json", zap.Int("length", len(jsonBytes))) - - s.writeResponse(conn, s.execute(handler, jsonBytes)) - // note: connection remains open - continue - } - Log.Error("can not read empty json") - status.M.NumSocketsGauge.WithLabelValues(conn.RemoteAddr().String()).Dec() - return - } - // adding to header byte by byte - header += string(headerBuffer[0:]) - } -} diff --git a/server/webserver.go b/server/webserver.go deleted file mode 100644 index da17041..0000000 --- a/server/webserver.go +++ /dev/null @@ -1,59 +0,0 @@ -package server - -import ( - "fmt" - "io/ioutil" - "net/http" - "strings" - - "go.uber.org/zap" - - . "github.com/foomo/contentserver/logger" - "github.com/foomo/contentserver/repo" -) - -type webServer struct { - path string - r *repo.Repo -} - -// NewWebServer returns a shiny new web server -func NewWebServer(path string, r *repo.Repo) http.Handler { - return &webServer{ - path: path, - r: r, - } -} - -func (s *webServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { - defer func() { - if r := recover(); r != nil { - Log.Error("Panic in handle connection", zap.String("error", fmt.Sprint(r))) - } - }() - - if r.Body == nil { - http.Error(w, "no body", http.StatusBadRequest) - return - } - jsonBytes, readErr := ioutil.ReadAll(r.Body) - if readErr != nil { - http.Error(w, "failed to read incoming request", http.StatusBadRequest) - return - } - h := Handler(strings.TrimPrefix(r.URL.Path, s.path+"/")) - if h == HandlerGetRepo { - s.r.WriteRepoBytes(w) - w.Header().Set("Content-Type", "application/json") - return - } - reply, errReply := handleRequest(s.r, h, jsonBytes, "webserver") - if errReply != nil { - http.Error(w, errReply.Error(), http.StatusInternalServerError) - return - } - _, err := w.Write(reply) - if err != nil { - Log.Error("failed to write webServer reply", zap.Error(err)) - } -} diff --git a/status/healthz.go b/status/healthz.go deleted file mode 100644 index 077aa11..0000000 --- a/status/healthz.go +++ /dev/null @@ -1,37 +0,0 @@ -package status - -import ( - "fmt" - "net/http" - - . "github.com/foomo/contentserver/logger" - jsoniter "github.com/json-iterator/go" - "go.uber.org/zap" -) - -var ( - json = jsoniter.ConfigCompatibleWithStandardLibrary -) - -func RunHealthzHandlerListener(address string, serviceName string) { - Log.Info(fmt.Sprintf("starting healthz handler on '%s'" + address)) - Log.Error("healthz server failed", zap.Error(http.ListenAndServe(address, HealthzHandler(serviceName)))) -} - -func HealthzHandler(serviceName string) http.Handler { - var ( - data = map[string]string{ - "service": serviceName, - } - status, _ = json.Marshal(data) - h = http.NewServeMux() - ) - h.Handle("/healthz", http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - _, err := w.Write(status) - if err != nil { - Log.Error("failed to write healthz status", zap.Error(err)) - } - })) - - return h -} diff --git a/status/metrics.go b/status/metrics.go deleted file mode 100644 index b79135a..0000000 --- a/status/metrics.go +++ /dev/null @@ -1,119 +0,0 @@ -package status - -import ( - "github.com/prometheus/client_golang/prometheus" -) - -// M is the Metrics instance -var M = newMetrics() - -const ( - namespace = "contentserver" - - metricLabelHandler = "handler" - metricLabelStatus = "status" - metricLabelSource = "source" - metricLabelRemote = "remote" -) - -// Metrics is the structure that holds all prometheus metrics -type Metrics struct { - ServiceRequestCounter *prometheus.CounterVec // count the number of requests for each service function - ServiceRequestDuration *prometheus.SummaryVec // observe the duration of requests for each service function - UpdatesRejectedCounter *prometheus.CounterVec // count the number of completed updates - UpdatesCompletedCounter *prometheus.CounterVec // count the number of rejected updates - UpdatesFailedCounter *prometheus.CounterVec // count the number of updates that had an error - UpdateDuration *prometheus.SummaryVec // observe the duration of each repo.update() call - ContentRequestCounter *prometheus.CounterVec // count the total number of content requests - NumSocketsGauge *prometheus.GaugeVec // keep track of the total number of open sockets - HistoryPersistFailedCounter *prometheus.CounterVec // count the number of failed attempts to persist the content history - InvalidNodeTreeRequests *prometheus.CounterVec // counts the number of invalid tree node requests -} - -// newMetrics can be used to instantiate a metrics instance -// since this function will also register each metric and metrics should only be registered once -// it is private -// the package exposes the initialized Metrics instance as the variable M. -func newMetrics() *Metrics { - return &Metrics{ - InvalidNodeTreeRequests: newCounterVec("invalid_node_tree_request_count", - "Counts the number of invalid tree nodes for a specific node"), - ServiceRequestCounter: newCounterVec( - "service_request_count", - "Count of requests for each handler", - metricLabelHandler, metricLabelStatus, metricLabelSource, - ), - ServiceRequestDuration: newSummaryVec( - "service_request_duration_seconds", - "Seconds to unmarshal requests, execute a service function and marshal its reponses", - metricLabelHandler, metricLabelStatus, metricLabelSource, - ), - UpdatesRejectedCounter: newCounterVec( - "updates_rejected_count", - "Number of updates that were rejected because the queue was full", - ), - UpdatesCompletedCounter: newCounterVec( - "updates_completed_count", - "Number of updates that were successfully completed", - ), - UpdatesFailedCounter: newCounterVec( - "updates_failed_count", - "Number of updates that failed due to an error", - ), - UpdateDuration: newSummaryVec( - "update_duration_seconds", - "Duration in seconds for each successful repo.update() call", - ), - ContentRequestCounter: newCounterVec( - "content_request_count", - "Number of requests for content", - metricLabelSource, - ), - NumSocketsGauge: newGaugeVec( - "num_sockets_total", - "Total number of currently open socket connections", - metricLabelRemote, - ), - HistoryPersistFailedCounter: newCounterVec( - "history_persist_failed_count", - "Number of failures to store the content history on the filesystem", - ), - } -} - -/* - * Metric constructors - */ - -func newSummaryVec(name, help string, labels ...string) *prometheus.SummaryVec { - vec := prometheus.NewSummaryVec( - prometheus.SummaryOpts{ - Namespace: namespace, - Name: name, - Help: help, - }, labels) - prometheus.MustRegister(vec) - return vec -} - -func newCounterVec(name, help string, labels ...string) *prometheus.CounterVec { - vec := prometheus.NewCounterVec( - prometheus.CounterOpts{ - Namespace: namespace, - Name: name, - Help: help, - }, labels) - prometheus.MustRegister(vec) - return vec -} - -func newGaugeVec(name, help string, labels ...string) *prometheus.GaugeVec { - vec := prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Namespace: namespace, - Name: name, - Help: help, - }, labels) - prometheus.MustRegister(vec) - return vec -} diff --git a/testing/client/client.go b/testing/client/client.go deleted file mode 100644 index 56e888b..0000000 --- a/testing/client/client.go +++ /dev/null @@ -1,57 +0,0 @@ -package main - -import ( - "flag" - "log" - "time" - - "github.com/foomo/contentserver/client" -) - -var ( - flagAddr = flag.String("addr", "http://127.0.0.1:9191/contentserver", "set addr") - flagGetRepo = flag.Bool("getRepo", false, "get repo") - flagUpdate = flag.Bool("update", true, "trigger content update") - flagNum = flag.Int("num", 100, "num repititions") - flagDelay = flag.Int("delay", 2, "delay in seconds") -) - -func main() { - - flag.Parse() - - c, errClient := client.NewHTTPClient(*flagAddr) - if errClient != nil { - log.Fatal(errClient) - } - - for i := 1; i <= *flagNum; i++ { - - if *flagUpdate { - go func(num int) { - log.Println("start update") - resp, errUpdate := c.Update() - if errUpdate != nil { - log.Fatal(errUpdate) - } - log.Println(num, "update done", resp) - }(i) - } - - if *flagGetRepo { - go func(num int) { - log.Println("GetRepo", num) - resp, err := c.GetRepo() - if err != nil { - // spew.Dump(resp) - log.Fatal("failed to get repo") - } - log.Println(num, "GetRepo done, got", len(resp), "dimensions") - }(i) - } - - time.Sleep(time.Duration(*flagDelay) * time.Second) - } - - log.Println("done!") -} diff --git a/testing/server/server.go b/testing/server/server.go deleted file mode 100644 index 588c158..0000000 --- a/testing/server/server.go +++ /dev/null @@ -1,35 +0,0 @@ -package main - -import ( - "flag" - "log" - "net/http" -) - -type testServer struct { - file string -} - -func main() { - - var ( - flagJSONFile = flag.String("json-file", "", "provide a json source file") - flagAddress = flag.String("addr", ":1234", "set the webserver address") - ) - flag.Parse() - - if *flagJSONFile == "" { - log.Fatal("js source file must be provided") - } - - ts := &testServer{ - file: *flagJSONFile, - } - - log.Println("start test server at", *flagAddress, "serving file:", ts.file) - log.Fatal(http.ListenAndServe(*flagAddress, ts)) -} - -func (ts *testServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { - http.ServeFile(w, r, ts.file) -}