foomo/contentserver
1
0
Fork 0
mirror of https://github.com/foomo/contentserver.git synced 2025-10-16 12:25:44 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Merge pull request #14 from foomo/feature/memdebug

Browse Source 
merge feature/memdebug
This commit is contained in:
Phil 2019-06-04 08:57:20 +02:00 committed by GitHub
commit fd0c81bc23
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
37 changed files with 2502 additions and 482 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.gitignore vendored
View File

@ -1,5 +1,11 @@
data
*.log
*.test
cprof-*
var
.* .*
*~ *~
/bin/ /bin/
/pkg/tmp/ /pkg/tmp/
/vendor
!.git* !.git*

13
.travis.yml
View File

@ -1,5 +1,10 @@
language: go language: go
go: go: "1.12"
- 1.10 os:
- 1.11 - linux
- tip dist: trusty
sudo: false
install: true
script:
- make dep
- make test

38
Dockerfile
View File

@ -1,21 +1,37 @@
FROM scratch ##############################
###### STAGE: BUILD ######
##############################
FROM golang:1.12.5 AS build-env
COPY bin/contentserver-linux-amd64 /usr/sbin/contentserver WORKDIR /src
# install ca root certificates COPY ./go.mod ./go.sum ./
# https://curl.haxx.se/docs/caextract.html RUN go mod download && go mod vendor && go install -i ./vendor/...
# http://blog.codeship.com/building-minimal-docker-containers-for-go-applications/
# does not work on docker for mac :( # Import the code from the context.
# ADD https://curl.haxx.se/ca/cacert.pem /etc/ssl/certs/ca-certificates.crt COPY ./ ./
ADD .cacert.pem /etc/ssl/certs/ca-certificates.crt
RUN GOARCH=amd64 GOOS=linux CGO_ENABLED=0 go build -o /contentserver
##############################
###### STAGE: PACKAGE ######
##############################
FROM alpine
ENV CONTENT_SERVER_LOG_LEVEL=error
ENV CONTENT_SERVER_ADDR=0.0.0.0:80 ENV CONTENT_SERVER_ADDR=0.0.0.0:80
ENV CONTENT_SERVER_VAR_DIR=/var/lib/contentserver ENV CONTENT_SERVER_VAR_DIR=/var/lib/contentserver
ENV LOG_JSON=1
RUN apk add --update --no-cache ca-certificates curl bash && rm -rf /var/cache/apk/*
COPY --from=build-env /contentserver /usr/sbin/contentserver
VOLUME $CONTENT_SERVER_VAR_DIR VOLUME $CONTENT_SERVER_VAR_DIR
EXPOSE 80
ENTRYPOINT ["/usr/sbin/contentserver"] ENTRYPOINT ["/usr/sbin/contentserver"]
CMD ["-address=$CONTENT_SERVER_ADDR", "-log-level=$CONTENT_SERVER_LOG_LEVEL", "-var-dir=$CONTENT_SERVER_VAR_DIR"] CMD ["-address=$CONTENT_SERVER_ADDR", "-var-dir=$CONTENT_SERVER_VAR_DIR"]
EXPOSE 80
EXPOSE 9200

74
Makefile
View File

@ -1,12 +1,20 @@
SHELL := /bin/bash SHELL := /bin/bash
TAG=`git describe --exact-match --tags $(git log -n1 --pretty='%h') 2>/dev/null || git rev-parse --abbrev-ref HEAD` TAG?=latest
IMAGE=docker-registry.bestbytes.net/contentserver
# Utils
all: build test all: build test
tag: tag:
echo $(TAG) echo $(TAG)
dep:
env GO111MODULE=on go mod download && env GO111MODULE=on go mod vendor && go install -i ./vendor/...
clean: clean:
rm -fv bin/contentserve* rm -fv bin/contentserve*
# Build
build: clean build: clean
go build -o bin/contentserver go build -o bin/contentserver
build-arch: clean build-arch: clean
@ -15,11 +23,71 @@ build-arch: clean
build-docker: clean build-arch build-docker: clean build-arch
curl https://curl.haxx.se/ca/cacert.pem > .cacert.pem curl https://curl.haxx.se/ca/cacert.pem > .cacert.pem
docker build -q . > .image_id docker build -q . > .image_id
docker tag `cat .image_id` docker-registry.bestbytes.net/contentserver:$(TAG) docker tag `cat .image_id` $(IMAGE):$(TAG)
echo "# tagged container `cat .image_id` as docker-registry.bestbytes.net/contentserver:$(TAG)" echo "# tagged container `cat .image_id` as $(IMAGE):$(TAG)"
rm -vf .image_id .cacert.pem rm -vf .image_id .cacert.pem
build-testclient:
go build -o bin/testclient -i github.com/foomo/contentserver/testing/client
build-testserver:
go build -o bin/testserver -i github.com/foomo/contentserver/testing/server
package: build package: build
pkg/build.sh pkg/build.sh
# Docker
docker-build:
docker build -t $(IMAGE):$(TAG) .
docker-push:
docker push $(IMAGE):$(TAG)
# Testing / benchmarks
test: test:
go test ./... go test ./...
bench:
go test -run=none -bench=. ./...
run-testserver:
bin/testserver -json-file var/cse-globus-stage-b-with-main-section.json
run-contentserver:
contentserver -var-dir var -webserver-address :9191 -address :9999 http://127.0.0.1:1234
run-contentserver-freeosmem:
contentserver -var-dir var -webserver-address :9191 -address :9999 -free-os-mem 1 http://127.0.0.1:1234
run-prometheus:
prometheus --config.file=prometheus/prometheus.yml
clean-var:
rm var/contentserver-repo-2019*
# Profiling
test-cpu-profile:
go test -cpuprofile=cprof-client github.com/foomo/contentserver/client
go tool pprof --text client.test cprof-client
go test -cpuprofile=cprof-repo github.com/foomo/contentserver/repo
go tool pprof --text repo.test cprof-repo
test-gctrace:
GODEBUG=gctrace=1 go test ./...
test-malloctrace:
GODEBUG=allocfreetrace=1 go test ./...
trace:
curl http://localhost:6060/debug/pprof/trace?seconds=60 > cs-trace
go tool trace cs-trace
pprof-heap-web:
go tool pprof -http=":8081" http://localhost:6060/debug/pprof/heap
pprof-cpu-web:
go tool pprof -http=":8081" http://localhost:6060/debug/pprof/profile

24
README.md
View File

@ -10,6 +10,10 @@ A Server written in GoLang to mix and resolve content from different content sou
It's up to you how you use it and which data you want to export to the server. Our intention was to write a fast and cache hazzle-free content server to mix different content sources. It's up to you how you use it and which data you want to export to the server. Our intention was to write a fast and cache hazzle-free content server to mix different content sources.
### Overview
<img src="graphics/Overview.svg" width="100%" height="500">
## Export Data ## Export Data
All you have to do is to provide a tree of content nodes as a JSON encoded RepoNode. All you have to do is to provide a tree of content nodes as a JSON encoded RepoNode.
@ -39,19 +43,31 @@ All you have to do is to provide a tree of content nodes as a JSON encoded RepoN
There is a PHP Proxy implementation for foomo in [Foomo.ContentServer](https://github.com/foomo/Foomo.ContentServer). Feel free to use it or to implement your own proxy in the language you love. The API should be easily to implement in every other framework and language, too. There is a PHP Proxy implementation for foomo in [Foomo.ContentServer](https://github.com/foomo/Foomo.ContentServer). Feel free to use it or to implement your own proxy in the language you love. The API should be easily to implement in every other framework and language, too.
## Update Flowchart
<img src="graphics/Update-Flow.svg" width="100%" height="700">
### Usage ### Usage
```bash ```bash
$ contentserver --help $ contentserver -h
Usage of contentserver: Usage of contentserver:
-address string -address string
address to bind host:port (default "127.0.0.1:8081") address to bind socket server host:port
-log-level string -debug
one of error, record, warning, notice, debug (default "record") toggle debug mode
-free-os-mem int
free OS mem every X minutes
-heap-dump int
dump heap every X minutes
-var-dir string -var-dir string
where to put my data (default "/var/lib/contentserver") where to put my data (default "/var/lib/contentserver")
-version -version
version info version info
-webserver-address string
address to bind web server host:port, when empty no webserver will be spawned
-webserver-path string
path to export the webserver on - useful when behind a proxy (default "/contentserver")
``` ```
## License ## License

51
client/client_test.go
View File

@ -1,7 +1,6 @@
package client package client
import ( import (
"encoding/json"
"net" "net"
"strconv" "strconv"
"sync" "sync"
@ -9,7 +8,7 @@ import (
"time" "time"
"github.com/foomo/contentserver/content" "github.com/foomo/contentserver/content"
"github.com/foomo/contentserver/log" . "github.com/foomo/contentserver/logger"
"github.com/foomo/contentserver/repo/mock" "github.com/foomo/contentserver/repo/mock"
"github.com/foomo/contentserver/requests" "github.com/foomo/contentserver/requests"
"github.com/foomo/contentserver/server" "github.com/foomo/contentserver/server"
@ -17,6 +16,15 @@ import (
const pathContentserver = "/contentserver" const pathContentserver = "/contentserver"
var (
testServerSocketAddr string
testServerWebserverAddr string
)
func init() {
SetupLogging(true, "contentserver_client_test.log")
}
func dump(t *testing.T, v interface{}) { func dump(t *testing.T, v interface{}) {
jsonBytes, err := json.MarshalIndent(v, "", " ") jsonBytes, err := json.MarshalIndent(v, "", " ")
if err != nil { if err != nil {
@ -43,21 +51,24 @@ func getAvailableAddr() string {
return "127.0.0.1:" + strconv.Itoa(getFreePort()) return "127.0.0.1:" + strconv.Itoa(getFreePort())
} }
var testServerSocketAddr string
var testServerWebserverAddr string
func initTestServer(t testing.TB) (socketAddr, webserverAddr string) { func initTestServer(t testing.TB) (socketAddr, webserverAddr string) {
socketAddr = getAvailableAddr() socketAddr = getAvailableAddr()
webserverAddr = getAvailableAddr() webserverAddr = getAvailableAddr()
testServer, varDir := mock.GetMockData(t) testServer, varDir := mock.GetMockData(t)
log.SelectedLevel = log.LevelError
go server.RunServerSocketAndWebServer( go func() {
testServer.URL+"/repo-two-dimensions.json", err := server.RunServerSocketAndWebServer(
socketAddr, testServer.URL+"/repo-two-dimensions.json",
webserverAddr, socketAddr,
pathContentserver, webserverAddr,
varDir, pathContentserver,
) varDir,
)
if err != nil {
t.Fatal("test server crashed: ", err)
}
}()
socketClient, errClient := NewClient(socketAddr, 1, time.Duration(time.Millisecond*100)) socketClient, errClient := NewClient(socketAddr, 1, time.Duration(time.Millisecond*100))
if errClient != nil { if errClient != nil {
panic(errClient) panic(errClient)
@ -69,6 +80,11 @@ func initTestServer(t testing.TB) (socketAddr, webserverAddr string) {
if err != nil { if err != nil {
continue continue
} }
if len(r) == 0 {
t.Fatal("received empty JSON from GetRepo")
}
if r["dimension_foo"].Nodes["id-a"].Data["baz"].(float64) == float64(1) { if r["dimension_foo"].Nodes["id-a"].Data["baz"].(float64) == float64(1) {
break break
} }
@ -139,10 +155,18 @@ func TestGetURIs(t *testing.T) {
func TestGetRepo(t *testing.T) { func TestGetRepo(t *testing.T) {
testWithClients(t, func(c *Client) { testWithClients(t, func(c *Client) {
time.Sleep(time.Millisecond * 100)
r, err := c.GetRepo() r, err := c.GetRepo()
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
if len(r) == 0 {
t.Fatal("received empty JSON from GetRepo")
}
if r["dimension_foo"].Nodes["id-a"].Data["baz"].(float64) != float64(1) { if r["dimension_foo"].Nodes["id-a"].Data["baz"].(float64) != float64(1) {
t.Fatal("failed to drill deep for data") t.Fatal("failed to drill deep for data")
} }
@ -235,5 +259,4 @@ func benchmarkClientAndServerGetContent(b testing.TB, numGroups, numCalls int, c
} }
// Wait for all HTTP fetches to complete. // Wait for all HTTP fetches to complete.
wg.Wait() wg.Wait()
return
} }

17
client/connectionpool.go
View File

@ -6,8 +6,8 @@ import (
) )
type connectionPool struct { type connectionPool struct {
server string server string
conn net.Conn // conn net.Conn
chanConnGet chan chan net.Conn chanConnGet chan chan net.Conn
chanConnReturn chan connReturn chanConnReturn chan connReturn
chanDrainPool chan int chanDrainPool chan int
@ -34,8 +34,11 @@ func (c *connectionPool) run(connectionPoolSize int, waitTimeout time.Duration)
entryTime time.Time entryTime time.Time
chanConn chan net.Conn chanConn chan net.Conn
} }
connectionPool := make(map[int]*poolEntry, connectionPoolSize)
waitPool := map[int]*waitPoolEntry{} var (
connectionPool = make(map[int]*poolEntry, connectionPoolSize)
waitPool = map[int]*waitPoolEntry{}
)
for i := 0; i < connectionPoolSize; i++ { for i := 0; i < connectionPoolSize; i++ {
connectionPool[i] = &poolEntry{ connectionPool[i] = &poolEntry{
conn: nil, conn: nil,
@ -110,8 +113,10 @@ RunLoop:
} }
} }
// waitpool cleanup // waitpool cleanup
waitPoolLoosers := []int{} var (
now := time.Now() waitPoolLoosers = []int{}
now = time.Now()
)
for i, waitPoolEntry := range waitPool { for i, waitPoolEntry := range waitPool {
if now.Sub(waitPoolEntry.entryTime) > waitTimeout { if now.Sub(waitPoolEntry.entryTime) > waitTimeout {
waitPoolLoosers = append(waitPoolLoosers, i) waitPoolLoosers = append(waitPoolLoosers, i)

1
client/httptransport.go
View File

@ -2,7 +2,6 @@ package client
import ( import (
"bytes" "bytes"
"encoding/json"
"errors" "errors"
"io/ioutil" "io/ioutil"
"net/http" "net/http"

25
client/sockettransport.go
View File

@ -1,7 +1,6 @@
package client package client
import ( import (
"encoding/json"
"errors" "errors"
"fmt" "fmt"
"io" "io"
@ -11,8 +10,11 @@ import (
"github.com/foomo/contentserver/responses" "github.com/foomo/contentserver/responses"
"github.com/foomo/contentserver/server" "github.com/foomo/contentserver/server"
jsoniter "github.com/json-iterator/go"
) )
var json = jsoniter.ConfigCompatibleWithStandardLibrary
type serverResponse struct { type serverResponse struct {
Reply interface{} Reply interface{}
} }
@ -62,8 +64,10 @@ func (c *socketTransport) call(handler server.Handler, request interface{}, resp
jsonBytes = append([]byte(fmt.Sprintf("%s:%d", handler, len(jsonBytes))), jsonBytes...) jsonBytes = append([]byte(fmt.Sprintf("%s:%d", handler, len(jsonBytes))), jsonBytes...)
// send request // send request
written := 0 var (
l := len(jsonBytes) written = 0
l = len(jsonBytes)
)
for written < l { for written < l {
n, err := conn.Write(jsonBytes[written:]) n, err := conn.Write(jsonBytes[written:])
if err != nil { if err != nil {
@ -74,9 +78,11 @@ func (c *socketTransport) call(handler server.Handler, request interface{}, resp
} }
// read response // read response
responseBytes := []byte{} var (
buf := make([]byte, 4096) responseBytes = []byte{}
responseLength := 0 buf = make([]byte, 4096)
responseLength = 0
)
for { for {
n, err := conn.Read(buf) n, err := conn.Read(buf)
if err != nil && err != io.EOF { if err != nil && err != io.EOF {
@ -105,12 +111,15 @@ func (c *socketTransport) call(handler server.Handler, request interface{}, resp
break break
} }
} }
// unmarshal response // unmarshal response
responseJSONErr := json.Unmarshal(responseBytes, &serverResponse{Reply: response}) responseJSONErr := json.Unmarshal(responseBytes, &serverResponse{Reply: response})
if responseJSONErr != nil { if responseJSONErr != nil {
// is it an error ? // is it an error ?
remoteErr := responses.Error{} var (
remoteErrJSONErr := json.Unmarshal(responseBytes, remoteErr) remoteErr = responses.Error{}
remoteErrJSONErr = json.Unmarshal(responseBytes, &remoteErr)
)
if remoteErrJSONErr == nil { if remoteErrJSONErr == nil {
returnConn(remoteErrJSONErr) returnConn(remoteErrJSONErr)
return remoteErr return remoteErr

2
content/content.go
View File

@ -4,6 +4,6 @@ package content
const ( const (
// Indent for json indentation // Indent for json indentation
Indent string = "\t" Indent string = "\t"
// PathSeparator seprator for paths in URIs // PathSeparator separator for paths in URIs
PathSeparator = "/" PathSeparator = "/"
) )

28
content/reponode.go
View File

@ -22,13 +22,13 @@ type RepoNode struct {
// published from - to is going to be an array of fromTos // published from - to is going to be an array of fromTos
} }
// NewRepoNode constructor // // NewRepoNode constructor
func NewRepoNode() *RepoNode { // func NewRepoNode() *RepoNode {
return &RepoNode{ // return &RepoNode{
Data: make(map[string]interface{}), // Data: make(map[string]interface{}, 0), // set initial size to zero explicitely?
Nodes: make(map[string]*RepoNode), // Nodes: make(map[string]*RepoNode, 0),
} // }
} // }
// WireParents helper method to reference from child to parent in a tree // WireParents helper method to reference from child to parent in a tree
// recursively // recursively
@ -52,15 +52,21 @@ func (node *RepoNode) InPath(path []*Item) bool {
// GetPath get a path for a repo node // GetPath get a path for a repo node
func (node *RepoNode) GetPath() []*Item { func (node *RepoNode) GetPath() []*Item {
parentNode := node.parent
pathLength := 0 var (
parentNode = node.parent
pathLength = 0
)
for parentNode != nil { for parentNode != nil {
parentNode = parentNode.parent parentNode = parentNode.parent
pathLength++ pathLength++
} }
parentNode = node.parent parentNode = node.parent
i := 0
path := make([]*Item, pathLength) var (
i = 0
path = make([]*Item, pathLength)
)
for parentNode != nil { for parentNode != nil {
path[i] = parentNode.ToItem([]string{}) path[i] = parentNode.ToItem([]string{})
parentNode = parentNode.parent parentNode = parentNode.parent

111
contentserver.go
View File

@ -3,11 +3,18 @@ package main
import ( import (
"flag" "flag"
"fmt" "fmt"
"net/http"
_ "net/http/pprof"
"os" "os"
"strings" "runtime/debug"
"time"
"github.com/foomo/contentserver/log" "github.com/apex/log"
. "github.com/foomo/contentserver/logger"
"github.com/foomo/contentserver/metrics"
"github.com/foomo/contentserver/server" "github.com/foomo/contentserver/server"
"github.com/foomo/contentserver/status"
"go.uber.org/zap"
) )
const ( const (
@ -16,30 +23,22 @@ const (
logLevelWarning = "warning" logLevelWarning = "warning"
logLevelRecord = "record" logLevelRecord = "record"
logLevelError = "error" logLevelError = "error"
ServiceName = "Content Server"
DefaultHealthzHandlerAddress = ":8080"
DefaultPrometheusListener = "127.0.0.1:9111"
) )
var ( var (
uniqushPushVersion = "content-server 1.4.1" flagAddress = flag.String("address", "", "address to bind socket server host:port")
showVersionFlag = flag.Bool("version", false, "version info") flagWebserverAddress = flag.String("webserver-address", "", "address to bind web server host:port, when empty no webserver will be spawned")
address = flag.String("address", "", "address to bind socket server host:port") flagWebserverPath = flag.String("webserver-path", "/contentserver", "path to export the webserver on - useful when behind a proxy")
webserverAddress = flag.String("webserver-address", "", "address to bind web server host:port, when empty no webserver will be spawned") flagVarDir = flag.String("var-dir", "/var/lib/contentserver", "where to put my data")
webserverPath = flag.String("webserver-path", "/contentserver", "path to export the webserver on - useful when behind a proxy")
varDir = flag.String("var-dir", "/var/lib/contentserver", "where to put my data") // debugging / profiling
logLevelOptions = []string{ flagDebug = flag.Bool("debug", false, "toggle debug mode")
logLevelError, flagFreeOSMem = flag.Int("free-os-mem", 0, "free OS mem every X minutes")
logLevelRecord, flagHeapDump = flag.Int("heap-dump", 0, "dump heap every X minutes")
logLevelWarning,
logLevelNotice,
logLevelDebug,
}
logLevel = flag.String(
"log-level",
logLevelRecord,
fmt.Sprintf(
"one of %s",
strings.Join(logLevelOptions, ", "),
),
)
) )
func exitUsage(code int) { func exitUsage(code int) {
@ -50,27 +49,55 @@ func exitUsage(code int) {
func main() { func main() {
flag.Parse() flag.Parse()
if *showVersionFlag {
fmt.Printf("%v\n", uniqushPushVersion) SetupLogging(*flagDebug, "contentserver.log")
return
go func() {
fmt.Println(http.ListenAndServe("localhost:6060", nil))
}()
if *flagFreeOSMem > 0 {
Log.Info("freeing OS memory every $interval minutes", zap.Int("interval", *flagFreeOSMem))
go func() {
for {
select {
case <-time.After(time.Duration(*flagFreeOSMem) * time.Minute):
log.Info("FreeOSMemory")
debug.FreeOSMemory()
}
}
}()
} }
if *flagHeapDump > 0 {
Log.Info("dumping heap every $interval minutes", zap.Int("interval", *flagHeapDump))
go func() {
for {
select {
case <-time.After(time.Duration(*flagFreeOSMem) * time.Minute):
log.Info("HeapDump")
f, err := os.Create("heapdump")
if err != nil {
panic("failed to create heap dump file")
}
debug.WriteHeapDump(f.Fd())
err = f.Close()
if err != nil {
panic("failed to create heap dump file")
}
}
}
}()
}
if len(flag.Args()) == 1 { if len(flag.Args()) == 1 {
fmt.Println(*address, flag.Arg(0)) fmt.Println(*flagAddress, flag.Arg(0))
level := log.LevelRecord
switch *logLevel { // kickoff metric handlers
case logLevelError: go metrics.RunPrometheusHandler(DefaultPrometheusListener)
level = log.LevelError go status.RunHealthzHandlerListener(DefaultHealthzHandlerAddress, ServiceName)
case logLevelRecord:
level = log.LevelRecord err := server.RunServerSocketAndWebServer(flag.Arg(0), *flagAddress, *flagWebserverAddress, *flagWebserverPath, *flagVarDir)
case logLevelWarning:
level = log.LevelWarning
case logLevelNotice:
level = log.LevelNotice
case logLevelDebug:
level = log.LevelDebug
}
log.SelectedLevel = level
err := server.RunServerSocketAndWebServer(flag.Arg(0), *address, *webserverAddress, *webserverPath, *varDir)
if err != nil { if err != nil {
fmt.Println("exiting with error", err) fmt.Println("exiting with error", err)
os.Exit(1) os.Exit(1)

BIN
contentserver.graffle Normal file
View File

Binary file not shown.

14
go.mod Normal file
View File

@ -0,0 +1,14 @@
module github.com/foomo/contentserver
require (
github.com/apex/log v1.1.0
github.com/davecgh/go-spew v1.1.1
github.com/json-iterator/go v1.1.6
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
github.com/modern-go/reflect2 v1.0.1 // indirect
github.com/pkg/errors v0.8.1 // indirect
github.com/prometheus/client_golang v0.9.2
go.uber.org/atomic v1.4.0 // indirect
go.uber.org/multierr v1.1.0 // indirect
go.uber.org/zap v1.10.0
)

34
go.sum Normal file
View File

@ -0,0 +1,34 @@
github.com/apex/log v1.1.0 h1:J5rld6WVFi6NxA6m8GJ1LJqu3+GiTFIt3mYv27gdQWI=
github.com/apex/log v1.1.0/go.mod h1:yA770aXIDQrhVOIGurT/pVdfCpSq1GQV/auzMN5fzvY=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs=
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740=
github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8=
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jOqq0gIVUe6Yk0/QMZ640k6NvkxcBf+8=
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE=
github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU=
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI=
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM=
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=

327
graphics/Horizontal Update.svg Normal file
View File

@ -0,0 +1,327 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
<svg xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns="http://www.w3.org/2000/svg" xmlns:xl="http://www.w3.org/1999/xlink" version="1.1" viewBox="133 101 1838 2500" width="1838" height="2500">
<defs>
<font-face font-family="Helvetica Neue" font-size="16" panose-1="2 0 5 3 0 0 0 2 0 4" units-per-em="1000" underline-position="-100" underline-thickness="50" slope="0" x-height="517" cap-height="714" ascent="951.9958" descent="-212.99744" font-weight="400">
<font-face-src>
<font-face-name name="HelveticaNeue"/>
</font-face-src>
</font-face>
<marker orient="auto" overflow="visible" markerUnits="strokeWidth" id="FilledArrow_Marker" stroke-linejoin="miter" stroke-miterlimit="10" viewBox="-1 -3 7 6" markerWidth="7" markerHeight="6" color="black">
<g>
<path d="M 4.8 0 L 0 -1.8 L 0 1.8 Z" fill="currentColor" stroke="currentColor" stroke-width="1"/>
</g>
</marker>
<font-face font-family="Futura" font-size="16" panose-1="2 11 6 2 2 2 4 2 3 3" units-per-em="1000" underline-position="-97.65625" underline-thickness="78.125" slope="0" x-height="482.4219" cap-height="761.2305" ascent="1038.5742" descent="-259.76562" font-weight="500">
<font-face-src>
<font-face-name name="Futura-Medium"/>
</font-face-src>
</font-face>
<font-face font-family="Helvetica Neue" font-size="16" panose-1="2 0 8 3 0 0 0 9 0 4" units-per-em="1000" underline-position="-100" underline-thickness="50" slope="0" x-height="517" cap-height="714" ascent="975.0061" descent="-216.99524" font-weight="700">
<font-face-src>
<font-face-name name="HelveticaNeue-Bold"/>
</font-face-src>
</font-face>
<marker orient="auto" overflow="visible" markerUnits="strokeWidth" id="FilledArrow_Marker_2" stroke-linejoin="miter" stroke-miterlimit="10" viewBox="-1 -3 7 6" markerWidth="7" markerHeight="6" color="#ff2600">
<g>
<path d="M 4.8 0 L 0 -1.8 L 0 1.8 Z" fill="currentColor" stroke="currentColor" stroke-width="1"/>
</g>
</marker>
<marker orient="auto" overflow="visible" markerUnits="strokeWidth" id="FilledArrow_Marker_3" stroke-linejoin="miter" stroke-miterlimit="10" viewBox="-1 -3 7 6" markerWidth="7" markerHeight="6" color="#ff2600">
<g>
<path d="M 4.8 0 L 0 -1.8 L 0 1.8 Z" fill="currentColor" stroke="currentColor" stroke-width="1"/>
</g>
</marker>
<font-face font-family="Futura" font-size="80" panose-1="2 11 6 2 2 2 4 2 3 3" units-per-em="1000" underline-position="-97.65625" underline-thickness="78.125" slope="0" x-height="482.4219" cap-height="761.2305" ascent="1038.5742" descent="-259.76562" font-weight="500">
<font-face-src>
<font-face-name name="Futura-Medium"/>
</font-face-src>
</font-face>
</defs>
<metadata> Produced by OmniGraffle 7.10.2
<dc:date>2019-05-29 10:21:17 +0000</dc:date>
</metadata>
<g id="Horizontal_Update" stroke-opacity="1" fill="none" stroke="none" stroke-dasharray="none" fill-opacity="1">
<title>Horizontal Update</title>
<rect fill="white" x="133" y="101" width="1838" height="2500"/>
<g id="Horizontal_Update: Layer 1">
<title>Layer 1</title>
<g id="Graphic_4">
<rect x="526" y="282" width="539" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(531 303.5)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="209.828" y="15">*Repo.Update()</tspan>
</text>
</g>
<g id="Graphic_5">
<rect x="526" y="425" width="539" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(531 446.5)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="200.644" y="15">*Repo.tryUpdate()</tspan>
</text>
</g>
<g id="Graphic_7">
<rect x="134" y="589" width="298" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(139 610.5)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="23.344" y="15">if updateErr != errUpdateRejected</tspan>
</text>
</g>
<g id="Line_8">
<line x1="693.9375" y1="489" x2="396.84877" y2="584.0684" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Graphic_12">
<rect x="134" y="770" width="298" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(139 791.5)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="43.848" y="15">*Repo.tryToRestoreCurrent()</tspan>
</text>
</g>
<g id="Line_13">
<line x1="283" y1="653" x2="283" y2="756.1" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Line_15">
<line x1="795.5" y1="346" x2="795.5" y2="411.1" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Graphic_16">
<rect x="1137" y="589" width="473" height="63" fill="white"/>
<rect x="1137" y="589" width="473" height="63" stroke="#ff2600" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(1142 610.5)" fill="#ff2600">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="#ff2600" x="162.756" y="15">*Repo.history.lock()</tspan>
</text>
</g>
<g id="Graphic_18">
<rect x="828" y="589" width="222" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(833 610.5)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="15.032" y="15">return errUpdateRejected</tspan>
</text>
</g>
<g id="Line_20">
<line x1="823.9375" y1="489" x2="902.0678" y2="578.29176" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Line_21">
<line x1="910.0427" y1="489" x2="1246.5472" y2="584.4788" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Graphic_22">
<rect x="824.0671" y="519.599" width="81.28906" height="32" fill="white"/>
<text transform="translate(829.0671 524.599)" fill="black">
<tspan font-family="Futura" font-size="16" font-weight="500" fill="black" x="0" y="17">queue full</tspan>
</text>
</g>
<g id="Graphic_23">
<rect x="1033.0312" y="521.1068" width="93.11719" height="32" fill="white"/>
<text transform="translate(1038.0312 526.1068)" fill="black">
<tspan font-family="Futura" font-size="16" font-weight="500" fill="black" x="2.46875" y="17">queue free </tspan>
</text>
</g>
<g id="Graphic_24">
<rect x="134" y="966" width="298" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(139 987.5)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="59.832" y="15">return updateResponse</tspan>
</text>
</g>
<g id="Line_25">
<line x1="283" y1="834" x2="283" y2="952.1" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Graphic_28">
<rect x="982.5" y="924" width="782" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(987.5 945.5)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="700" fill="black" x="299.496" y="16">*Repo.updateRoutine()</tspan>
</text>
</g>
<g id="Graphic_29">
<rect x="982.5" y="987" width="782" height="1072" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Graphic_31">
<rect x="1013.5" y="1023" width="359.1709" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(1018.5 1044.5)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="13.305457" y="15">resChan &lt;- *Repo.updateInProgressChannel:</tspan>
</text>
</g>
<g id="Graphic_32">
<rect x="1062.5" y="1110" width="359.1709" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(1067.5 1131.5)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="121.24146" y="15">*Repo.update()</tspan>
</text>
</g>
<g id="Graphic_33">
<rect x="1124.9145" y="1197" width="359.1709" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(1129.9145 1218.5)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="134.88146" y="15">*Repo.get()</tspan>
</text>
</g>
<g id="Graphic_34">
<rect x="1195.5" y="1284" width="359.1709" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(1200.5 1305.5)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="67.75346" y="15">*Repo.loadNodesFromJSON()</tspan>
</text>
</g>
<g id="Graphic_35">
<rect x="1259.5" y="1371" width="359.1709" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(1264.5 1392.5)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="107.47346" y="15">*Repo.loadNodes()</tspan>
</text>
</g>
<g id="Line_36">
<line x1="1373.5" y1="826" x2="1373.5" y2="910.1" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-dasharray="8.0,8.0" stroke-width="2"/>
</g>
<g id="Graphic_37">
<rect x="1302.5" y="1458" width="409.1709" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(1307.5 1461.828)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="0" y="15">for dimension, newNode := range nodes {</tspan>
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="0" y="33.448"> *Repo.updateDimension(dimension, newNode)</tspan>
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="0" y="51.895996">}</tspan>
</text>
</g>
<g id="Graphic_38">
<rect x="1352.5" y="1549" width="359.1709" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(1357.5 1570.5)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="700" fill="black" x="47.369457" y="16">*Repo.dimensionUpdateRoutine()</tspan>
</text>
</g>
<g id="Graphic_39">
<rect x="1352.5" y="1612" width="359.1709" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(1357.5 1633.5)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="79.44946" y="15">*Repo._updateDimension()</tspan>
</text>
</g>
<g id="Graphic_40">
<rect x="1352.5" y="1675" width="359.1709" height="343" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Graphic_41">
<rect x="1373.5" y="1689" width="298" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(1378.5 1710.5)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="60.152" y="15">newNode.WireParents()</tspan>
</text>
</g>
<g id="Graphic_42">
<rect x="1373.5" y="1772.5" width="298" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(1378.5 1794)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="90.072" y="15">buildDirectory()</tspan>
</text>
</g>
<g id="Graphic_43">
<rect x="1373.5" y="1856" width="298" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(1378.5 1877.5)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="99.872" y="15">wireAliases()</tspan>
</text>
</g>
<g id="Graphic_44">
<rect x="1373.5" y="1939.5" width="298" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(1378.5 1961)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="20.8" y="15">dimensionUpdateDoneChan &lt;- err</tspan>
</text>
</g>
<g id="Graphic_45">
<rect x="982.5" y="2186" width="359.1709" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(987.5 2207.5)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="71.76146" y="15">*Repo.history.add(jsonBytes)</tspan>
</text>
</g>
<g id="Line_46">
<line x1="1210.0304" y1="2060" x2="1175.7356" y2="2172.6591" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-dasharray="8.0,8.0" stroke-width="2"/>
</g>
<g id="Graphic_48">
<rect x="982.5" y="2537" width="359.1709" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(987.5 2558.5)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="62.79346" y="15">resultChan &lt;- updateResponse</tspan>
</text>
</g>
<g id="Line_47">
<line x1="1162.0855" y1="2488" x2="1162.0855" y2="2523.1" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Line_49">
<line x1="1265.4012" y1="1174" x2="1273.6646" y2="1185.5183" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Line_50">
<line x1="1330.8681" y1="1261" x2="1340.5898" y2="1272.9824" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Line_51">
<line x1="1398.9935" y1="1348" x2="1407.5333" y2="1359.6088" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Line_52">
<line x1="1464.4878" y1="1435" x2="1473.739" y2="1446.8363" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Line_53">
<line x1="1522.5" y1="1753" x2="1522.5" y2="1758.6" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Line_54">
<line x1="1522.5" y1="1836.5" x2="1522.5" y2="1842.1" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Line_55">
<line x1="1522.5" y1="1920" x2="1522.5" y2="1925.6" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Graphic_57">
<rect x="485.6393" y="523.2829" width="102.32812" height="32" fill="white"/>
<text transform="translate(490.6393 528.2829)" fill="black">
<tspan font-family="Futura" font-size="16" font-weight="500" fill="black" x="0" y="17">update error</tspan>
</text>
</g>
<g id="Graphic_60">
<rect x="519" y="589" width="222" height="63" fill="white"/>
<rect x="519" y="589" width="222" height="63" stroke="#ff2600" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(524 610.5)" fill="#ff2600">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="#ff2600" x="15.032" y="15">return errUpdateRejected</tspan>
</text>
</g>
<g id="Line_59">
<line x1="762.7027" y1="489" x2="671.9604" y2="578.91994" marker-end="url(#FilledArrow_Marker_2)" stroke="#ff2600" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Graphic_58">
<rect x="663.3299" y="519.599" width="104.69531" height="32" fill="white"/>
<text transform="translate(668.3299 524.599)" fill="#ff2600">
<tspan font-family="Futura" font-size="16" font-weight="500" fill="#ff2600" x="0" y="17">lockfile exists</tspan>
</text>
</g>
<g id="Graphic_61">
<rect x="1137" y="762" width="473" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(1142 783.5)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="8.42" y="15">*Repo.updateInProgressChan &lt;- make(chan updateResponse)</tspan>
</text>
</g>
<g id="Line_62">
<line x1="1373.5" y1="653" x2="1373.5" y2="748.1" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-dasharray="8.0,8.0" stroke-width="2"/>
</g>
<g id="Graphic_63">
<rect x="982.5" y="2305" width="359.1709" height="63" fill="white"/>
<rect x="982.5" y="2305" width="359.1709" height="63" stroke="#ff2600" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(987.5 2326.5)" fill="#ff2600">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="#ff2600" x="96.94546" y="15">*Repo.history.unlock()</tspan>
</text>
</g>
<g id="Line_64">
<line x1="1162.0855" y1="2250" x2="1162.0855" y2="2291.1" marker-end="url(#FilledArrow_Marker_2)" stroke="#ff2600" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Graphic_66">
<rect x="982.5" y="2424" width="359.1709" height="63" fill="white"/>
<rect x="982.5" y="2424" width="359.1709" height="63" stroke="#ff2600" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(987.5 2445.5)" fill="#ff2600">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="#ff2600" x="58.27346" y="15">*Repo.history.</tspan>
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="#ff2600" y="15">broadcastUpdate()</tspan>
</text>
</g>
<g id="Line_67">
<line x1="1162.0855" y1="2369" x2="1162.0855" y2="2410.1" marker-end="url(#FilledArrow_Marker_3)" stroke="#ff2600" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Graphic_68">
<text transform="translate(139 106.61719)" fill="black">
<tspan font-family="Futura" font-size="80" font-weight="500" fill="black" x="0" y="83">Contentserver Horizontal Scaling: Update Flow</tspan>
</text>
</g>
<g id="Graphic_69">
<path d="M 1341.6709 2305 L 1700.8418 2305 L 1700.8418 2368 L 1341.6709 2368 Z" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-dasharray="2.0,8.0" stroke-width="2"/>
<text transform="translate(1346.6709 2326.5)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="117.84146" y="15">Remove lockfile</tspan>
</text>
</g>
<g id="Graphic_70">
<path d="M 1341.6709 2424 L 1700.8418 2424 L 1700.8418 2487 L 1341.6709 2487 Z" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-dasharray="2.0,8.0" stroke-width="2"/>
<text transform="translate(1346.6709 2445.5)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="76.04946" y="15">Broadcast update via NATS</tspan>
</text>
</g>
<g id="Graphic_71">
<path d="M 1610 589 L 1969.171 589 L 1969.171 652 L 1610 652 Z" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-dasharray="2.0,8.0" stroke-width="2"/>
<text transform="translate(1615 610.5)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="123.62546" y="15">Create lockfile</tspan>
</text>
</g>
</g>
</g>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 22 KiB

391
graphics/Horizontal.svg Normal file
View File

@ -0,0 +1,391 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
<svg xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns="http://www.w3.org/2000/svg" xmlns:xl="http://www.w3.org/1999/xlink" version="1.1" viewBox="-263 -474 2433 1740" width="2433" height="1740">
<defs>
<font-face font-family="Helvetica Neue" font-size="25" panose-1="2 0 8 3 0 0 0 9 0 4" units-per-em="1000" underline-position="-100" underline-thickness="50" slope="0" x-height="517" cap-height="714" ascent="975.0061" descent="-216.99524" font-weight="700">
<font-face-src>
<font-face-name name="HelveticaNeue-Bold"/>
</font-face-src>
</font-face>
<font-face font-family="Helvetica Neue" font-size="16" panose-1="2 0 5 3 0 0 0 2 0 4" units-per-em="1000" underline-position="-100" underline-thickness="50" slope="0" x-height="517" cap-height="714" ascent="951.9958" descent="-212.99744" font-weight="400">
<font-face-src>
<font-face-name name="HelveticaNeue"/>
</font-face-src>
</font-face>
<font-face font-family="Futura" font-size="16" panose-1="2 11 6 2 2 2 4 2 3 3" units-per-em="1000" underline-position="-97.65625" underline-thickness="78.125" slope="0" x-height="482.4219" cap-height="761.2305" ascent="1038.5742" descent="-259.76562" font-weight="500">
<font-face-src>
<font-face-name name="Futura-Medium"/>
</font-face-src>
</font-face>
<marker orient="auto" overflow="visible" markerUnits="strokeWidth" id="FilledArrow_Marker" stroke-linejoin="miter" stroke-miterlimit="10" viewBox="-1 -3 7 6" markerWidth="7" markerHeight="6" color="black">
<g>
<path d="M 4.8 0 L 0 -1.8 L 0 1.8 Z" fill="currentColor" stroke="currentColor" stroke-width="1"/>
</g>
</marker>
</defs>
<metadata> Produced by OmniGraffle 7.10.2
<dc:date>2019-05-29 10:21:17 +0000</dc:date>
</metadata>
<g id="Horizontal" stroke-opacity="1" fill="none" stroke="none" stroke-dasharray="none" fill-opacity="1">
<title>Horizontal</title>
<rect fill="white" x="-263" y="-474" width="2433" height="1740"/>
<g id="Horizontal: Layer 1">
<title>Layer 1</title>
<g id="Graphic_35">
<rect x="-65" y="991" width="341" height="273" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Graphic_32">
<rect x="775" y="984" width="554" height="231" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Graphic_20">
<rect x="-65" y="195" width="956" height="619" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Graphic_36">
<rect x="-262" y="342" width="384" height="408" fill="white"/>
<rect x="-262" y="342" width="384" height="408" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Graphic_21">
<rect x="465" y="361.8998" width="384" height="398.1002" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Graphic_2">
<rect x="-65" y="121.5" width="956" height="73.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(-60 142.75)" fill="black">
<tspan font-family="Helvetica Neue" font-size="25" font-weight="700" fill="black" x="390.1" y="24">contentserver</tspan>
</text>
</g>
<g id="Graphic_3">
<rect x="140.5" y="425.4524" width="306" height="84" fill="white"/>
<rect x="140.5" y="425.4524" width="306" height="84" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(145.5 457.4524)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="107.864" y="15">Web server</tspan>
</text>
</g>
<g id="Graphic_4">
<rect x="140.5" y="567.5476" width="306" height="84" fill="white"/>
<rect x="140.5" y="567.5476" width="306" height="84" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(145.5 599.5476)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="81.04" y="15">TCP Socket server</tspan>
</text>
</g>
<g id="Graphic_5">
<rect x="886" y="-368.5" width="306" height="84" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(891 -336.5)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="87.992" y="15">CONTENT.JSON</tspan>
</text>
</g>
<g id="Graphic_6">
<rect x="847" y="-473" width="384" height="67" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(852 -455)" fill="black">
<tspan font-family="Helvetica Neue" font-size="25" font-weight="700" fill="black" x="21.475" y="24">Content Source Webservice</tspan>
</text>
</g>
<g id="Graphic_7">
<rect x="465" y="285" width="384" height="76.89978" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(470 307.9499)" fill="black">
<tspan font-family="Helvetica Neue" font-size="25" font-weight="700" fill="black" x="155.525" y="24">Repo</tspan>
</text>
</g>
<g id="Graphic_9">
<rect x="670" y="396.94934" width="156" height="87.88546" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(675 430.89207)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="34.48" y="15">RepoNode</tspan>
</text>
</g>
<g id="Graphic_11">
<rect x="495" y="396.94934" width="156" height="87.88546" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(500 430.89207)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="35.208" y="15">Dimension</tspan>
</text>
</g>
<g id="Graphic_22">
<rect x="670" y="509.94493" width="156" height="87.88546" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(675 543.88767)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="34.48" y="15">RepoNode</tspan>
</text>
</g>
<g id="Graphic_23">
<rect x="495" y="509.94493" width="156" height="87.88546" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(500 543.88767)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="35.208" y="15">Dimension</tspan>
</text>
</g>
<g id="Graphic_24">
<rect x="847" y="-406" width="384" height="162.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Graphic_25">
<rect x="-223" y="446.06663" width="306" height="54.961924" fill="white"/>
<rect x="-223" y="446.06663" width="306" height="54.961924" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(-218 463.5476)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="106.512" y="15">GetContent</tspan>
</text>
</g>
<g id="Graphic_26">
<rect x="-223" y="379" width="306" height="54.961924" fill="white"/>
<rect x="-223" y="379" width="306" height="54.961924" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(-218 396.48096)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="117.784" y="15">GetURIs</tspan>
</text>
</g>
<g id="Graphic_27">
<rect x="-223" y="513.13327" width="306" height="54.961924" fill="white"/>
<rect x="-223" y="513.13327" width="306" height="54.961924" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(-218 530.6142)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="111.704" y="15">GetNodes</tspan>
</text>
</g>
<g id="Graphic_28">
<rect x="-223" y="580.1999" width="306" height="54.961924" fill="white"/>
<rect x="-223" y="580.1999" width="306" height="54.961924" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(-218 597.68086)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="121.624" y="15">Update</tspan>
</text>
</g>
<g id="Graphic_29">
<rect x="-223" y="650.5381" width="306" height="54.961924" fill="white"/>
<rect x="-223" y="650.5381" width="306" height="54.961924" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(-218 668.019)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="116" y="15">GetRepo</tspan>
</text>
</g>
<g id="Graphic_30">
<rect x="831.2656" y="1071" width="441.46875" height="46" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(836.2656 1084)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="102.52637" y="15">contentserver-repo-current.json</tspan>
</text>
</g>
<g id="Graphic_31">
<rect x="775" y="917" width="554" height="67" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(780 935)" fill="black">
<tspan font-family="Helvetica Neue" font-size="25" font-weight="700" fill="black" x="115.975" y="24">Var Directory (aka History)</tspan>
</text>
</g>
<g id="Graphic_34">
<rect x="-65" y="924" width="341" height="67" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(-60 942)" fill="black">
<tspan font-family="Helvetica Neue" font-size="25" font-weight="700" fill="black" x="36.7875" y="24">Logfile (JSON or TXT)</tspan>
</text>
</g>
<g id="Graphic_37">
<rect x="-262" y="275" width="384" height="67" fill="white"/>
<rect x="-262" y="275" width="384" height="67" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(-257 293)" fill="black">
<tspan font-family="Helvetica Neue" font-size="25" font-weight="700" fill="black" x="166.4125" y="24">API</tspan>
</text>
</g>
<g id="Graphic_39">
<rect x="670" y="623.10984" width="156" height="87.88546" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(675 657.0526)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="65" y="15">…</tspan>
</text>
</g>
<g id="Graphic_40">
<rect x="495" y="623.10984" width="156" height="87.88546" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(500 657.0526)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="65" y="15">…</tspan>
</text>
</g>
<g id="Graphic_43">
<rect x="831.2656" y="1011.5" width="441.46875" height="46" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(836.2656 1024.5)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="97.70237" y="15">contentserver-repo-[Timestamp].json</tspan>
</text>
</g>
<g id="Graphic_45">
<rect x="-42" y="1011.5" width="301" height="46" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(-37 1023.5)" fill="black">
<tspan font-family="Futura" font-size="16" font-weight="500" fill="black" x="111.19531" y="17">Log entry</tspan>
</text>
</g>
<g id="Graphic_44">
<rect x="-42" y="1071" width="301" height="46" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(-37 1083)" fill="black">
<tspan font-family="Futura" font-size="16" font-weight="500" fill="black" x="111.19531" y="17">Log entry</tspan>
</text>
</g>
<g id="Graphic_46">
<rect x="-42" y="1130.5" width="301" height="46" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(-37 1142.5)" fill="black">
<tspan font-family="Futura" font-size="16" font-weight="500" fill="black" x="111.19531" y="17">Log entry</tspan>
</text>
</g>
<g id="Graphic_47">
<rect x="-42" y="1190" width="301" height="46" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(-37 1202)" fill="black">
<tspan font-family="Futura" font-size="16" font-weight="500" fill="black" x="111.19531" y="17">Log entry</tspan>
</text>
</g>
<g id="Line_48">
<line x1="461.9265" y1="120.5" x2="922.1852" y2="-234.61976" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Line_56">
<line x1="202.23013" y1="815" x2="136.16397" y2="912.3267" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Line_57">
<line x1="857.8643" y1="815" x2="991.9924" y2="908.6168" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Graphic_59">
<rect x="1213" y="187" width="956" height="619" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Graphic_60">
<rect x="1016" y="334" width="384" height="408" fill="white"/>
<rect x="1016" y="334" width="384" height="408" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Graphic_61">
<rect x="1743" y="353.8998" width="384" height="398.1002" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Graphic_62">
<rect x="1213" y="113.5" width="956" height="73.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(1218 134.75)" fill="black">
<tspan font-family="Helvetica Neue" font-size="25" font-weight="700" fill="black" x="390.1" y="24">contentserver</tspan>
</text>
</g>
<g id="Graphic_63">
<rect x="1418.5" y="417.4524" width="306" height="84" fill="white"/>
<rect x="1418.5" y="417.4524" width="306" height="84" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(1423.5 449.4524)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="107.864" y="15">Web server</tspan>
</text>
</g>
<g id="Graphic_64">
<rect x="1418.5" y="559.5476" width="306" height="84" fill="white"/>
<rect x="1418.5" y="559.5476" width="306" height="84" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(1423.5 591.5476)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="81.04" y="15">TCP Socket server</tspan>
</text>
</g>
<g id="Graphic_65">
<rect x="1743" y="277" width="384" height="76.89978" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(1748 299.9499)" fill="black">
<tspan font-family="Helvetica Neue" font-size="25" font-weight="700" fill="black" x="155.525" y="24">Repo</tspan>
</text>
</g>
<g id="Graphic_66">
<rect x="1948" y="388.94934" width="156" height="87.88546" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(1953 422.89207)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="34.48" y="15">RepoNode</tspan>
</text>
</g>
<g id="Graphic_67">
<rect x="1773" y="388.94934" width="156" height="87.88546" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(1778 422.89207)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="35.208" y="15">Dimension</tspan>
</text>
</g>
<g id="Graphic_68">
<rect x="1948" y="501.94493" width="156" height="87.88546" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(1953 535.88767)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="34.48" y="15">RepoNode</tspan>
</text>
</g>
<g id="Graphic_69">
<rect x="1773" y="501.94493" width="156" height="87.88546" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(1778 535.88767)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="35.208" y="15">Dimension</tspan>
</text>
</g>
<g id="Graphic_70">
<rect x="1055" y="438.06663" width="306" height="54.961924" fill="white"/>
<rect x="1055" y="438.06663" width="306" height="54.961924" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(1060 455.5476)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="106.512" y="15">GetContent</tspan>
</text>
</g>
<g id="Graphic_71">
<rect x="1055" y="371" width="306" height="54.961924" fill="white"/>
<rect x="1055" y="371" width="306" height="54.961924" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(1060 388.48096)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="117.784" y="15">GetURIs</tspan>
</text>
</g>
<g id="Graphic_72">
<rect x="1055" y="505.13327" width="306" height="54.961924" fill="white"/>
<rect x="1055" y="505.13327" width="306" height="54.961924" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(1060 522.6142)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="111.704" y="15">GetNodes</tspan>
</text>
</g>
<g id="Graphic_73">
<rect x="1055" y="572.1999" width="306" height="54.961924" fill="white"/>
<rect x="1055" y="572.1999" width="306" height="54.961924" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(1060 589.68086)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="121.624" y="15">Update</tspan>
</text>
</g>
<g id="Graphic_74">
<rect x="1055" y="642.5381" width="306" height="54.961924" fill="white"/>
<rect x="1055" y="642.5381" width="306" height="54.961924" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(1060 660.019)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="116" y="15">GetRepo</tspan>
</text>
</g>
<g id="Graphic_75">
<rect x="1016" y="267" width="384" height="67" fill="white"/>
<rect x="1016" y="267" width="384" height="67" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(1021 285)" fill="black">
<tspan font-family="Helvetica Neue" font-size="25" font-weight="700" fill="black" x="166.4125" y="24">API</tspan>
</text>
</g>
<g id="Graphic_76">
<rect x="1948" y="615.10984" width="156" height="87.88546" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(1953 649.0526)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="65" y="15">…</tspan>
</text>
</g>
<g id="Graphic_77">
<rect x="1773" y="615.10984" width="156" height="87.88546" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(1778 649.0526)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="65" y="15">…</tspan>
</text>
</g>
<g id="Line_78">
<line x1="1903.2618" y1="807" x2="1967.6353" y2="901.1666" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Line_79">
<line x1="1253.9747" y1="807" x2="1111.0744" y2="908.5285" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Line_84">
<line x1="1639.1832" y1="112.5" x2="1162.3254" y2="-234.90403" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Graphic_85">
<rect x="1828" y="979.816" width="341" height="273" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Graphic_86">
<rect x="1828" y="912.816" width="341" height="67" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(1833 930.816)" fill="black">
<tspan font-family="Helvetica Neue" font-size="25" font-weight="700" fill="black" x="36.7875" y="24">Logfile (JSON or TXT)</tspan>
</text>
</g>
<g id="Graphic_87">
<rect x="1851" y="1000.316" width="301" height="46" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(1856 1012.316)" fill="black">
<tspan font-family="Futura" font-size="16" font-weight="500" fill="black" x="111.19531" y="17">Log entry</tspan>
</text>
</g>
<g id="Graphic_88">
<rect x="1851" y="1059.816" width="301" height="46" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(1856 1071.816)" fill="black">
<tspan font-family="Futura" font-size="16" font-weight="500" fill="black" x="111.19531" y="17">Log entry</tspan>
</text>
</g>
<g id="Graphic_89">
<rect x="1851" y="1119.316" width="301" height="46" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(1856 1131.316)" fill="black">
<tspan font-family="Futura" font-size="16" font-weight="500" fill="black" x="111.19531" y="17">Log entry</tspan>
</text>
</g>
<g id="Graphic_90">
<rect x="1851" y="1178.816" width="301" height="46" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(1856 1190.816)" fill="black">
<tspan font-family="Futura" font-size="16" font-weight="500" fill="black" x="111.19531" y="17">Log entry</tspan>
</text>
</g>
<g id="Graphic_91">
<rect x="831.2656" y="1137" width="441.46875" height="46" fill="#ccc"/>
<rect x="831.2656" y="1137" width="441.46875" height="46" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(836.2656 1150)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="135.42237" y="15">updateInProgress.lock</tspan>
</text>
</g>
</g>
</g>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 25 KiB

250
graphics/Overview.svg Normal file
View File

@ -0,0 +1,250 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
<svg xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns="http://www.w3.org/2000/svg" xmlns:xl="http://www.w3.org/1999/xlink" version="1.1" viewBox="-549 -177 1441 1431" width="1441" height="1431">
<defs>
<font-face font-family="Helvetica Neue" font-size="16" panose-1="2 0 8 3 0 0 0 9 0 4" units-per-em="1000" underline-position="-100" underline-thickness="50" slope="0" x-height="517" cap-height="714" ascent="975.0061" descent="-216.99524" font-weight="700">
<font-face-src>
<font-face-name name="HelveticaNeue-Bold"/>
</font-face-src>
</font-face>
<font-face font-family="Helvetica Neue" font-size="16" panose-1="2 0 5 3 0 0 0 2 0 4" units-per-em="1000" underline-position="-100" underline-thickness="50" slope="0" x-height="517" cap-height="714" ascent="951.9958" descent="-212.99744" font-weight="400">
<font-face-src>
<font-face-name name="HelveticaNeue"/>
</font-face-src>
</font-face>
<font-face font-family="Futura" font-size="16" panose-1="2 11 6 2 2 2 4 2 3 3" units-per-em="1000" underline-position="-97.65625" underline-thickness="78.125" slope="0" x-height="482.4219" cap-height="761.2305" ascent="1038.5742" descent="-259.76562" font-weight="500">
<font-face-src>
<font-face-name name="Futura-Medium"/>
</font-face-src>
</font-face>
<marker orient="auto" overflow="visible" markerUnits="strokeWidth" id="FilledArrow_Marker" stroke-linejoin="miter" stroke-miterlimit="10" viewBox="-1 -3 7 6" markerWidth="7" markerHeight="6" color="black">
<g>
<path d="M 4.8 0 L 0 -1.8 L 0 1.8 Z" fill="currentColor" stroke="currentColor" stroke-width="1"/>
</g>
</marker>
</defs>
<metadata> Produced by OmniGraffle 7.10.2
<dc:date>2019-05-29 10:21:17 +0000</dc:date>
</metadata>
<g id="Overview" stroke-opacity="1" fill="none" stroke="none" stroke-dasharray="none" fill-opacity="1">
<title>Overview</title>
<rect fill="white" x="-549" y="-177" width="1441" height="1431"/>
<g id="Overview: Layer 1">
<title>Layer 1</title>
<g id="Graphic_35">
<rect x="546" y="979.7753" width="341" height="273" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Graphic_32">
<rect x="-69" y="979.816" width="554" height="185.45933" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Graphic_20">
<rect x="-65" y="195" width="956" height="619" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Graphic_36">
<rect x="-262" y="342" width="384" height="408" fill="white"/>
<rect x="-262" y="342" width="384" height="408" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Graphic_21">
<rect x="465" y="361.8998" width="384" height="398.1002" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Graphic_2">
<rect x="-65" y="121.5" width="956" height="73.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(-60 148.25)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="700" fill="black" x="419.944" y="16">contentserver</tspan>
</text>
</g>
<g id="Graphic_3">
<rect x="140.5" y="425.4524" width="306" height="84" fill="white"/>
<rect x="140.5" y="425.4524" width="306" height="84" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(145.5 457.4524)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="107.864" y="15">Web server</tspan>
</text>
</g>
<g id="Graphic_4">
<rect x="140.5" y="567.5476" width="306" height="84" fill="white"/>
<rect x="140.5" y="567.5476" width="306" height="84" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(145.5 599.5476)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="81.04" y="15">TCP Socket server</tspan>
</text>
</g>
<g id="Graphic_5">
<rect x="260" y="-71.5" width="306" height="84" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(265 -39.5)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="87.992" y="15">CONTENT.JSON</tspan>
</text>
</g>
<g id="Graphic_6">
<rect x="221" y="-176" width="384" height="67" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(226 -152.5)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="700" fill="black" x="81.064" y="16">Content Source Webservice</tspan>
</text>
</g>
<g id="Graphic_7">
<rect x="465" y="285" width="384" height="76.89978" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(470 313.4499)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="700" fill="black" x="166.856" y="16">Repo</tspan>
</text>
</g>
<g id="Graphic_9">
<rect x="670" y="396.94934" width="156" height="87.88546" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(675 430.89207)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="34.48" y="15">RepoNode</tspan>
</text>
</g>
<g id="Graphic_11">
<rect x="495" y="396.94934" width="156" height="87.88546" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(500 430.89207)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="35.208" y="15">Dimension</tspan>
</text>
</g>
<g id="Graphic_22">
<rect x="670" y="509.94493" width="156" height="87.88546" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(675 543.88767)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="34.48" y="15">RepoNode</tspan>
</text>
</g>
<g id="Graphic_23">
<rect x="495" y="509.94493" width="156" height="87.88546" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(500 543.88767)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="35.208" y="15">Dimension</tspan>
</text>
</g>
<g id="Graphic_24">
<rect x="221" y="-109" width="384" height="162.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Graphic_25">
<rect x="-223" y="446.06663" width="306" height="54.961924" fill="white"/>
<rect x="-223" y="446.06663" width="306" height="54.961924" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(-218 463.5476)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="106.512" y="15">GetContent</tspan>
</text>
</g>
<g id="Graphic_26">
<rect x="-223" y="379" width="306" height="54.961924" fill="white"/>
<rect x="-223" y="379" width="306" height="54.961924" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(-218 396.48096)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="117.784" y="15">GetURIs</tspan>
</text>
</g>
<g id="Graphic_27">
<rect x="-223" y="513.13327" width="306" height="54.961924" fill="white"/>
<rect x="-223" y="513.13327" width="306" height="54.961924" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(-218 530.6142)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="111.704" y="15">GetNodes</tspan>
</text>
</g>
<g id="Graphic_28">
<rect x="-223" y="580.1999" width="306" height="54.961924" fill="white"/>
<rect x="-223" y="580.1999" width="306" height="54.961924" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(-218 597.68086)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="121.624" y="15">Update</tspan>
</text>
</g>
<g id="Graphic_29">
<rect x="-223" y="650.5381" width="306" height="54.961924" fill="white"/>
<rect x="-223" y="650.5381" width="306" height="54.961924" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(-218 668.019)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="116" y="15">GetRepo</tspan>
</text>
</g>
<g id="Graphic_30">
<rect x="-12.734375" y="1088.724" width="441.46875" height="46" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(-7.734375 1101.724)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="102.52637" y="15">contentserver-repo-current.json</tspan>
</text>
</g>
<g id="Graphic_31">
<rect x="-69" y="912.816" width="554" height="67" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(-64 936.316)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="700" fill="black" x="172.144" y="16">Var Directory (aka History)</tspan>
</text>
</g>
<g id="Graphic_34">
<rect x="546" y="912.7753" width="341" height="67" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(551 936.2753)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="700" fill="black" x="83.124" y="16">Logfile (JSON or TXT)</tspan>
</text>
</g>
<g id="Graphic_37">
<rect x="-262" y="275" width="384" height="67" fill="white"/>
<rect x="-262" y="275" width="384" height="67" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(-257 298.5)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="700" fill="black" x="173.824" y="16">API</tspan>
</text>
</g>
<g id="Graphic_39">
<rect x="670" y="623.10984" width="156" height="87.88546" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(675 657.0526)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="65" y="15">…</tspan>
</text>
</g>
<g id="Graphic_40">
<rect x="495" y="623.10984" width="156" height="87.88546" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(500 657.0526)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="65" y="15">…</tspan>
</text>
</g>
<g id="Graphic_41">
<rect x="-12.734375" y="1010.3673" width="441.46875" height="46" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(-7.734375 1023.3673)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="97.70237" y="15">contentserver-repo-[Timestamp].json</tspan>
</text>
</g>
<g id="Graphic_45">
<rect x="569" y="1000.2753" width="301" height="46" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(574 1012.2753)" fill="black">
<tspan font-family="Futura" font-size="16" font-weight="500" fill="black" x="111.19531" y="17">Log entry</tspan>
</text>
</g>
<g id="Graphic_44">
<rect x="569" y="1059.7753" width="301" height="46" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(574 1071.7753)" fill="black">
<tspan font-family="Futura" font-size="16" font-weight="500" fill="black" x="111.19531" y="17">Log entry</tspan>
</text>
</g>
<g id="Graphic_46">
<rect x="569" y="1119.2753" width="301" height="46" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(574 1131.2753)" fill="black">
<tspan font-family="Futura" font-size="16" font-weight="500" fill="black" x="111.19531" y="17">Log entry</tspan>
</text>
</g>
<g id="Graphic_47">
<rect x="569" y="1178.7753" width="301" height="46" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(574 1190.7753)" fill="black">
<tspan font-family="Futura" font-size="16" font-weight="500" fill="black" x="111.19531" y="17">Log entry</tspan>
</text>
</g>
<g id="Line_48">
<line x1="413" y1="120.5" x2="413" y2="67.4" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Line_49">
<line x1="-469.47585" y1="537.66146" x2="-237.4888" y2="419.19996" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Line_50">
<line x1="-469.47585" y1="537.66146" x2="-238.52136" y2="480.43586" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Line_51">
<line x1="-469.47585" y1="537.66146" x2="-236.89965" y2="539.3806" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Line_53">
<line x1="-464.51305" y1="540.5356" x2="-238.39468" y2="605.75816" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Line_52">
<line x1="-469.47585" y1="537.66146" x2="-237.1631" y2="672.2017" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Line_56">
<line x1="626.31374" y1="815" x2="685.49386" y2="901.1427" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Line_57">
<line x1="268.92984" y1="815" x2="229.43732" y2="900.1143" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Graphic_58">
<rect x="-548" y="500.94756" width="119.31591" height="84" fill="white"/>
<rect x="-548" y="500.94756" width="119.31591" height="84" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(-543 532.94756)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="34.065957" y="15">Client</tspan>
</text>
</g>
</g>
</g>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 16 KiB

260
graphics/Update-Flow.svg Normal file
View File

@ -0,0 +1,260 @@
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
<svg xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns="http://www.w3.org/2000/svg" xmlns:xl="http://www.w3.org/1999/xlink" version="1.1" viewBox="314 98 1205 2185" width="1205" height="2185">
<defs>
<font-face font-family="Helvetica Neue" font-size="16" panose-1="2 0 5 3 0 0 0 2 0 4" units-per-em="1000" underline-position="-100" underline-thickness="50" slope="0" x-height="517" cap-height="714" ascent="951.9958" descent="-212.99744" font-weight="400">
<font-face-src>
<font-face-name name="HelveticaNeue"/>
</font-face-src>
</font-face>
<marker orient="auto" overflow="visible" markerUnits="strokeWidth" id="FilledArrow_Marker" stroke-linejoin="miter" stroke-miterlimit="10" viewBox="-1 -3 7 6" markerWidth="7" markerHeight="6" color="black">
<g>
<path d="M 4.8 0 L 0 -1.8 L 0 1.8 Z" fill="currentColor" stroke="currentColor" stroke-width="1"/>
</g>
</marker>
<font-face font-family="Futura" font-size="16" panose-1="2 11 6 2 2 2 4 2 3 3" units-per-em="1000" underline-position="-97.65625" underline-thickness="78.125" slope="0" x-height="482.4219" cap-height="761.2305" ascent="1038.5742" descent="-259.76562" font-weight="500">
<font-face-src>
<font-face-name name="Futura-Medium"/>
</font-face-src>
</font-face>
<font-face font-family="Helvetica Neue" font-size="16" panose-1="2 0 8 3 0 0 0 9 0 4" units-per-em="1000" underline-position="-100" underline-thickness="50" slope="0" x-height="517" cap-height="714" ascent="975.0061" descent="-216.99524" font-weight="700">
<font-face-src>
<font-face-name name="HelveticaNeue-Bold"/>
</font-face-src>
</font-face>
<font-face font-family="Futura" font-size="80" panose-1="2 11 6 2 2 2 4 2 3 3" units-per-em="1000" underline-position="-97.65625" underline-thickness="78.125" slope="0" x-height="482.4219" cap-height="761.2305" ascent="1038.5742" descent="-259.76562" font-weight="500">
<font-face-src>
<font-face-name name="Futura-Medium"/>
</font-face-src>
</font-face>
</defs>
<metadata> Produced by OmniGraffle 7.10.2
<dc:date>2019-05-29 10:21:17 +0000</dc:date>
</metadata>
<g id="Update-Flow" stroke-opacity="1" fill="none" stroke="none" stroke-dasharray="none" fill-opacity="1">
<title>Update-Flow</title>
<rect fill="white" x="314" y="98" width="1205" height="2185"/>
<g id="Update-Flow: Layer 1">
<title>Layer 1</title>
<g id="Graphic_4">
<rect x="526" y="282" width="298" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(531 303.5)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="89.328" y="15">*Repo.Update()</tspan>
</text>
</g>
<g id="Graphic_5">
<rect x="526" y="425" width="298" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(531 446.5)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="80.144" y="15">*Repo.tryUpdate()</tspan>
</text>
</g>
<g id="Graphic_7">
<rect x="315" y="587" width="298" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(320 608.5)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="23.344" y="15">if updateErr != errUpdateRejected</tspan>
</text>
</g>
<g id="Line_8">
<line x1="632.66975" y1="489" x2="516.5623" y2="578.1441" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Graphic_10">
<rect x="545.27846" y="518.599" width="56" height="32" fill="white"/>
<text transform="translate(550.27846 523.599)" fill="black">
<tspan font-family="Futura" font-size="16" font-weight="500" fill="black" x="0" y="17">failure</tspan>
</text>
</g>
<g id="Graphic_12">
<rect x="315" y="768" width="298" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(320 789.5)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="43.848" y="15">*Repo.tryToRestoreCurrent()</tspan>
</text>
</g>
<g id="Line_13">
<line x1="464" y1="651" x2="464" y2="754.1" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Line_15">
<line x1="675" y1="346" x2="675" y2="411.1" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Graphic_16">
<rect x="1045" y="768" width="473" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(1050 789.5)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="8.42" y="15">*Repo.updateInProgressChan &lt;- make(chan updateResponse)</tspan>
</text>
</g>
<g id="Graphic_18">
<rect x="736" y="768" width="222" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(741 789.5)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="15.032" y="15">return errUpdateRejected</tspan>
</text>
</g>
<g id="Graphic_19">
<rect x="522.1144" y="518.599" width="102.32812" height="32" fill="white"/>
<text transform="translate(527.1144 523.599)" fill="black">
<tspan font-family="Futura" font-size="16" font-weight="500" fill="black" x="0" y="17">update error</tspan>
</text>
</g>
<g id="Line_20">
<line x1="691.2974" y1="489" x2="824.9201" y2="755.4686" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Line_21">
<line x1="732.4672" y1="489" x2="1212.8041" y2="760.6497" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Graphic_22">
<rect x="716.2705" y="603.8538" width="81.28906" height="32" fill="white"/>
<text transform="translate(721.2705 608.8538)" fill="black">
<tspan font-family="Futura" font-size="16" font-weight="500" fill="black" x="0" y="17">queue full</tspan>
</text>
</g>
<g id="Graphic_23">
<rect x="924.7738" y="608.0878" width="93.11719" height="32" fill="white"/>
<text transform="translate(929.7738 613.0878)" fill="black">
<tspan font-family="Futura" font-size="16" font-weight="500" fill="black" x="2.46875" y="17">queue free </tspan>
</text>
</g>
<g id="Graphic_24">
<rect x="315" y="964" width="298" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(320 985.5)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="59.832" y="15">return updateResponse</tspan>
</text>
</g>
<g id="Line_25">
<line x1="464" y1="832" x2="464" y2="950.1" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Graphic_28">
<rect x="736" y="883" width="782" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(741 904.5)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="700" fill="black" x="299.496" y="16">*Repo.updateRoutine()</tspan>
</text>
</g>
<g id="Graphic_29">
<rect x="736" y="946" width="782" height="1072" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Graphic_31">
<rect x="767" y="982" width="359.1709" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(772 1003.5)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="13.305457" y="15">resChan &lt;- *Repo.updateInProgressChannel:</tspan>
</text>
</g>
<g id="Graphic_32">
<rect x="816" y="1069" width="359.1709" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(821 1090.5)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="121.24146" y="15">*Repo.update()</tspan>
</text>
</g>
<g id="Graphic_33">
<rect x="878.4145" y="1156" width="359.1709" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(883.4145 1177.5)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="134.88146" y="15">*Repo.get()</tspan>
</text>
</g>
<g id="Graphic_34">
<rect x="949" y="1243" width="359.1709" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(954 1264.5)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="67.75346" y="15">*Repo.loadNodesFromJSON()</tspan>
</text>
</g>
<g id="Graphic_35">
<rect x="1013" y="1330" width="359.1709" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(1018 1351.5)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="107.47346" y="15">*Repo.loadNodes()</tspan>
</text>
</g>
<g id="Line_36">
<line x1="1237.837" y1="832" x2="1181.0111" y2="874.2976" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-dasharray="8.0,8.0" stroke-width="2"/>
</g>
<g id="Graphic_37">
<rect x="1056" y="1417" width="409.1709" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(1061 1420.828)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="0" y="15">for dimension, newNode := range nodes {</tspan>
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="0" y="33.448"> *Repo.updateDimension(dimension, newNode)</tspan>
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="0" y="51.895996">}</tspan>
</text>
</g>
<g id="Graphic_38">
<rect x="1106" y="1508" width="359.1709" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(1111 1529.5)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="700" fill="black" x="47.369457" y="16">*Repo.dimensionUpdateRoutine()</tspan>
</text>
</g>
<g id="Graphic_39">
<rect x="1106" y="1571" width="359.1709" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(1111 1592.5)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="79.44946" y="15">*Repo._updateDimension()</tspan>
</text>
</g>
<g id="Graphic_40">
<rect x="1106" y="1634" width="359.1709" height="343" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Graphic_41">
<rect x="1127" y="1648" width="298" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(1132 1669.5)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="60.152" y="15">newNode.WireParents()</tspan>
</text>
</g>
<g id="Graphic_42">
<rect x="1127" y="1731.5" width="298" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(1132 1753)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="90.072" y="15">buildDirectory()</tspan>
</text>
</g>
<g id="Graphic_43">
<rect x="1127" y="1815" width="298" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(1132 1836.5)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="99.872" y="15">wireAliases()</tspan>
</text>
</g>
<g id="Graphic_44">
<rect x="1127" y="1898.5" width="298" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(1132 1920)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="20.8" y="15">dimensionUpdateDoneChan &lt;- err</tspan>
</text>
</g>
<g id="Graphic_45">
<rect x="736" y="2078" width="359.1709" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(741 2099.5)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="71.76146" y="15">*Repo.history.add(jsonBytes)</tspan>
</text>
</g>
<g id="Line_46">
<line x1="946.0763" y1="2019" x2="930.6539" y2="2064.7752" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-dasharray="8.0,8.0" stroke-width="2"/>
</g>
<g id="Graphic_48">
<rect x="766.5855" y="2219" width="298" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
<text transform="translate(771.5855 2240.5)" fill="black">
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="32.208" y="15">resultChan &lt;- updateResponse</tspan>
</text>
</g>
<g id="Line_47">
<line x1="915.5855" y1="2142" x2="915.5855" y2="2205.1" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Line_49">
<line x1="1018.9012" y1="1133" x2="1027.1646" y2="1144.5183" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Line_50">
<line x1="1084.3681" y1="1220" x2="1094.0898" y2="1231.9824" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Line_51">
<line x1="1152.4935" y1="1307" x2="1161.0333" y2="1318.6088" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Line_52">
<line x1="1217.9878" y1="1394" x2="1227.2391" y2="1405.8363" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Line_53">
<line x1="1276" y1="1712" x2="1276" y2="1717.6" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Line_54">
<line x1="1276" y1="1795.5" x2="1276" y2="1801.1" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Line_55">
<line x1="1276" y1="1879" x2="1276" y2="1884.6" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
</g>
<g id="Graphic_57">
<text transform="translate(411 103)" fill="black">
<tspan font-family="Futura" font-size="80" font-weight="500" fill="black" x="0" y="83">Contentserver</tspan>
<tspan font-family="Futura" font-size="80" font-weight="500" fill="black" y="83">:</tspan>
<tspan font-family="Futura" font-size="80" font-weight="500" fill="black" y="83"> Update Flow</tspan>
</text>
</g>
</g>
</g>
</svg>
Side by Side

After

Width:  |  Height:  |  Size: 17 KiB

79
log/log.go
View File

@ -1,79 +0,0 @@
package log
import (
"fmt"
"strings"
"time"
)
// Level logging level enum
type Level int
const (
// LevelError an error - as bad as it gets
LevelError Level = 0
// LevelRecord put this to the logs in any case
LevelRecord Level = 1
// LevelWarning not that bad
LevelWarning Level = 2
// LevelNotice almost on debug level
LevelNotice Level = 3
// LevelDebug we are debugging
LevelDebug Level = 4
)
// SelectedLevel selected log level
var SelectedLevel = LevelDebug
var prefices = map[Level]string{
LevelRecord: "record : ",
LevelError: "error : ",
LevelWarning: "warning : ",
LevelNotice: "notice : ",
LevelDebug: "debug : ",
}
func log(msg string, level Level) string {
if level <= SelectedLevel {
prefix := time.Now().Format(time.RFC3339Nano) + " " + prefices[level]
lines := strings.Split(msg, "\n")
for i := 0; i < len(lines); i++ {
fmt.Println(level, prefix+lines[i])
}
}
return msg
}
func logThings(msgs []interface{}, level Level) string {
r := ""
for _, msg := range msgs {
r += "\n" + fmt.Sprint(msg)
}
r = strings.Trim(r, "\n")
return log(r, level)
}
// Debug write debug messages to the log
func Debug(msgs ...interface{}) string {
return logThings(msgs, LevelDebug)
}
// Notice write notice messages to the log
func Notice(msgs ...interface{}) string {
return logThings(msgs, LevelNotice)
}
// Warning write warning messages to the log
func Warning(msgs ...interface{}) string {
return logThings(msgs, LevelWarning)
}
// Record write record messages to the log
func Record(msgs ...interface{}) string {
return logThings(msgs, LevelRecord)
}
// Error write error messages to the log
func Error(msgs ...interface{}) string {
return logThings(msgs, LevelError)
}

43
logger/log.go Normal file
View File

@ -0,0 +1,43 @@
package logger
import (
"log"
"os"
"go.uber.org/zap"
)
var (
// Log is the logger instance exposed by this package
// call Setup() prior to using it
// want JSON output? Set LOG_JSON env var to 1!
Log *zap.Logger
)
// SetupLogging configures the logger
func SetupLogging(debug bool, outputPath string) {
var (
zc zap.Config
err error
)
if debug {
zc = zap.NewDevelopmentConfig()
zc.OutputPaths = append(zc.OutputPaths, outputPath)
} else {
zc = zap.NewProductionConfig()
zc.OutputPaths = append(zc.OutputPaths, outputPath)
}
if os.Getenv("LOG_JSON") == "1" {
zc.Encoding = "json"
} else {
zc.Encoding = "console"
}
Log, err = zc.Build()
if err != nil {
log.Fatalf("can't initialize zap logger: %v", err)
}
}

21
metrics/prometheus.go Normal file
View File

@ -0,0 +1,21 @@
package metrics
import (
"net/http"
. "github.com/foomo/contentserver/logger"
"github.com/prometheus/client_golang/prometheus/promhttp"
"go.uber.org/zap"
)
const metricsRoute = "/metrics"
func RunPrometheusHandler(listener string) {
Log.Info("starting prometheus handler on",
zap.String("address", listener),
zap.String("route", metricsRoute),
)
Log.Error("prometheus listener failed",
zap.Error(http.ListenAndServe(listener, promhttp.Handler())),
)
}

14
prometheus/prometheus.yml Normal file
View File

@ -0,0 +1,14 @@
# reference: https://prometheus.io/docs/prometheus/latest/configuration/configuration/
global:
scrape_interval: 15s
scrape_timeout: 15s
#evaluation_interval: 15s
scrape_configs:
- job_name: contentserver
metrics_path: /metrics
scheme: http
static_configs:
- targets:
- 127.0.0.1:9111

73
repo/history.go
View File

@ -1,19 +1,28 @@
package repo package repo
import ( import (
"bytes"
"errors"
"flag"
"fmt"
"io"
"io/ioutil" "io/ioutil"
"os" "os"
"path" "path"
"sort" "sort"
"strings" "strings"
"time" "time"
"errors"
"fmt" . "github.com/foomo/contentserver/logger"
"go.uber.org/zap"
) )
const historyRepoJSONPrefix = "contentserver-repo-" const (
const historyRepoJSONSuffix = ".json" historyRepoJSONPrefix = "contentserver-repo-"
const maxHistoryVersions = 20 historyRepoJSONSuffix = ".json"
)
var flagMaxHistoryVersions = flag.Int("max-history", 2, "set the maximum number of content backup files")
type history struct { type history struct {
varDir string varDir string
@ -26,18 +35,34 @@ func newHistory(varDir string) *history {
} }
func (h *history) add(jsonBytes []byte) error { func (h *history) add(jsonBytes []byte) error {
// historic file name
filename := path.Join(h.varDir, historyRepoJSONPrefix+time.Now().Format(time.RFC3339Nano)+historyRepoJSONSuffix) var (
err := ioutil.WriteFile(filename, jsonBytes, 0644) // historiy file name
filename = path.Join(h.varDir, historyRepoJSONPrefix+time.Now().Format(time.RFC3339Nano)+historyRepoJSONSuffix)
err = ioutil.WriteFile(filename, jsonBytes, 0644)
)
if err != nil { if err != nil {
return err return err
} }
Log.Info("adding content backup", zap.String("file", filename))
// current filename // current filename
return ioutil.WriteFile(h.getCurrentFilename(), jsonBytes, 0644) err = ioutil.WriteFile(h.getCurrentFilename(), jsonBytes, 0644)
if err != nil {
return err
}
err = h.cleanup()
if err != nil {
Log.Error("an error occured while cleaning up my history", zap.Error(err))
return err
}
return nil
} }
func (h *history) getHistory() (files []string, err error) { func (h *history) getHistory() (files []string, err error) {
files = []string{}
fileInfos, err := ioutil.ReadDir(h.varDir) fileInfos, err := ioutil.ReadDir(h.varDir)
if err != nil { if err != nil {
return return
@ -56,14 +81,16 @@ func (h *history) getHistory() (files []string, err error) {
} }
func (h *history) cleanup() error { func (h *history) cleanup() error {
files, err := h.getFilesForCleanup(maxHistoryVersions) files, err := h.getFilesForCleanup(*flagMaxHistoryVersions)
if err != nil { if err != nil {
return err return err
} }
for _, f := range files { for _, f := range files {
Log.Info("removing outdated backup", zap.String("file", f))
err := os.Remove(f) err := os.Remove(f)
if err != nil { if err != nil {
return errors.New(fmt.Sprintf("could not remove file %s : %s", f, err.Error())) return fmt.Errorf("could not remove file %s : %s", f, err.Error())
} }
} }
@ -75,8 +102,16 @@ func (h *history) getFilesForCleanup(historyVersions int) (files []string, err e
if err != nil { if err != nil {
return nil, errors.New("could not generate file cleanup list: " + err.Error()) return nil, errors.New("could not generate file cleanup list: " + err.Error())
} }
if len(contentFiles) > historyVersions {
for i := historyVersions; i < len(contentFiles); i++ { // fmt.Println("contentFiles:")
// for _, f := range contentFiles {
// fmt.Println(f)
// }
// -1 to remove the current backup file from the number of items
// so that only files with a timestamp are compared
if len(contentFiles)-1 > historyVersions {
for i := historyVersions + 1; i < len(contentFiles); i++ {
// ignore current repository file to fall back on // ignore current repository file to fall back on
if contentFiles[i] == h.getCurrentFilename() { if contentFiles[i] == h.getCurrentFilename() {
continue continue
@ -91,6 +126,12 @@ func (h *history) getCurrentFilename() string {
return path.Join(h.varDir, historyRepoJSONPrefix+"current"+historyRepoJSONSuffix) return path.Join(h.varDir, historyRepoJSONPrefix+"current"+historyRepoJSONSuffix)
} }
func (h *history) getCurrent() (jsonBytes []byte, err error) { func (h *history) getCurrent(buf *bytes.Buffer) (err error) {
return ioutil.ReadFile(h.getCurrentFilename()) f, err := os.Open(h.getCurrentFilename())
if err != nil {
return err
}
defer f.Close()
_, err = io.Copy(buf, f)
return err
} }

39
repo/history_test.go
View File

@ -18,31 +18,45 @@ func testHistory() *history {
} }
func TestHistoryCurrent(t *testing.T) { func TestHistoryCurrent(t *testing.T) {
h := testHistory() var (
test := []byte("test") h = testHistory()
h.add(test) test = []byte("test")
current, err := h.getCurrent() b bytes.Buffer
)
err := h.add(test)
if err != nil {
t.Fatal("failed to add: ", err)
}
err = h.getCurrent(&b)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
if bytes.Compare(current, test) != 0 { if !bytes.Equal(b.Bytes(), test) {
t.Fatal(fmt.Sprintf("expected %q, got %q", string(test), string(current))) t.Fatal(fmt.Sprintf("expected %q, got %q", string(test), string(b.Bytes())))
} }
} }
func TestHistoryCleanup(t *testing.T) { func TestHistoryCleanup(t *testing.T) {
h := testHistory() h := testHistory()
for i := 0; i < 50; i++ { for i := 0; i < 50; i++ {
h.add([]byte(fmt.Sprint(i))) err := h.add([]byte(fmt.Sprint(i)))
if err != nil {
t.Fatal("failed to add: ", err)
}
time.Sleep(time.Millisecond * 5) time.Sleep(time.Millisecond * 5)
} }
h.cleanup() err := h.cleanup()
if err != nil {
t.Fatal("failed to run cleanup: ", err)
}
files, err := h.getHistory() files, err := h.getHistory()
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
if len(files) != maxHistoryVersions {
t.Fatal("history too long", len(files), "instead of", maxHistoryVersions) // -1 for ignoring the current content backup file
if len(files)-1 != *flagMaxHistoryVersions {
t.Fatal("history too long", len(files), "instead of", *flagMaxHistoryVersions)
} }
} }
@ -51,7 +65,6 @@ func TestHistoryOrder(t *testing.T) {
h.varDir = "testdata/order" h.varDir = "testdata/order"
files, err := h.getHistory() files, err := h.getHistory()
if err != nil { if err != nil {
t.Fatal("error not expected") t.Fatal("error not expected")
} }
@ -69,11 +82,9 @@ func TestGetFilesForCleanup(t *testing.T) {
if err != nil { if err != nil {
t.Fatal("error not expected") t.Fatal("error not expected")
} }
assertStringEqual(t, "testdata/order/contentserver-repo-2017-10-22.json", files[0]) assertStringEqual(t, "testdata/order/contentserver-repo-2017-10-21.json", files[0])
assertStringEqual(t, "testdata/order/contentserver-repo-2017-10-21.json", files[1])
} }
func assertStringEqual(t *testing.T, expected, actual string) { func assertStringEqual(t *testing.T, expected, actual string) {
if expected != actual { if expected != actual {
t.Errorf("expected string %s differs from the actual %s", expected, actual) t.Errorf("expected string %s differs from the actual %s", expected, actual)

194
repo/loader.go
View File

@ -1,50 +1,86 @@
package repo package repo
import ( import (
"encoding/json"
"errors" "errors"
"fmt" "fmt"
"io/ioutil" "io"
"net/http" "net/http"
"time" "time"
"github.com/foomo/contentserver/content" "github.com/foomo/contentserver/content"
"github.com/foomo/contentserver/log" . "github.com/foomo/contentserver/logger"
"github.com/foomo/contentserver/status"
jsoniter "github.com/json-iterator/go"
"go.uber.org/zap"
) )
var (
json = jsoniter.ConfigCompatibleWithStandardLibrary
errUpdateRejected = errors.New("update rejected: queue full")
)
type updateResponse struct {
repoRuntime int64
err error
}
func (repo *Repo) updateRoutine() { func (repo *Repo) updateRoutine() {
go func() { for {
for { select {
log.Debug("update routine is about to select") case resChan := <-repo.updateInProgressChannel:
select { Log.Info("waiting for update to complete", zap.String("chan", fmt.Sprintf("%p", resChan)))
case newDimension := <-repo.updateChannel: start := time.Now()
log.Debug("update routine received a new dimension: " + newDimension.Dimension)
err := repo._updateDimension(newDimension.Dimension, newDimension.Node) repoRuntime, errUpdate := repo.update()
log.Debug("update routine received result") if errUpdate != nil {
if err != nil { status.M.UpdatesFailedCounter.WithLabelValues(errUpdate.Error()).Inc()
log.Debug(" update routine error: " + err.Error())
}
repo.updateDoneChannel <- err
} }
resChan <- updateResponse{
repoRuntime: repoRuntime,
err: errUpdate,
}
duration := time.Since(start)
Log.Info("update completed", zap.Duration("duration", duration), zap.String("chan", fmt.Sprintf("%p", resChan)))
status.M.UpdatesCompletedCounter.WithLabelValues().Inc()
status.M.UpdateDuration.WithLabelValues().Observe(duration.Seconds())
} }
}() }
}
func (repo *Repo) dimensionUpdateRoutine() {
for newDimension := range repo.dimensionUpdateChannel {
Log.Info("dimensionUpdateRoutine received a new dimension", zap.String("dimension", newDimension.Dimension))
err := repo._updateDimension(newDimension.Dimension, newDimension.Node)
Log.Info("dimensionUpdateRoutine received result")
if err != nil {
Log.Debug("update dimension failed", zap.Error(err))
}
repo.dimensionUpdateDoneChannel <- err
}
} }
func (repo *Repo) updateDimension(dimension string, node *content.RepoNode) error { func (repo *Repo) updateDimension(dimension string, node *content.RepoNode) error {
repo.updateChannel <- &repoDimension{ Log.Debug("trying to push dimension into update channel", zap.String("dimension", dimension), zap.String("nodeName", node.Name))
repo.dimensionUpdateChannel <- &repoDimension{
Dimension: dimension, Dimension: dimension,
Node: node, Node: node,
} }
return <-repo.updateDoneChannel Log.Debug("waiting for done signal")
return <-repo.dimensionUpdateDoneChannel
} }
// do not call directly, but only through channel // do not call directly, but only through channel
func (repo *Repo) _updateDimension(dimension string, newNode *content.RepoNode) error { func (repo *Repo) _updateDimension(dimension string, newNode *content.RepoNode) error {
newNode.WireParents() newNode.WireParents()
newDirectory := make(map[string]*content.RepoNode)
newURIDirectory := make(map[string]*content.RepoNode)
err := builDirectory(newNode, newDirectory, newURIDirectory) var (
newDirectory = make(map[string]*content.RepoNode)
newURIDirectory = make(map[string]*content.RepoNode)
err = buildDirectory(newNode, newDirectory, newURIDirectory)
)
if err != nil { if err != nil {
return errors.New("update dimension \"" + dimension + "\" failed when building its directory:: " + err.Error()) return errors.New("update dimension \"" + dimension + "\" failed when building its directory:: " + err.Error())
} }
@ -53,23 +89,43 @@ func (repo *Repo) _updateDimension(dimension string, newNode *content.RepoNode)
return err return err
} }
// ---------------------------------------------
// copy old datastructure to prevent concurrent map access
// collect other dimension in the Directory
newRepoDirectory := map[string]*Dimension{} newRepoDirectory := map[string]*Dimension{}
for d, D := range repo.Directory { for d, D := range repo.Directory {
if d != dimension { if d != dimension {
newRepoDirectory[d] = D newRepoDirectory[d] = D
} }
} }
// add the new dimension
newRepoDirectory[dimension] = &Dimension{ newRepoDirectory[dimension] = &Dimension{
Node: newNode, Node: newNode,
Directory: newDirectory, Directory: newDirectory,
URIDirectory: newURIDirectory, URIDirectory: newURIDirectory,
} }
repo.Directory = newRepoDirectory repo.Directory = newRepoDirectory
// ---------------------------------------------
// @TODO: why not update only the dimension that has changed instead?
// repo.Directory[dimension] = &Dimension{
// Node: newNode,
// Directory: newDirectory,
// URIDirectory: newURIDirectory,
// }
// ---------------------------------------------
return nil return nil
} }
func builDirectory(dirNode *content.RepoNode, directory map[string]*content.RepoNode, uRIDirectory map[string]*content.RepoNode) error { func buildDirectory(dirNode *content.RepoNode, directory map[string]*content.RepoNode, uRIDirectory map[string]*content.RepoNode) error {
log.Debug("repo.buildDirectory: " + dirNode.ID)
// Log.Debug("buildDirectory", zap.String("ID", dirNode.ID))
existingNode, ok := directory[dirNode.ID] existingNode, ok := directory[dirNode.ID]
if ok { if ok {
return errors.New("duplicate node with id:" + existingNode.ID) return errors.New("duplicate node with id:" + existingNode.ID)
@ -81,7 +137,7 @@ func builDirectory(dirNode *content.RepoNode, directory map[string]*content.Repo
} }
uRIDirectory[dirNode.URI] = dirNode uRIDirectory[dirNode.URI] = dirNode
for _, childNode := range dirNode.Nodes { for _, childNode := range dirNode.Nodes {
err := builDirectory(childNode, directory, uRIDirectory) err := buildDirectory(childNode, directory, uRIDirectory)
if err != nil { if err != nil {
return err return err
} }
@ -102,74 +158,98 @@ func wireAliases(directory map[string]*content.RepoNode) error {
return nil return nil
} }
func loadNodesFromJSON(jsonBytes []byte) (nodes map[string]*content.RepoNode, err error) { func (repo *Repo) loadNodesFromJSON() (nodes map[string]*content.RepoNode, err error) {
nodes = make(map[string]*content.RepoNode) nodes = make(map[string]*content.RepoNode)
err = json.Unmarshal(jsonBytes, &nodes) err = json.Unmarshal(repo.jsonBuf.Bytes(), &nodes)
return nodes, err return nodes, err
} }
func (repo *Repo) tryToRestoreCurrent() error { func (repo *Repo) tryToRestoreCurrent() (err error) {
currentJSONBytes, err := repo.history.getCurrent() err = repo.history.getCurrent(&repo.jsonBuf)
if err != nil { if err != nil {
return err return err
} }
return repo.loadJSONBytes(currentJSONBytes) return repo.loadJSONBytes()
} }
func get(URL string) (data []byte, err error) { func (repo *Repo) get(URL string) (err error) {
response, err := http.Get(URL) response, err := http.Get(URL)
if err != nil { if err != nil {
return data, err return err
} }
defer response.Body.Close() defer response.Body.Close()
if response.StatusCode != http.StatusOK { if response.StatusCode != http.StatusOK {
return data, fmt.Errorf("Bad HTTP Response: %q", response.Status) return fmt.Errorf("Bad HTTP Response: %q", response.Status)
} }
return ioutil.ReadAll(response.Body)
// Log.Info(ansi.Red + "RESETTING BUFFER" + ansi.Reset)
repo.jsonBuf.Reset()
// Log.Info(ansi.Green + "LOADING DATA INTO BUFFER" + ansi.Reset)
_, err = io.Copy(&repo.jsonBuf, response.Body)
return err
} }
func (repo *Repo) update() (repoRuntime int64, jsonBytes []byte, err error) { func (repo *Repo) update() (repoRuntime int64, err error) {
startTimeRepo := time.Now().UnixNano() startTimeRepo := time.Now().UnixNano()
jsonBytes, err = get(repo.server) err = repo.get(repo.server)
repoRuntime = time.Now().UnixNano() - startTimeRepo repoRuntime = time.Now().UnixNano() - startTimeRepo
if err != nil { if err != nil {
// we have no json to load - the repo server did not reply // we have no json to load - the repo server did not reply
log.Debug("we have no json to load - the repo server did not reply", err) Log.Debug("failed to load json", zap.Error(err))
return repoRuntime, jsonBytes, err return repoRuntime, err
} }
log.Debug("loading json from: "+repo.server, string(jsonBytes)) Log.Debug("loading json", zap.String("server", repo.server), zap.Int("length", len(repo.jsonBuf.Bytes())))
nodes, err := loadNodesFromJSON(jsonBytes) nodes, err := repo.loadNodesFromJSON()
if err != nil { if err != nil {
// could not load nodes from json // could not load nodes from json
return repoRuntime, jsonBytes, err return repoRuntime, err
} }
err = repo.loadNodes(nodes) err = repo.loadNodes(nodes)
if err != nil { if err != nil {
// repo failed to load nodes // repo failed to load nodes
return repoRuntime, jsonBytes, err return repoRuntime, err
} }
return repoRuntime, jsonBytes, nil return repoRuntime, nil
} }
func (repo *Repo) loadJSONBytes(jsonBytes []byte) error { // limit ressources and allow only one update request at once
nodes, err := loadNodesFromJSON(jsonBytes) func (repo *Repo) tryUpdate() (repoRuntime int64, err error) {
c := make(chan updateResponse)
select {
case repo.updateInProgressChannel <- c:
Log.Info("update request added to queue")
ur := <-c
return ur.repoRuntime, ur.err
default:
Log.Info("update request rejected, queue is full")
status.M.UpdatesRejectedCounter.WithLabelValues().Inc()
return 0, errUpdateRejected
}
}
func (repo *Repo) loadJSONBytes() error {
nodes, err := repo.loadNodesFromJSON()
if err != nil { if err != nil {
log.Debug("could not parse json", string(jsonBytes)) data := repo.jsonBuf.Bytes()
if len(data) > 10 {
Log.Debug("could not parse json",
zap.String("jsonStart", string(data[:10])),
zap.String("jsonStart", string(data[len(data)-10:])),
)
}
return err return err
} }
err = repo.loadNodes(nodes) err = repo.loadNodes(nodes)
if err == nil { if err == nil {
historyErr := repo.history.add(jsonBytes) historyErr := repo.history.add(repo.jsonBuf.Bytes())
if historyErr != nil { if historyErr != nil {
log.Warning("could not add valid json to history:" + historyErr.Error()) Log.Error("could not add valid json to history", zap.Error(historyErr))
status.M.HistoryPersistFailedCounter.WithLabelValues(historyErr.Error()).Inc()
} else { } else {
log.Record("added valid json to history") Log.Info("added valid json to history")
}
cleanUpErr := repo.history.cleanup()
if cleanUpErr != nil {
log.Warning("an error occured while cleaning up my history:", cleanUpErr)
} else {
log.Record("cleaned up history")
} }
} }
return err return err
@ -179,10 +259,10 @@ func (repo *Repo) loadNodes(newNodes map[string]*content.RepoNode) error {
newDimensions := []string{} newDimensions := []string{}
for dimension, newNode := range newNodes { for dimension, newNode := range newNodes {
newDimensions = append(newDimensions, dimension) newDimensions = append(newDimensions, dimension)
log.Debug("loading nodes for dimension " + dimension) Log.Debug("loading nodes for dimension", zap.String("dimension", dimension))
loadErr := repo.updateDimension(dimension, newNode) loadErr := repo.updateDimension(dimension, newNode)
if loadErr != nil { if loadErr != nil {
log.Debug(" failed to load " + dimension + ": " + loadErr.Error()) Log.Debug("failed to load", zap.String("dimension", dimension), zap.Error(loadErr))
return loadErr return loadErr
} }
} }
@ -197,7 +277,7 @@ func (repo *Repo) loadNodes(newNodes map[string]*content.RepoNode) error {
// we need to throw away orphaned dimensions // we need to throw away orphaned dimensions
for dimension := range repo.Directory { for dimension := range repo.Directory {
if !dimensionIsValid(dimension) { if !dimensionIsValid(dimension) {
log.Notice("removing orphaned dimension:" + dimension) Log.Info("removing orphaned dimension", zap.String("dimension", dimension))
delete(repo.Directory, dimension) delete(repo.Directory, dimension)
} }
} }

3
repo/mock/mock.go
View File

@ -9,13 +9,12 @@ import (
"testing" "testing"
"time" "time"
"github.com/foomo/contentserver/log"
"github.com/foomo/contentserver/requests" "github.com/foomo/contentserver/requests"
) )
// GetMockData mock data to run a repo // GetMockData mock data to run a repo
func GetMockData(t testing.TB) (server *httptest.Server, varDir string) { func GetMockData(t testing.TB) (server *httptest.Server, varDir string) {
log.SelectedLevel = log.LevelError
_, filename, _, _ := runtime.Caller(0) _, filename, _, _ := runtime.Caller(0)
mockDir := path.Dir(filename) mockDir := path.Dir(filename)

179
repo/repo.go
View File

@ -1,17 +1,25 @@
package repo package repo
import ( import (
"bytes"
"errors" "errors"
"fmt" "fmt"
"io"
"os"
"strings" "strings"
"time" "time"
"github.com/foomo/contentserver/status"
"github.com/foomo/contentserver/content" "github.com/foomo/contentserver/content"
"github.com/foomo/contentserver/log" . "github.com/foomo/contentserver/logger"
"github.com/foomo/contentserver/requests" "github.com/foomo/contentserver/requests"
"github.com/foomo/contentserver/responses" "github.com/foomo/contentserver/responses"
"go.uber.org/zap"
) )
const maxGetURIForNodeRecursionLevel = 1000
// Dimension dimension in a repo // Dimension dimension in a repo
type Dimension struct { type Dimension struct {
Directory map[string]*content.RepoNode Directory map[string]*content.RepoNode
@ -21,11 +29,17 @@ type Dimension struct {
// Repo content repositiory // Repo content repositiory
type Repo struct { type Repo struct {
server string server string
Directory map[string]*Dimension Directory map[string]*Dimension
updateChannel chan *repoDimension // updateLock sync.Mutex
updateDoneChannel chan error dimensionUpdateChannel chan *repoDimension
history *history dimensionUpdateDoneChannel chan error
history *history
updateInProgressChannel chan chan updateResponse
// jsonBytes []byte
jsonBuf bytes.Buffer
} }
type repoDimension struct { type repoDimension struct {
@ -35,22 +49,29 @@ type repoDimension struct {
// NewRepo constructor // NewRepo constructor
func NewRepo(server string, varDir string) *Repo { func NewRepo(server string, varDir string) *Repo {
log.Notice("creating new repo for " + server)
log.Notice(" using var dir:" + varDir) Log.Info("creating new repo",
zap.String("server", server),
zap.String("varDir", varDir),
)
repo := &Repo{ repo := &Repo{
server: server, server: server,
Directory: map[string]*Dimension{}, Directory: map[string]*Dimension{},
history: newHistory(varDir), history: newHistory(varDir),
updateChannel: make(chan *repoDimension), dimensionUpdateChannel: make(chan *repoDimension),
updateDoneChannel: make(chan error), dimensionUpdateDoneChannel: make(chan error),
updateInProgressChannel: make(chan chan updateResponse, 0),
} }
go repo.updateRoutine() go repo.updateRoutine()
log.Record("trying to restore pervious state") go repo.dimensionUpdateRoutine()
Log.Info("trying to restore previous state")
restoreErr := repo.tryToRestoreCurrent() restoreErr := repo.tryToRestoreCurrent()
if restoreErr != nil { if restoreErr != nil {
log.Record(" could not restore previous repo content:" + restoreErr.Error()) Log.Error(" could not restore previous repo content", zap.Error(restoreErr))
} else { } else {
log.Record(" restored previous repo content") Log.Info("restored previous repo content")
} }
return repo return repo
} }
@ -70,10 +91,18 @@ func (repo *Repo) GetNodes(r *requests.Nodes) map[string]*content.Node {
} }
func (repo *Repo) getNodes(nodeRequests map[string]*requests.Node, env *requests.Env) map[string]*content.Node { func (repo *Repo) getNodes(nodeRequests map[string]*requests.Node, env *requests.Env) map[string]*content.Node {
nodes := map[string]*content.Node{}
path := []*content.Item{} var (
nodes = map[string]*content.Node{}
path = []*content.Item{}
)
for nodeName, nodeRequest := range nodeRequests { for nodeName, nodeRequest := range nodeRequests {
log.Debug(" adding node " + nodeName + " " + nodeRequest.ID)
if nodeName == "" || nodeRequest.ID == "" {
Log.Info("invalid node request", zap.Error(errors.New("nodeName or nodeRequest.ID empty")))
continue
}
Log.Debug("adding node", zap.String("name", nodeName), zap.String("requestID", nodeRequest.ID))
groups := env.Groups groups := env.Groups
if len(nodeRequest.Groups) > 0 { if len(nodeRequest.Groups) > 0 {
@ -84,26 +113,29 @@ func (repo *Repo) getNodes(nodeRequests map[string]*requests.Node, env *requests
nodes[nodeName] = nil nodes[nodeName] = nil
if !ok && nodeRequest.Dimension == "" { if !ok && nodeRequest.Dimension == "" {
log.Debug(" could not get dimension root node for dimension " + nodeRequest.Dimension) Log.Debug("could not get dimension root node", zap.String("dimension", nodeRequest.Dimension))
for _, dimension := range env.Dimensions { for _, dimension := range env.Dimensions {
dimensionNode, ok = repo.Directory[dimension] dimensionNode, ok = repo.Directory[dimension]
if ok { if ok {
log.Debug(" searched for root node in env.dimension " + dimension + " with success") Log.Debug("found root node in env.Dimensions", zap.String("dimension", dimension))
break break
} }
log.Debug(" searched for root node in env.dimension " + dimension + " without success") Log.Debug("could NOT find root node in env.Dimensions", zap.String("dimension", dimension))
} }
} }
if !ok { if !ok {
log.Warning("could not get dimension root node for nodeRequest.Dimension: " + nodeRequest.Dimension) Log.Error("could not get dimension root node", zap.String("nodeRequest.Dimension", nodeRequest.Dimension))
continue continue
} }
treeNode, ok := dimensionNode.Directory[nodeRequest.ID] treeNode, ok := dimensionNode.Directory[nodeRequest.ID]
if ok { if ok {
nodes[nodeName] = repo.getNode(treeNode, nodeRequest.Expand, nodeRequest.MimeTypes, path, 0, groups, nodeRequest.DataFields) nodes[nodeName] = repo.getNode(treeNode, nodeRequest.Expand, nodeRequest.MimeTypes, path, 0, groups, nodeRequest.DataFields)
} else { } else {
log.Warning("you are requesting an invalid tree node for " + nodeName + " : " + nodeRequest.ID) Log.Error("an invalid tree node was requested",
zap.String("nodeName", nodeName),
zap.String("ID", nodeRequest.ID),
)
} }
} }
return nodes return nodes
@ -122,18 +154,18 @@ func (repo *Repo) GetContent(r *requests.Content) (c *content.SiteContent, err e
// add more input validation // add more input validation
err = repo.validateContentRequest(r) err = repo.validateContentRequest(r)
if err != nil { if err != nil {
log.Debug("repo.GetContent invalid request", err) Log.Error("repo.GetContent invalid request", zap.Error(err))
return return
} }
log.Debug("repo.GetContent: ", r.URI) Log.Debug("repo.GetContent", zap.String("URI", r.URI))
c = content.NewSiteContent() c = content.NewSiteContent()
resolved, resolvedURI, resolvedDimension, node := repo.resolveContent(r.Env.Dimensions, r.URI) resolved, resolvedURI, resolvedDimension, node := repo.resolveContent(r.Env.Dimensions, r.URI)
if resolved { if resolved {
if !node.CanBeAccessedByGroups(r.Env.Groups) { if !node.CanBeAccessedByGroups(r.Env.Groups) {
log.Notice("401 for " + r.URI) Log.Warn("resolvecontent got status 401", zap.String("URI", r.URI))
c.Status = content.StatusForbidden c.Status = content.StatusForbidden
} else { } else {
log.Notice("200 for " + r.URI) Log.Info("resolvecontent got status 200", zap.String("URI", r.URI))
c.Status = content.StatusOk c.Status = content.StatusOk
c.Data = node.Data c.Data = node.Data
} }
@ -149,15 +181,22 @@ func (repo *Repo) GetContent(r *requests.Content) (c *content.SiteContent, err e
} }
c.URIs = uris c.URIs = uris
} else { } else {
log.Notice("404 for " + r.URI) Log.Info("resolvecontent got status 404", zap.String("URI", r.URI))
c.Status = content.StatusNotFound c.Status = content.StatusNotFound
c.Dimension = r.Env.Dimensions[0] c.Dimension = r.Env.Dimensions[0]
} }
if log.SelectedLevel == log.LevelDebug {
log.Debug(fmt.Sprintf("resolved: %v, uri: %v, dim: %v, n: %v", resolved, resolvedURI, resolvedDimension, node)) Log.Debug("got content",
} zap.Bool("resolved", resolved),
if resolved == false { zap.String("resolvedURI", resolvedURI),
log.Debug("repo.GetContent", r.URI, "could not be resolved falling back to default dimension", r.Env.Dimensions[0]) zap.String("resolvedDimension", resolvedDimension),
zap.String("nodeName", node.Name),
)
if !resolved {
Log.Debug("failed to resolve, falling back to default dimension",
zap.String("URI", r.URI),
zap.String("defaultDimension", r.Env.Dimensions[0]),
)
// r.Env.Dimensions is validated => we can access it // r.Env.Dimensions is validated => we can access it
resolvedDimension = r.Env.Dimensions[0] resolvedDimension = r.Env.Dimensions[0]
} }
@ -180,13 +219,35 @@ func (repo *Repo) GetRepo() map[string]*content.RepoNode {
return response return response
} }
// WriteRepoBytes get the whole repo in all dimensions
// reads the JSON history file from the Filesystem and copies it directly in to the supplied buffer
// the result is wrapped as service response, e.g: {"reply": <contentData>}
func (repo *Repo) WriteRepoBytes(w io.Writer) {
f, err := os.Open(repo.history.getCurrentFilename())
if err != nil {
Log.Error("failed to serve Repo JSON", zap.Error(err))
}
w.Write([]byte("{\"reply\":"))
_, err = io.Copy(w, f)
if err != nil {
Log.Error("failed to serve Repo JSON", zap.Error(err))
}
w.Write([]byte("}"))
}
// Update - reload contents of repository with json from repo.server // Update - reload contents of repository with json from repo.server
func (repo *Repo) Update() (updateResponse *responses.Update) { func (repo *Repo) Update() (updateResponse *responses.Update) {
floatSeconds := func(nanoSeconds int64) float64 { floatSeconds := func(nanoSeconds int64) float64 {
return float64(float64(nanoSeconds) / float64(1000000000.0)) return float64(float64(nanoSeconds) / float64(1000000000.0))
} }
Log.Info("Update triggered")
// Log.Info(ansi.Yellow + "BUFFER LENGTH BEFORE tryUpdate(): " + strconv.Itoa(len(repo.jsonBuf.Bytes())) + ansi.Reset)
startTime := time.Now().UnixNano() startTime := time.Now().UnixNano()
updateRepotime, jsonBytes, updateErr := repo.update() updateRepotime, updateErr := repo.tryUpdate()
updateResponse = &responses.Update{} updateResponse = &responses.Update{}
updateResponse.Stats.RepoRuntime = floatSeconds(updateRepotime) updateResponse.Stats.RepoRuntime = floatSeconds(updateRepotime)
@ -195,20 +256,26 @@ func (repo *Repo) Update() (updateResponse *responses.Update) {
updateResponse.Stats.NumberOfNodes = -1 updateResponse.Stats.NumberOfNodes = -1
updateResponse.Stats.NumberOfURIs = -1 updateResponse.Stats.NumberOfURIs = -1
// let us try to restore the world from a file // let us try to restore the world from a file
log.Error("could not update repository:" + updateErr.Error()) Log.Error("could not update repository:", zap.Error(updateErr))
// Log.Info(ansi.Yellow + "BUFFER LENGTH AFTER ERROR: " + strconv.Itoa(len(repo.jsonBuf.Bytes())) + ansi.Reset)
updateResponse.ErrorMessage = updateErr.Error() updateResponse.ErrorMessage = updateErr.Error()
restoreErr := repo.tryToRestoreCurrent()
if restoreErr != nil { // only try to restore if the update failed during processing
log.Error("failed to restore preceding repo version: " + restoreErr.Error()) if updateErr != errUpdateRejected {
} else { restoreErr := repo.tryToRestoreCurrent()
log.Record("restored current repo from local history") if restoreErr != nil {
Log.Error("failed to restore preceding repo version", zap.Error(restoreErr))
} else {
Log.Info("restored current repo from local history")
}
} }
} else { } else {
updateResponse.Success = true updateResponse.Success = true
// persist the currently loaded one // persist the currently loaded one
historyErr := repo.history.add(jsonBytes) historyErr := repo.history.add(repo.jsonBuf.Bytes())
if historyErr != nil { if historyErr != nil {
log.Warning("could not persist current repo in history: " + historyErr.Error()) Log.Error("could not persist current repo in history", zap.Error(historyErr))
status.M.HistoryPersistFailedCounter.WithLabelValues(historyErr.Error()).Inc()
} }
// add some stats // add some stats
for dimension := range repo.Directory { for dimension := range repo.Directory {
@ -223,11 +290,7 @@ func (repo *Repo) Update() (updateResponse *responses.Update) {
// resolveContent find content in a repository // resolveContent find content in a repository
func (repo *Repo) resolveContent(dimensions []string, URI string) (resolved bool, resolvedURI string, resolvedDimension string, repoNode *content.RepoNode) { func (repo *Repo) resolveContent(dimensions []string, URI string) (resolved bool, resolvedURI string, resolvedDimension string, repoNode *content.RepoNode) {
parts := strings.Split(URI, content.PathSeparator) parts := strings.Split(URI, content.PathSeparator)
resolved = false Log.Debug("repo.ResolveContent", zap.String("URI", URI))
resolvedURI = ""
resolvedDimension = ""
repoNode = nil
log.Debug("repo.ResolveContent: " + URI)
for i := len(parts); i > 0; i-- { for i := len(parts); i > 0; i-- {
testURI := strings.Join(parts[0:i], content.PathSeparator) testURI := strings.Join(parts[0:i], content.PathSeparator)
if testURI == "" { if testURI == "" {
@ -235,11 +298,13 @@ func (repo *Repo) resolveContent(dimensions []string, URI string) (resolved bool
} }
for _, dimension := range dimensions { for _, dimension := range dimensions {
if d, ok := repo.Directory[dimension]; ok { if d, ok := repo.Directory[dimension]; ok {
log.Debug(" testing[" + dimension + "]: " + testURI) Log.Debug("checking",
zap.String("dimension", dimension),
zap.String("URI", testURI),
)
if repoNode, ok := d.URIDirectory[testURI]; ok { if repoNode, ok := d.URIDirectory[testURI]; ok {
resolved = true resolved = true
log.Debug(" found => " + testURI) Log.Debug("found node", zap.String("URI", testURI), zap.String("destination", repoNode.DestinationID))
log.Debug(" destination " + fmt.Sprint(repoNode.DestinationID))
if len(repoNode.DestinationID) > 0 { if len(repoNode.DestinationID) > 0 {
if destionationNode, destinationNodeOk := d.Directory[repoNode.DestinationID]; destinationNodeOk { if destionationNode, destinationNodeOk := d.Directory[repoNode.DestinationID]; destinationNodeOk {
repoNode = destionationNode repoNode = destionationNode
@ -253,8 +318,6 @@ func (repo *Repo) resolveContent(dimensions []string, URI string) (resolved bool
return return
} }
const maxGetURIForNodeRecursionLevel = 1000
func (repo *Repo) getURIForNode(dimension string, repoNode *content.RepoNode, recursionLevel int64) (uri string) { func (repo *Repo) getURIForNode(dimension string, repoNode *content.RepoNode, recursionLevel int64) (uri string) {
if len(repoNode.LinkID) == 0 { if len(repoNode.LinkID) == 0 {
uri = repoNode.URI uri = repoNode.URI
@ -263,7 +326,7 @@ func (repo *Repo) getURIForNode(dimension string, repoNode *content.RepoNode, re
linkedNode, ok := repo.Directory[dimension].Directory[repoNode.LinkID] linkedNode, ok := repo.Directory[dimension].Directory[repoNode.LinkID]
if ok { if ok {
if recursionLevel > maxGetURIForNodeRecursionLevel { if recursionLevel > maxGetURIForNodeRecursionLevel {
log.Error("maxGetURIForNodeRecursionLevel reached for", repoNode.ID, "link id", repoNode.LinkID, "in dimension", dimension) Log.Error("maxGetURIForNodeRecursionLevel reached", zap.String("repoNode.ID", repoNode.ID), zap.String("linkID", repoNode.LinkID), zap.String("dimension", dimension))
return "" return ""
} }
return repo.getURIForNode(dimension, linkedNode, recursionLevel+1) return repo.getURIForNode(dimension, linkedNode, recursionLevel+1)
@ -282,7 +345,7 @@ func (repo *Repo) getURI(dimension string, id string) string {
func (repo *Repo) getNode(repoNode *content.RepoNode, expanded bool, mimeTypes []string, path []*content.Item, level int, groups []string, dataFields []string) *content.Node { func (repo *Repo) getNode(repoNode *content.RepoNode, expanded bool, mimeTypes []string, path []*content.Item, level int, groups []string, dataFields []string) *content.Node {
node := content.NewNode() node := content.NewNode()
node.Item = repoNode.ToItem(dataFields) node.Item = repoNode.ToItem(dataFields)
log.Debug("repo.GetNode: " + repoNode.ID) Log.Debug("getNode", zap.String("ID", repoNode.ID))
for _, childID := range repoNode.Index { for _, childID := range repoNode.Index {
childNode := repoNode.Nodes[childID] childNode := repoNode.Nodes[childID]
if (level == 0 || expanded || !expanded && childNode.InPath(path)) && !childNode.Hidden && childNode.CanBeAccessedByGroups(groups) && childNode.IsOneOfTheseMimeTypes(mimeTypes) { if (level == 0 || expanded || !expanded && childNode.InPath(path)) && !childNode.Hidden && childNode.CanBeAccessedByGroups(groups) && childNode.IsOneOfTheseMimeTypes(mimeTypes) {
@ -327,6 +390,6 @@ func (repo *Repo) hasDimension(d string) bool {
return hasDimension return hasDimension
} }
func uriKeyForState(state string, uri string) string { // func uriKeyForState(state string, uri string) string {
return state + "-" + uri // return state + "-" + uri
} // }

93
repo/repo_test.go
View File

@ -3,11 +3,18 @@ package repo
import ( import (
"strings" "strings"
"testing" "testing"
"time"
. "github.com/foomo/contentserver/logger"
_ "github.com/foomo/contentserver/logger"
"github.com/foomo/contentserver/repo/mock" "github.com/foomo/contentserver/repo/mock"
"github.com/foomo/contentserver/requests" "github.com/foomo/contentserver/requests"
) )
func init() {
SetupLogging(true, "contentserver_repo_test.log")
}
func assertRepoIsEmpty(t *testing.T, r *Repo, empty bool) { func assertRepoIsEmpty(t *testing.T, r *Repo, empty bool) {
if empty { if empty {
if len(r.Directory) > 0 { if len(r.Directory) > 0 {
@ -15,15 +22,18 @@ func assertRepoIsEmpty(t *testing.T, r *Repo, empty bool) {
} }
} else { } else {
if len(r.Directory) == 0 { if len(r.Directory) == 0 {
t.Fatal("directory should not have been empty, but it is") t.Fatal("directory is empty, but should have been not")
} }
} }
} }
func TestLoad404(t *testing.T) { func TestLoad404(t *testing.T) {
mockServer, varDir := mock.GetMockData(t) var (
server := mockServer.URL + "/repo-no-have" mockServer, varDir = mock.GetMockData(t)
r := NewRepo(server, varDir) server = mockServer.URL + "/repo-no-have"
r = NewRepo(server, varDir)
)
time.Sleep(500 * time.Millisecond)
response := r.Update() response := r.Update()
if response.Success { if response.Success {
t.Fatal("can not get a repo, if the server responds with a 404") t.Fatal("can not get a repo, if the server responds with a 404")
@ -31,9 +41,12 @@ func TestLoad404(t *testing.T) {
} }
func TestLoadBrokenRepo(t *testing.T) { func TestLoadBrokenRepo(t *testing.T) {
mockServer, varDir := mock.GetMockData(t) var (
server := mockServer.URL + "/repo-broken-json.json" mockServer, varDir = mock.GetMockData(t)
r := NewRepo(server, varDir) server = mockServer.URL + "/repo-broken-json.json"
r = NewRepo(server, varDir)
)
time.Sleep(500 * time.Millisecond)
response := r.Update() response := r.Update()
if response.Success { if response.Success {
t.Fatal("how could we load a broken json") t.Fatal("how could we load a broken json")
@ -42,13 +55,17 @@ func TestLoadBrokenRepo(t *testing.T) {
func TestLoadRepo(t *testing.T) { func TestLoadRepo(t *testing.T) {
mockServer, varDir := mock.GetMockData(t) var (
server := mockServer.URL + "/repo-ok.json" mockServer, varDir = mock.GetMockData(t)
r := NewRepo(server, varDir) server = mockServer.URL + "/repo-ok.json"
r = NewRepo(server, varDir)
)
assertRepoIsEmpty(t, r, true) assertRepoIsEmpty(t, r, true)
response := r.Update() response := r.Update()
assertRepoIsEmpty(t, r, false) assertRepoIsEmpty(t, r, false)
if response.Success == false {
if !response.Success {
t.Fatal("could not load valid repo") t.Fatal("could not load valid repo")
} }
if response.Stats.OwnRuntime > response.Stats.RepoRuntime { if response.Stats.OwnRuntime > response.Stats.RepoRuntime {
@ -63,23 +80,61 @@ func TestLoadRepo(t *testing.T) {
assertRepoIsEmpty(t, nr, false) assertRepoIsEmpty(t, nr, false)
} }
func BenchmarkLoadRepo(b *testing.B) {
var (
t = &testing.T{}
mockServer, varDir = mock.GetMockData(t)
server = mockServer.URL + "/repo-ok.json"
r = NewRepo(server, varDir)
)
if len(r.Directory) > 0 {
b.Fatal("directory should have been empty, but is not")
}
b.ReportAllocs()
b.ResetTimer()
for n := 0; n < b.N; n++ {
response := r.Update()
if len(r.Directory) == 0 {
b.Fatal("directory is empty, but should have been not")
}
if !response.Success {
b.Fatal("could not load valid repo")
}
}
}
func TestLoadRepoDuplicateUris(t *testing.T) { func TestLoadRepoDuplicateUris(t *testing.T) {
mockServer, varDir := mock.GetMockData(t)
server := mockServer.URL + "/repo-duplicate-uris.json" var (
r := NewRepo(server, varDir) mockServer, varDir = mock.GetMockData(t)
server = mockServer.URL + "/repo-duplicate-uris.json"
r = NewRepo(server, varDir)
)
time.Sleep(500 * time.Millisecond)
response := r.Update() response := r.Update()
if response.Success { if response.Success {
t.Fatal("there are duplicates, this repo update should have failed") t.Fatal("there are duplicates, this repo update should have failed")
} }
if !strings.Contains(response.ErrorMessage, "update dimension") { if !strings.Contains(response.ErrorMessage, "update dimension") {
t.Fatal("error message not as expected") t.Fatal("error message not as expected: " + response.ErrorMessage)
} }
} }
func TestDimensionHygiene(t *testing.T) { func TestDimensionHygiene(t *testing.T) {
mockServer, varDir := mock.GetMockData(t)
server := mockServer.URL + "/repo-two-dimensions.json" var (
r := NewRepo(server, varDir) mockServer, varDir = mock.GetMockData(t)
server = mockServer.URL + "/repo-two-dimensions.json"
r = NewRepo(server, varDir)
)
time.Sleep(500 * time.Millisecond)
response := r.Update() response := r.Update()
if !response.Success { if !response.Success {
t.Fatal("well those two dimension should be fine") t.Fatal("well those two dimension should be fine")
@ -98,6 +153,7 @@ func getTestRepo(path string, t *testing.T) *Repo {
mockServer, varDir := mock.GetMockData(t) mockServer, varDir := mock.GetMockData(t)
server := mockServer.URL + path server := mockServer.URL + path
r := NewRepo(server, varDir) r := NewRepo(server, varDir)
time.Sleep(500 * time.Millisecond)
response := r.Update() response := r.Update()
if !response.Success { if !response.Success {
t.Fatal("well those two dimension should be fine") t.Fatal("well those two dimension should be fine")
@ -138,6 +194,7 @@ func TestLinkIds(t *testing.T) {
mockServer, varDir := mock.GetMockData(t) mockServer, varDir := mock.GetMockData(t)
server := mockServer.URL + "/repo-link-ok.json" server := mockServer.URL + "/repo-link-ok.json"
r := NewRepo(server, varDir) r := NewRepo(server, varDir)
time.Sleep(500 * time.Millisecond)
response := r.Update() response := r.Update()
if !response.Success { if !response.Success {
t.Fatal("those links should have been fine") t.Fatal("those links should have been fine")

93
server/handlerequest.go
View File

@ -1,31 +1,38 @@
package server package server
import ( import (
"encoding/json" "time"
"errors"
"fmt"
"github.com/foomo/contentserver/log" "go.uber.org/zap"
. "github.com/foomo/contentserver/logger"
"github.com/foomo/contentserver/repo" "github.com/foomo/contentserver/repo"
"github.com/foomo/contentserver/requests" "github.com/foomo/contentserver/requests"
"github.com/foomo/contentserver/responses" "github.com/foomo/contentserver/responses"
"github.com/foomo/contentserver/status"
) )
func handleRequest(r *repo.Repo, handler Handler, jsonBytes []byte) (replyBytes []byte, err error) { func handleRequest(r *repo.Repo, handler Handler, jsonBytes []byte, source string) (replyBytes []byte, err error) {
var reply interface{} var (
var apiErr error reply interface{}
var jsonErr error apiErr error
jsonErr error
processIfJSONIsOk := func(err error, processingFunc func()) { start = time.Now()
if err != nil { processIfJSONIsOk = func(err error, processingFunc func()) {
jsonErr = err if err != nil {
return jsonErr = err
return
}
processingFunc()
} }
processingFunc() )
} status.M.ContentRequestCounter.WithLabelValues(source).Inc()
// handle and process
switch handler { switch handler {
// case HandlerGetRepo: // This case is handled prior to handleRequest being called.
// since the resulting bytes are written directly in to the http.ResponseWriter / net.Connection
case HandlerGetURIs: case HandlerGetURIs:
getURIRequest := &requests.URIs{} getURIRequest := &requests.URIs{}
processIfJSONIsOk(json.Unmarshal(jsonBytes, &getURIRequest), func() { processIfJSONIsOk(json.Unmarshal(jsonBytes, &getURIRequest), func() {
@ -46,38 +53,46 @@ func handleRequest(r *repo.Repo, handler Handler, jsonBytes []byte) (replyBytes
processIfJSONIsOk(json.Unmarshal(jsonBytes, &updateRequest), func() { processIfJSONIsOk(json.Unmarshal(jsonBytes, &updateRequest), func() {
reply = r.Update() reply = r.Update()
}) })
case HandlerGetRepo:
repoRequest := &requests.Repo{}
processIfJSONIsOk(json.Unmarshal(jsonBytes, &repoRequest), func() {
reply = r.GetRepo()
})
default: default:
err = errors.New(log.Error(" can not handle this one " + handler)) reply = responses.NewError(1, "unknown handler: "+string(handler))
errorResponse := responses.NewError(1, "unknown handler")
reply = errorResponse
} }
addMetrics(handler, start, jsonErr, apiErr, source)
// error handling
if jsonErr != nil { if jsonErr != nil {
err = jsonErr Log.Error("could not read incoming json", zap.Error(jsonErr))
log.Error(" could not read incoming json:", jsonErr) reply = responses.NewError(2, "could not read incoming json "+jsonErr.Error())
errorResponse := responses.NewError(2, "could not read incoming json "+jsonErr.Error())
reply = errorResponse
} else if apiErr != nil { } else if apiErr != nil {
log.Error(" an API error occured:", apiErr) Log.Error("an API error occured", zap.Error(apiErr))
err = apiErr
reply = responses.NewError(3, "internal error "+apiErr.Error()) reply = responses.NewError(3, "internal error "+apiErr.Error())
} }
return encodeReply(reply) return encodeReply(reply)
} }
func encodeReply(reply interface{}) (replyBytes []byte, err error) { func addMetrics(handlerName Handler, start time.Time, errJSON error, errAPI error, source string) {
encodedBytes, jsonReplyErr := json.MarshalIndent(map[string]interface{}{
"reply": reply, var (
}, "", " ") duration = time.Since(start)
if jsonReplyErr != nil { s = "succeeded"
err = jsonReplyErr )
log.Error(" could not encode reply " + fmt.Sprint(jsonReplyErr)) if errJSON != nil || errAPI != nil {
} else { s = "failed"
replyBytes = encodedBytes
} }
return replyBytes, err
status.M.ServiceRequestCounter.WithLabelValues(string(handlerName), s, source).Inc()
status.M.ServiceRequestDuration.WithLabelValues(string(handlerName), s, source).Observe(float64(duration.Seconds()))
}
// encodeReply takes an interface and encodes it as JSON
// it returns the resulting JSON and a marshalling error
func encodeReply(reply interface{}) (replyBytes []byte, err error) {
replyBytes, err = json.Marshal(map[string]interface{}{
"reply": reply,
})
if err != nil {
Log.Error("could not encode reply", zap.Error(err))
}
return
} }

78
server/server.go
View File

@ -5,9 +5,16 @@ import (
"fmt" "fmt"
"net" "net"
"net/http" "net/http"
"os"
"github.com/foomo/contentserver/log" . "github.com/foomo/contentserver/logger"
"github.com/foomo/contentserver/repo" "github.com/foomo/contentserver/repo"
jsoniter "github.com/json-iterator/go"
"go.uber.org/zap"
)
var (
json = jsoniter.ConfigCompatibleWithStandardLibrary
) )
// Handler type // Handler type
@ -34,23 +41,42 @@ func Run(server string, address string, varDir string) error {
func RunServerSocketAndWebServer( func RunServerSocketAndWebServer(
server string, server string,
address string, address string,
webserverAdresss string, webserverAddress string,
webserverPath string, webserverPath string,
varDir string, varDir string,
) error { ) error {
if address == "" && webserverAdresss == "" { if address == "" && webserverAddress == "" {
return errors.New("one of the addresses needs to be set") return errors.New("one of the addresses needs to be set")
} }
log.Record("building repo with content from " + server) Log.Info("building repo with content", zap.String("server", server))
r := repo.NewRepo(server, varDir) r := repo.NewRepo(server, varDir)
go r.Update()
// start initial update and handle error
go func() {
resp := r.Update()
if !resp.Success {
Log.Error("failed to update",
zap.String("error", resp.ErrorMessage),
zap.Int("NumberOfNodes", resp.Stats.NumberOfNodes),
zap.Int("NumberOfURIs", resp.Stats.NumberOfURIs),
zap.Float64("OwnRuntime", resp.Stats.OwnRuntime),
zap.Float64("RepoRuntime", resp.Stats.RepoRuntime),
)
os.Exit(1)
}
}()
// update can run in bg // update can run in bg
chanErr := make(chan error) chanErr := make(chan error)
if address != "" { if address != "" {
Log.Info("starting socketserver", zap.String("address", address))
go runSocketServer(r, address, chanErr) go runSocketServer(r, address, chanErr)
} }
if webserverAdresss != "" { if webserverAddress != "" {
go runWebserver(r, webserverAdresss, webserverPath, chanErr) Log.Info("starting webserver", zap.String("webserverAddress", webserverAddress))
go runWebserver(r, webserverAddress, webserverPath, chanErr)
} }
return <-chanErr return <-chanErr
} }
@ -61,12 +87,7 @@ func runWebserver(
path string, path string,
chanErr chan error, chanErr chan error,
) { ) {
s, errNew := NewWebServer(path, r) chanErr <- http.ListenAndServe(address, NewWebServer(path, r))
if errNew != nil {
chanErr <- errNew
return
}
chanErr <- http.ListenAndServe(address, s)
} }
func runSocketServer( func runSocketServer(
@ -74,31 +95,32 @@ func runSocketServer(
address string, address string,
chanErr chan error, chanErr chan error,
) { ) {
s := &socketServer{ // create socket server
stats: newStats(), s := newSocketServer(repo)
repo: repo,
} // listen on socket
ln, err := net.Listen("tcp", address) ln, errListen := net.Listen("tcp", address)
if err != nil { if errListen != nil {
err = errors.New("RunSocketServer: could not start the on \"" + address + "\" - error: " + fmt.Sprint(err)) Log.Error("runSocketServer: could not start",
// failed to create socket zap.String("address", address),
log.Error(err) zap.Error(errListen),
chanErr <- err )
chanErr <- errors.New("runSocketServer: could not start the on \"" + address + "\" - error: " + fmt.Sprint(errListen))
return return
} }
// there we go
log.Record("RunSocketServer: started to listen on " + address) Log.Info("runSocketServer: started listening", zap.String("address", address))
for { for {
// this blocks until connection or error // this blocks until connection or error
conn, err := ln.Accept() conn, err := ln.Accept()
if err != nil { if err != nil {
log.Error("RunSocketServer: could not accept connection" + fmt.Sprint(err)) Log.Error("runSocketServer: could not accept connection", zap.Error(err))
continue continue
} }
log.Debug("new connection")
// a goroutine handles conn so that the loop can accept other connections // a goroutine handles conn so that the loop can accept other connections
go func() { go func() {
log.Debug("accepted connection") Log.Debug("accepted connection", zap.String("source", conn.RemoteAddr().String()))
s.handleConnection(conn) s.handleConnection(conn)
conn.Close() conn.Close()
// log.Debug("connection closed") // log.Debug("connection closed")

129
server/socketserver.go
View File

@ -1,48 +1,34 @@
package server package server
import ( import (
"bytes"
"errors" "errors"
"fmt" "fmt"
"net" "net"
"strconv" "strconv"
"strings" "strings"
"time"
"github.com/foomo/contentserver/log" "go.uber.org/zap"
. "github.com/foomo/contentserver/logger"
"github.com/foomo/contentserver/repo" "github.com/foomo/contentserver/repo"
"github.com/foomo/contentserver/responses" "github.com/foomo/contentserver/responses"
"github.com/foomo/contentserver/status"
) )
// simple internal request counter const sourceSocketServer = "socketserver"
type stats struct {
requests int64
chanCount chan int
}
func newStats() *stats {
s := &stats{
requests: 0,
chanCount: make(chan int),
}
go func() {
for {
select {
case <-s.chanCount:
s.requests++
s.chanCount <- 1
}
}
}()
return s
}
func (s *stats) countRequest() {
s.chanCount <- 1
<-s.chanCount
}
type socketServer struct { type socketServer struct {
stats *stats repo *repo.Repo
repo *repo.Repo metrics *status.Metrics
}
// newSocketServer returns a shiny new socket server
func newSocketServer(repo *repo.Repo) *socketServer {
return &socketServer{
repo: repo,
}
} }
func extractHandlerAndJSONLentgh(header string) (handler Handler, jsonLength int, err error) { func extractHandlerAndJSONLentgh(header string) (handler Handler, jsonLength int, err error) {
@ -58,14 +44,22 @@ func extractHandlerAndJSONLentgh(header string) (handler Handler, jsonLength int
} }
func (s *socketServer) execute(handler Handler, jsonBytes []byte) (reply []byte) { func (s *socketServer) execute(handler Handler, jsonBytes []byte) (reply []byte) {
s.stats.countRequest() Log.Debug("incoming json buffer", zap.Int("length", len(jsonBytes)))
log.Notice("socketServer.execute: ", s.stats.requests, ", ", handler)
if log.SelectedLevel == log.LevelDebug { if handler == HandlerGetRepo {
log.Debug(" incoming json buffer:", string(jsonBytes))
var (
b bytes.Buffer
start = time.Now()
)
s.repo.WriteRepoBytes(&b)
addMetrics(handler, start, nil, nil, sourceSocketServer)
return b.Bytes()
} }
reply, handlingError := handleRequest(s.repo, handler, jsonBytes)
reply, handlingError := handleRequest(s.repo, handler, jsonBytes, sourceSocketServer)
if handlingError != nil { if handlingError != nil {
log.Error("socketServer.execute handlingError :", handlingError) Log.Error("socketServer.execute failed", zap.Error(handlingError))
} }
return reply return reply
} }
@ -73,32 +67,39 @@ func (s *socketServer) execute(handler Handler, jsonBytes []byte) (reply []byte)
func (s *socketServer) writeResponse(conn net.Conn, reply []byte) { func (s *socketServer) writeResponse(conn net.Conn, reply []byte) {
headerBytes := []byte(strconv.Itoa(len(reply))) headerBytes := []byte(strconv.Itoa(len(reply)))
reply = append(headerBytes, reply...) reply = append(headerBytes, reply...)
log.Debug(" replying: " + string(reply)) Log.Debug("replying", zap.String("reply", string(reply)))
n, writeError := conn.Write(reply) n, writeError := conn.Write(reply)
if writeError != nil { if writeError != nil {
log.Error("socketServer.writeResponse: could not write my reply: " + fmt.Sprint(writeError)) Log.Error("socketServer.writeResponse: could not write reply", zap.Error(writeError))
return return
} }
if n < len(reply) { if n < len(reply) {
log.Error(fmt.Sprintf("socketServer.writeResponse: write too short %q instead of %q", n, len(reply))) Log.Error("socketServer.writeResponse: write too short",
zap.Int("got", n),
zap.Int("expected", len(reply)),
)
return return
} }
log.Debug(" replied. waiting for next request on open connection") Log.Debug("replied. waiting for next request on open connection")
} }
func (s *socketServer) handleConnection(conn net.Conn) { func (s *socketServer) handleConnection(conn net.Conn) {
log.Debug("socketServer.handleConnection") Log.Debug("socketServer.handleConnection")
var headerBuffer [1]byte status.M.NumSocketsGauge.WithLabelValues(conn.RemoteAddr().String()).Inc()
header := ""
i := 0 var (
headerBuffer [1]byte
header = ""
i = 0
)
for { for {
i++ i++
// fmt.Println("---->", i) // fmt.Println("---->", i)
// let us read with 1 byte steps on conn until we find "{" // let us read with 1 byte steps on conn until we find "{"
_, readErr := conn.Read(headerBuffer[0:]) _, readErr := conn.Read(headerBuffer[0:])
if readErr != nil { if readErr != nil {
log.Debug(" looks like the client closed the connection: ", readErr) Log.Debug("looks like the client closed the connection", zap.Error(readErr))
status.M.NumSocketsGauge.WithLabelValues(conn.RemoteAddr().String()).Dec()
return return
} }
// read next byte // read next byte
@ -109,44 +110,54 @@ func (s *socketServer) handleConnection(conn net.Conn) {
// reset header // reset header
header = "" header = ""
if headerErr != nil { if headerErr != nil {
log.Error("invalid request could not read header", headerErr) Log.Error("invalid request could not read header", zap.Error(headerErr))
encodedErr, encodingErr := encodeReply(responses.NewError(4, "invalid header "+headerErr.Error())) encodedErr, encodingErr := encodeReply(responses.NewError(4, "invalid header "+headerErr.Error()))
if encodingErr == nil { if encodingErr == nil {
s.writeResponse(conn, encodedErr) s.writeResponse(conn, encodedErr)
} else { } else {
log.Error("could not respond to invalid request", encodingErr) Log.Error("could not respond to invalid request", zap.Error(encodingErr))
} }
return return
} }
log.Debug(fmt.Sprintf(" found json with %d bytes", jsonLength)) Log.Debug("found json", zap.Int("length", jsonLength))
if jsonLength > 0 { if jsonLength > 0 {
// let us try to read some json
jsonBytes := make([]byte, jsonLength) var (
// let us try to read some json
jsonBytes = make([]byte, jsonLength)
jsonLengthCurrent = 1
readRound = 0
)
// that is "{" // that is "{"
jsonBytes[0] = 123 jsonBytes[0] = 123
jsonLengthCurrent := 1
readRound := 0
for jsonLengthCurrent < jsonLength { for jsonLengthCurrent < jsonLength {
readRound++ readRound++
readLength, jsonReadErr := conn.Read(jsonBytes[jsonLengthCurrent:jsonLength]) readLength, jsonReadErr := conn.Read(jsonBytes[jsonLengthCurrent:jsonLength])
if jsonReadErr != nil { if jsonReadErr != nil {
//@fixme we need to force a read timeout (SetReadDeadline?), if expected jsonLength is lower than really sent bytes (e.g. if client implements protocol wrong) //@fixme we need to force a read timeout (SetReadDeadline?), if expected jsonLength is lower than really sent bytes (e.g. if client implements protocol wrong)
//@todo should we check for io.EOF here //@todo should we check for io.EOF here
log.Error(" could not read json - giving up with this client connection" + fmt.Sprint(jsonReadErr)) Log.Error("could not read json - giving up with this client connection", zap.Error(jsonReadErr))
status.M.NumSocketsGauge.WithLabelValues(conn.RemoteAddr().String()).Dec()
return return
} }
jsonLengthCurrent += readLength jsonLengthCurrent += readLength
log.Debug(fmt.Sprintf(" read so far %d of %d bytes in read cycle %d", jsonLengthCurrent, jsonLength, readRound)) Log.Debug("read cycle status",
zap.Int("jsonLengthCurrent", jsonLengthCurrent),
zap.Int("jsonLength", jsonLength),
zap.Int("readRound", readRound),
)
} }
if log.SelectedLevel == log.LevelDebug { Log.Debug("read json", zap.Int("length", len(jsonBytes)))
log.Debug(" read json: " + string(jsonBytes))
}
s.writeResponse(conn, s.execute(handler, jsonBytes)) s.writeResponse(conn, s.execute(handler, jsonBytes))
// note: connection remains open // note: connection remains open
continue continue
} }
log.Error("can not read empty json") Log.Error("can not read empty json")
status.M.NumSocketsGauge.WithLabelValues(conn.RemoteAddr().String()).Dec()
return return
} }
// adding to header byte by byte // adding to header byte by byte

30
server/webserver.go
View File

@ -4,22 +4,27 @@ import (
"io/ioutil" "io/ioutil"
"net/http" "net/http"
"strings" "strings"
"time"
"go.uber.org/zap"
. "github.com/foomo/contentserver/logger"
"github.com/foomo/contentserver/repo" "github.com/foomo/contentserver/repo"
) )
const sourceWebserver = "webserver"
type webServer struct { type webServer struct {
r *repo.Repo
path string path string
r *repo.Repo
} }
// NewWebServer returns a shiny new web server // NewWebServer returns a shiny new web server
func NewWebServer(path string, r *repo.Repo) (s http.Handler, err error) { func NewWebServer(path string, r *repo.Repo) http.Handler {
s = &webServer{ return &webServer{
r: r,
path: path, path: path,
r: r,
} }
return
} }
func (s *webServer) ServeHTTP(w http.ResponseWriter, r *http.Request) { func (s *webServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
@ -33,10 +38,21 @@ func (s *webServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
http.Error(w, "failed to read incoming request", http.StatusBadRequest) http.Error(w, "failed to read incoming request", http.StatusBadRequest)
return return
} }
reply, errReply := handleRequest(s.r, Handler(strings.TrimPrefix(r.URL.Path, s.path+"/")), jsonBytes) h := Handler(strings.TrimPrefix(r.URL.Path, s.path+"/"))
if h == HandlerGetRepo {
start := time.Now()
s.r.WriteRepoBytes(w)
w.Header().Set("Content-Type", "application/json")
addMetrics(h, start, nil, nil, sourceWebserver)
return
}
reply, errReply := handleRequest(s.r, h, jsonBytes, "webserver")
if errReply != nil { if errReply != nil {
http.Error(w, errReply.Error(), http.StatusInternalServerError) http.Error(w, errReply.Error(), http.StatusInternalServerError)
return return
} }
w.Write(reply) _, err := w.Write(reply)
if err != nil {
Log.Error("failed to write webServer reply", zap.Error(err))
}
} }

37
status/healthz.go Normal file
View File

@ -0,0 +1,37 @@
package status
import (
"fmt"
"net/http"
. "github.com/foomo/contentserver/logger"
jsoniter "github.com/json-iterator/go"
"go.uber.org/zap"
)
var (
json = jsoniter.ConfigCompatibleWithStandardLibrary
)
func RunHealthzHandlerListener(address string, serviceName string) {
Log.Info(fmt.Sprintf("starting healthz handler on '%s'" + address))
Log.Error("healthz server failed", zap.Error(http.ListenAndServe(address, HealthzHandler(serviceName))))
}
func HealthzHandler(serviceName string) http.Handler {
var (
data = map[string]string{
"service": serviceName,
}
status, _ = json.Marshal(data)
h = http.NewServeMux()
)
h.Handle("/healthz", http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
_, err := w.Write(status)
if err != nil {
Log.Error("failed to write healthz status", zap.Error(err))
}
}))
return h
}

119
status/metrics.go Normal file
View File

@ -0,0 +1,119 @@
package status
import (
"github.com/prometheus/client_golang/prometheus"
)
// M is the Metrics instance
var M = newMetrics()
const (
namespace = "contentserver"
metricLabelHandler = "handler"
metricLabelStatus = "status"
metricLabelSource = "source"
metricLabelRemote = "remote"
metricLabelError = "error"
)
// Metrics is the structure that holds all prometheus metrics
type Metrics struct {
ServiceRequestCounter *prometheus.CounterVec // count the number of requests for each service function
ServiceRequestDuration *prometheus.SummaryVec // observe the duration of requests for each service function
UpdatesRejectedCounter *prometheus.CounterVec // count the number of completed updates
UpdatesCompletedCounter *prometheus.CounterVec // count the number of rejected updates
UpdatesFailedCounter *prometheus.CounterVec // count the number of updates that had an error
UpdateDuration *prometheus.SummaryVec // observe the duration of each repo.update() call
ContentRequestCounter *prometheus.CounterVec // count the total number of content requests
NumSocketsGauge *prometheus.GaugeVec // keep track of the total number of open sockets
HistoryPersistFailedCounter *prometheus.CounterVec // count the number of failed attempts to persist the content history
}
// newMetrics can be used to instantiate a metrics instance
// since this function will also register each metric and metrics should only be registered once
// it is private
// the package exposes the initialized Metrics instance as the variable M.
func newMetrics() *Metrics {
return &Metrics{
ServiceRequestCounter: newCounterVec(
"service_request_count",
"Count of requests for each handler",
metricLabelHandler, metricLabelStatus, metricLabelSource,
),
ServiceRequestDuration: newSummaryVec(
"service_request_duration_seconds",
"Seconds to unmarshal requests, execute a service function and marshal its reponses",
metricLabelHandler, metricLabelStatus, metricLabelSource,
),
UpdatesRejectedCounter: newCounterVec(
"updates_rejected_count",
"Number of updates that were rejected because the queue was full",
),
UpdatesCompletedCounter: newCounterVec(
"updates_completed_count",
"Number of updates that were successfully completed",
),
UpdatesFailedCounter: newCounterVec(
"updates_failed_count",
"Number of updates that failed due to an error",
metricLabelError,
),
UpdateDuration: newSummaryVec(
"update_duration_seconds",
"Duration in seconds for each successful repo.update() call",
),
ContentRequestCounter: newCounterVec(
"content_request_count",
"Number of requests for content",
metricLabelSource,
),
NumSocketsGauge: newGaugeVec(
"num_sockets_total",
"Total number of currently open socket connections",
metricLabelRemote,
),
HistoryPersistFailedCounter: newCounterVec(
"history_persist_failed_count",
"Number of failures to store the content history on the filesystem",
metricLabelError,
),
}
}
/*
* Metric constructors
*/
func newSummaryVec(name, help string, labels ...string) *prometheus.SummaryVec {
vec := prometheus.NewSummaryVec(
prometheus.SummaryOpts{
Namespace: namespace,
Name: name,
Help: help,
}, labels)
prometheus.MustRegister(vec)
return vec
}
func newCounterVec(name, help string, labels ...string) *prometheus.CounterVec {
vec := prometheus.NewCounterVec(
prometheus.CounterOpts{
Namespace: namespace,
Name: name,
Help: help,
}, labels)
prometheus.MustRegister(vec)
return vec
}
func newGaugeVec(name, help string, labels ...string) *prometheus.GaugeVec {
vec := prometheus.NewGaugeVec(
prometheus.GaugeOpts{
Namespace: namespace,
Name: name,
Help: help,
}, labels)
prometheus.MustRegister(vec)
return vec
}

59
testing/client/client.go Normal file
View File

@ -0,0 +1,59 @@
package main
import (
"flag"
"log"
"time"
"github.com/davecgh/go-spew/spew"
"github.com/foomo/contentserver/client"
)
var (
flagAddr = flag.String("addr", "http://127.0.0.1:9191/contentserver", "set addr")
flagGetRepo = flag.Bool("getRepo", false, "get repo")
flagUpdate = flag.Bool("update", true, "trigger content update")
flagNum = flag.Int("num", 100, "num repititions")
flagDelay = flag.Int("delay", 2, "delay in seconds")
)
func main() {
flag.Parse()
c, errClient := client.NewHTTPClient(*flagAddr)
if errClient != nil {
log.Fatal(errClient)
}
for i := 1; i <= *flagNum; i++ {
if *flagUpdate {
go func(num int) {
log.Println("start update")
resp, errUpdate := c.Update()
if errUpdate != nil {
spew.Dump(resp)
log.Fatal(errUpdate)
}
log.Println(num, "update done", resp)
}(i)
}
if *flagGetRepo {
go func(num int) {
log.Println("GetRepo", num)
resp, err := c.GetRepo()
if err != nil {
// spew.Dump(resp)
log.Fatal("failed to get repo")
}
log.Println(num, "GetRepo done, got", len(resp), "dimensions")
}(i)
}
time.Sleep(time.Duration(*flagDelay) * time.Second)
}
log.Println("done!")
}

35
testing/server/server.go Normal file
View File

@ -0,0 +1,35 @@
package main
import (
"flag"
"log"
"net/http"
)
type testServer struct {
file string
}
func main() {
var (
flagJSONFile = flag.String("json-file", "", "provide a json source file")
flagAddress = flag.String("addr", ":1234", "set the webserver address")
)
flag.Parse()
if *flagJSONFile == "" {
log.Fatal("js source file must be provided")
}
ts := &testServer{
file: *flagJSONFile,
}
log.Println("start test server at", *flagAddress, "serving file:", ts.file)
log.Fatal(http.ListenAndServe(*flagAddress, ts))
}
func (ts *testServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
http.ServeFile(w, r, ts.file)
}