mirror of
https://github.com/foomo/contentserver.git
synced 2025-10-16 12:25:44 +00:00
Merge pull request #14 from foomo/feature/memdebug
merge feature/memdebug
This commit is contained in:
Phil
GitHub
commit
fd0c81bc23
6
.gitignore
vendored
6
.gitignore
vendored
@ -1,5 +1,11 @@
|
||||
data
|
||||
*.log
|
||||
*.test
|
||||
cprof-*
|
||||
var
|
||||
.*
|
||||
*~
|
||||
/bin/
|
||||
/pkg/tmp/
|
||||
/vendor
|
||||
!.git*
|
||||
|
||||
13
.travis.yml
13
.travis.yml
@ -1,5 +1,10 @@
|
||||
language: go
|
||||
go:
|
||||
- 1.10
|
||||
- 1.11
|
||||
- tip
|
||||
go: "1.12"
|
||||
os:
|
||||
- linux
|
||||
dist: trusty
|
||||
sudo: false
|
||||
install: true
|
||||
script:
|
||||
- make dep
|
||||
- make test
|
||||
|
||||
38
Dockerfile
38
Dockerfile
@ -1,21 +1,37 @@
|
||||
FROM scratch
|
||||
##############################
|
||||
###### STAGE: BUILD ######
|
||||
##############################
|
||||
FROM golang:1.12.5 AS build-env
|
||||
|
||||
COPY bin/contentserver-linux-amd64 /usr/sbin/contentserver
|
||||
WORKDIR /src
|
||||
|
||||
# install ca root certificates
|
||||
# https://curl.haxx.se/docs/caextract.html
|
||||
# http://blog.codeship.com/building-minimal-docker-containers-for-go-applications/
|
||||
# does not work on docker for mac :(
|
||||
# ADD https://curl.haxx.se/ca/cacert.pem /etc/ssl/certs/ca-certificates.crt
|
||||
ADD .cacert.pem /etc/ssl/certs/ca-certificates.crt
|
||||
COPY ./go.mod ./go.sum ./
|
||||
RUN go mod download && go mod vendor && go install -i ./vendor/...
|
||||
|
||||
# Import the code from the context.
|
||||
COPY ./ ./
|
||||
|
||||
RUN GOARCH=amd64 GOOS=linux CGO_ENABLED=0 go build -o /contentserver
|
||||
|
||||
##############################
|
||||
###### STAGE: PACKAGE ######
|
||||
##############################
|
||||
FROM alpine
|
||||
|
||||
ENV CONTENT_SERVER_LOG_LEVEL=error
|
||||
ENV CONTENT_SERVER_ADDR=0.0.0.0:80
|
||||
ENV CONTENT_SERVER_VAR_DIR=/var/lib/contentserver
|
||||
ENV LOG_JSON=1
|
||||
|
||||
RUN apk add --update --no-cache ca-certificates curl bash && rm -rf /var/cache/apk/*
|
||||
|
||||
COPY --from=build-env /contentserver /usr/sbin/contentserver
|
||||
|
||||
|
||||
VOLUME $CONTENT_SERVER_VAR_DIR
|
||||
EXPOSE 80
|
||||
|
||||
ENTRYPOINT ["/usr/sbin/contentserver"]
|
||||
|
||||
CMD ["-address=$CONTENT_SERVER_ADDR", "-log-level=$CONTENT_SERVER_LOG_LEVEL", "-var-dir=$CONTENT_SERVER_VAR_DIR"]
|
||||
CMD ["-address=$CONTENT_SERVER_ADDR", "-var-dir=$CONTENT_SERVER_VAR_DIR"]
|
||||
|
||||
EXPOSE 80
|
||||
EXPOSE 9200
|
||||
|
||||
74
Makefile
74
Makefile
@ -1,12 +1,20 @@
|
||||
SHELL := /bin/bash
|
||||
|
||||
TAG=`git describe --exact-match --tags $(git log -n1 --pretty='%h') 2>/dev/null || git rev-parse --abbrev-ref HEAD`
|
||||
TAG?=latest
|
||||
IMAGE=docker-registry.bestbytes.net/contentserver
|
||||
|
||||
# Utils
|
||||
|
||||
all: build test
|
||||
tag:
|
||||
echo $(TAG)
|
||||
dep:
|
||||
env GO111MODULE=on go mod download && env GO111MODULE=on go mod vendor && go install -i ./vendor/...
|
||||
clean:
|
||||
rm -fv bin/contentserve*
|
||||
|
||||
# Build
|
||||
|
||||
build: clean
|
||||
go build -o bin/contentserver
|
||||
build-arch: clean
|
||||
@ -15,11 +23,71 @@ build-arch: clean
|
||||
build-docker: clean build-arch
|
||||
curl https://curl.haxx.se/ca/cacert.pem > .cacert.pem
|
||||
docker build -q . > .image_id
|
||||
docker tag `cat .image_id` docker-registry.bestbytes.net/contentserver:$(TAG)
|
||||
echo "# tagged container `cat .image_id` as docker-registry.bestbytes.net/contentserver:$(TAG)"
|
||||
docker tag `cat .image_id` $(IMAGE):$(TAG)
|
||||
echo "# tagged container `cat .image_id` as $(IMAGE):$(TAG)"
|
||||
rm -vf .image_id .cacert.pem
|
||||
|
||||
build-testclient:
|
||||
go build -o bin/testclient -i github.com/foomo/contentserver/testing/client
|
||||
|
||||
build-testserver:
|
||||
go build -o bin/testserver -i github.com/foomo/contentserver/testing/server
|
||||
|
||||
package: build
|
||||
pkg/build.sh
|
||||
|
||||
# Docker
|
||||
|
||||
docker-build:
|
||||
docker build -t $(IMAGE):$(TAG) .
|
||||
|
||||
docker-push:
|
||||
docker push $(IMAGE):$(TAG)
|
||||
|
||||
# Testing / benchmarks
|
||||
|
||||
test:
|
||||
go test ./...
|
||||
|
||||
bench:
|
||||
go test -run=none -bench=. ./...
|
||||
|
||||
run-testserver:
|
||||
bin/testserver -json-file var/cse-globus-stage-b-with-main-section.json
|
||||
|
||||
run-contentserver:
|
||||
contentserver -var-dir var -webserver-address :9191 -address :9999 http://127.0.0.1:1234
|
||||
|
||||
run-contentserver-freeosmem:
|
||||
contentserver -var-dir var -webserver-address :9191 -address :9999 -free-os-mem 1 http://127.0.0.1:1234
|
||||
|
||||
run-prometheus:
|
||||
prometheus --config.file=prometheus/prometheus.yml
|
||||
|
||||
clean-var:
|
||||
rm var/contentserver-repo-2019*
|
||||
|
||||
# Profiling
|
||||
|
||||
test-cpu-profile:
|
||||
go test -cpuprofile=cprof-client github.com/foomo/contentserver/client
|
||||
go tool pprof --text client.test cprof-client
|
||||
|
||||
go test -cpuprofile=cprof-repo github.com/foomo/contentserver/repo
|
||||
go tool pprof --text repo.test cprof-repo
|
||||
|
||||
test-gctrace:
|
||||
GODEBUG=gctrace=1 go test ./...
|
||||
|
||||
test-malloctrace:
|
||||
GODEBUG=allocfreetrace=1 go test ./...
|
||||
|
||||
trace:
|
||||
curl http://localhost:6060/debug/pprof/trace?seconds=60 > cs-trace
|
||||
go tool trace cs-trace
|
||||
|
||||
pprof-heap-web:
|
||||
go tool pprof -http=":8081" http://localhost:6060/debug/pprof/heap
|
||||
|
||||
pprof-cpu-web:
|
||||
go tool pprof -http=":8081" http://localhost:6060/debug/pprof/profile
|
||||
24
README.md
24
README.md
@ -10,6 +10,10 @@ A Server written in GoLang to mix and resolve content from different content sou
|
||||
|
||||
It's up to you how you use it and which data you want to export to the server. Our intention was to write a fast and cache hazzle-free content server to mix different content sources.
|
||||
|
||||
### Overview
|
||||
|
||||
<img src="graphics/Overview.svg" width="100%" height="500">
|
||||
|
||||
## Export Data
|
||||
|
||||
All you have to do is to provide a tree of content nodes as a JSON encoded RepoNode.
|
||||
@ -39,19 +43,31 @@ All you have to do is to provide a tree of content nodes as a JSON encoded RepoN
|
||||
|
||||
There is a PHP Proxy implementation for foomo in [Foomo.ContentServer](https://github.com/foomo/Foomo.ContentServer). Feel free to use it or to implement your own proxy in the language you love. The API should be easily to implement in every other framework and language, too.
|
||||
|
||||
## Update Flowchart
|
||||
|
||||
<img src="graphics/Update-Flow.svg" width="100%" height="700">
|
||||
|
||||
### Usage
|
||||
|
||||
```bash
|
||||
$ contentserver --help
|
||||
$ contentserver -h
|
||||
Usage of contentserver:
|
||||
-address string
|
||||
address to bind host:port (default "127.0.0.1:8081")
|
||||
-log-level string
|
||||
one of error, record, warning, notice, debug (default "record")
|
||||
address to bind socket server host:port
|
||||
-debug
|
||||
toggle debug mode
|
||||
-free-os-mem int
|
||||
free OS mem every X minutes
|
||||
-heap-dump int
|
||||
dump heap every X minutes
|
||||
-var-dir string
|
||||
where to put my data (default "/var/lib/contentserver")
|
||||
-version
|
||||
version info
|
||||
-webserver-address string
|
||||
address to bind web server host:port, when empty no webserver will be spawned
|
||||
-webserver-path string
|
||||
path to export the webserver on - useful when behind a proxy (default "/contentserver")
|
||||
```
|
||||
|
||||
## License
|
||||
|
||||
@ -1,7 +1,6 @@
|
||||
package client
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net"
|
||||
"strconv"
|
||||
"sync"
|
||||
@ -9,7 +8,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/foomo/contentserver/content"
|
||||
"github.com/foomo/contentserver/log"
|
||||
. "github.com/foomo/contentserver/logger"
|
||||
"github.com/foomo/contentserver/repo/mock"
|
||||
"github.com/foomo/contentserver/requests"
|
||||
"github.com/foomo/contentserver/server"
|
||||
@ -17,6 +16,15 @@ import (
|
||||
|
||||
const pathContentserver = "/contentserver"
|
||||
|
||||
var (
|
||||
testServerSocketAddr string
|
||||
testServerWebserverAddr string
|
||||
)
|
||||
|
||||
func init() {
|
||||
SetupLogging(true, "contentserver_client_test.log")
|
||||
}
|
||||
|
||||
func dump(t *testing.T, v interface{}) {
|
||||
jsonBytes, err := json.MarshalIndent(v, "", " ")
|
||||
if err != nil {
|
||||
@ -43,21 +51,24 @@ func getAvailableAddr() string {
|
||||
return "127.0.0.1:" + strconv.Itoa(getFreePort())
|
||||
}
|
||||
|
||||
var testServerSocketAddr string
|
||||
var testServerWebserverAddr string
|
||||
|
||||
func initTestServer(t testing.TB) (socketAddr, webserverAddr string) {
|
||||
socketAddr = getAvailableAddr()
|
||||
webserverAddr = getAvailableAddr()
|
||||
testServer, varDir := mock.GetMockData(t)
|
||||
log.SelectedLevel = log.LevelError
|
||||
go server.RunServerSocketAndWebServer(
|
||||
testServer.URL+"/repo-two-dimensions.json",
|
||||
socketAddr,
|
||||
webserverAddr,
|
||||
pathContentserver,
|
||||
varDir,
|
||||
)
|
||||
|
||||
go func() {
|
||||
err := server.RunServerSocketAndWebServer(
|
||||
testServer.URL+"/repo-two-dimensions.json",
|
||||
socketAddr,
|
||||
webserverAddr,
|
||||
pathContentserver,
|
||||
varDir,
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal("test server crashed: ", err)
|
||||
}
|
||||
}()
|
||||
|
||||
socketClient, errClient := NewClient(socketAddr, 1, time.Duration(time.Millisecond*100))
|
||||
if errClient != nil {
|
||||
panic(errClient)
|
||||
@ -69,6 +80,11 @@ func initTestServer(t testing.TB) (socketAddr, webserverAddr string) {
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
|
||||
if len(r) == 0 {
|
||||
t.Fatal("received empty JSON from GetRepo")
|
||||
}
|
||||
|
||||
if r["dimension_foo"].Nodes["id-a"].Data["baz"].(float64) == float64(1) {
|
||||
break
|
||||
}
|
||||
@ -139,10 +155,18 @@ func TestGetURIs(t *testing.T) {
|
||||
|
||||
func TestGetRepo(t *testing.T) {
|
||||
testWithClients(t, func(c *Client) {
|
||||
|
||||
time.Sleep(time.Millisecond * 100)
|
||||
|
||||
r, err := c.GetRepo()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
if len(r) == 0 {
|
||||
t.Fatal("received empty JSON from GetRepo")
|
||||
}
|
||||
|
||||
if r["dimension_foo"].Nodes["id-a"].Data["baz"].(float64) != float64(1) {
|
||||
t.Fatal("failed to drill deep for data")
|
||||
}
|
||||
@ -235,5 +259,4 @@ func benchmarkClientAndServerGetContent(b testing.TB, numGroups, numCalls int, c
|
||||
}
|
||||
// Wait for all HTTP fetches to complete.
|
||||
wg.Wait()
|
||||
return
|
||||
}
|
||||
|
||||
@ -6,8 +6,8 @@ import (
|
||||
)
|
||||
|
||||
type connectionPool struct {
|
||||
server string
|
||||
conn net.Conn
|
||||
server string
|
||||
// conn net.Conn
|
||||
chanConnGet chan chan net.Conn
|
||||
chanConnReturn chan connReturn
|
||||
chanDrainPool chan int
|
||||
@ -34,8 +34,11 @@ func (c *connectionPool) run(connectionPoolSize int, waitTimeout time.Duration)
|
||||
entryTime time.Time
|
||||
chanConn chan net.Conn
|
||||
}
|
||||
connectionPool := make(map[int]*poolEntry, connectionPoolSize)
|
||||
waitPool := map[int]*waitPoolEntry{}
|
||||
|
||||
var (
|
||||
connectionPool = make(map[int]*poolEntry, connectionPoolSize)
|
||||
waitPool = map[int]*waitPoolEntry{}
|
||||
)
|
||||
for i := 0; i < connectionPoolSize; i++ {
|
||||
connectionPool[i] = &poolEntry{
|
||||
conn: nil,
|
||||
@ -110,8 +113,10 @@ RunLoop:
|
||||
}
|
||||
}
|
||||
// waitpool cleanup
|
||||
waitPoolLoosers := []int{}
|
||||
now := time.Now()
|
||||
var (
|
||||
waitPoolLoosers = []int{}
|
||||
now = time.Now()
|
||||
)
|
||||
for i, waitPoolEntry := range waitPool {
|
||||
if now.Sub(waitPoolEntry.entryTime) > waitTimeout {
|
||||
waitPoolLoosers = append(waitPoolLoosers, i)
|
||||
|
||||
@ -2,7 +2,6 @@ package client
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
|
||||
@ -1,7 +1,6 @@
|
||||
package client
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
@ -11,8 +10,11 @@ import (
|
||||
|
||||
"github.com/foomo/contentserver/responses"
|
||||
"github.com/foomo/contentserver/server"
|
||||
jsoniter "github.com/json-iterator/go"
|
||||
)
|
||||
|
||||
var json = jsoniter.ConfigCompatibleWithStandardLibrary
|
||||
|
||||
type serverResponse struct {
|
||||
Reply interface{}
|
||||
}
|
||||
@ -62,8 +64,10 @@ func (c *socketTransport) call(handler server.Handler, request interface{}, resp
|
||||
jsonBytes = append([]byte(fmt.Sprintf("%s:%d", handler, len(jsonBytes))), jsonBytes...)
|
||||
|
||||
// send request
|
||||
written := 0
|
||||
l := len(jsonBytes)
|
||||
var (
|
||||
written = 0
|
||||
l = len(jsonBytes)
|
||||
)
|
||||
for written < l {
|
||||
n, err := conn.Write(jsonBytes[written:])
|
||||
if err != nil {
|
||||
@ -74,9 +78,11 @@ func (c *socketTransport) call(handler server.Handler, request interface{}, resp
|
||||
}
|
||||
|
||||
// read response
|
||||
responseBytes := []byte{}
|
||||
buf := make([]byte, 4096)
|
||||
responseLength := 0
|
||||
var (
|
||||
responseBytes = []byte{}
|
||||
buf = make([]byte, 4096)
|
||||
responseLength = 0
|
||||
)
|
||||
for {
|
||||
n, err := conn.Read(buf)
|
||||
if err != nil && err != io.EOF {
|
||||
@ -105,12 +111,15 @@ func (c *socketTransport) call(handler server.Handler, request interface{}, resp
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
// unmarshal response
|
||||
responseJSONErr := json.Unmarshal(responseBytes, &serverResponse{Reply: response})
|
||||
if responseJSONErr != nil {
|
||||
// is it an error ?
|
||||
remoteErr := responses.Error{}
|
||||
remoteErrJSONErr := json.Unmarshal(responseBytes, remoteErr)
|
||||
var (
|
||||
remoteErr = responses.Error{}
|
||||
remoteErrJSONErr = json.Unmarshal(responseBytes, &remoteErr)
|
||||
)
|
||||
if remoteErrJSONErr == nil {
|
||||
returnConn(remoteErrJSONErr)
|
||||
return remoteErr
|
||||
|
||||
@ -4,6 +4,6 @@ package content
|
||||
const (
|
||||
// Indent for json indentation
|
||||
Indent string = "\t"
|
||||
// PathSeparator seprator for paths in URIs
|
||||
// PathSeparator separator for paths in URIs
|
||||
PathSeparator = "/"
|
||||
)
|
||||
|
||||
@ -22,13 +22,13 @@ type RepoNode struct {
|
||||
// published from - to is going to be an array of fromTos
|
||||
}
|
||||
|
||||
// NewRepoNode constructor
|
||||
func NewRepoNode() *RepoNode {
|
||||
return &RepoNode{
|
||||
Data: make(map[string]interface{}),
|
||||
Nodes: make(map[string]*RepoNode),
|
||||
}
|
||||
}
|
||||
// // NewRepoNode constructor
|
||||
// func NewRepoNode() *RepoNode {
|
||||
// return &RepoNode{
|
||||
// Data: make(map[string]interface{}, 0), // set initial size to zero explicitely?
|
||||
// Nodes: make(map[string]*RepoNode, 0),
|
||||
// }
|
||||
// }
|
||||
|
||||
// WireParents helper method to reference from child to parent in a tree
|
||||
// recursively
|
||||
@ -52,15 +52,21 @@ func (node *RepoNode) InPath(path []*Item) bool {
|
||||
|
||||
// GetPath get a path for a repo node
|
||||
func (node *RepoNode) GetPath() []*Item {
|
||||
parentNode := node.parent
|
||||
pathLength := 0
|
||||
|
||||
var (
|
||||
parentNode = node.parent
|
||||
pathLength = 0
|
||||
)
|
||||
for parentNode != nil {
|
||||
parentNode = parentNode.parent
|
||||
pathLength++
|
||||
}
|
||||
parentNode = node.parent
|
||||
i := 0
|
||||
path := make([]*Item, pathLength)
|
||||
|
||||
var (
|
||||
i = 0
|
||||
path = make([]*Item, pathLength)
|
||||
)
|
||||
for parentNode != nil {
|
||||
path[i] = parentNode.ToItem([]string{})
|
||||
parentNode = parentNode.parent
|
||||
|
||||
111
contentserver.go
111
contentserver.go
@ -3,11 +3,18 @@ package main
|
||||
import (
|
||||
"flag"
|
||||
"fmt"
|
||||
"net/http"
|
||||
_ "net/http/pprof"
|
||||
"os"
|
||||
"strings"
|
||||
"runtime/debug"
|
||||
"time"
|
||||
|
||||
"github.com/foomo/contentserver/log"
|
||||
"github.com/apex/log"
|
||||
. "github.com/foomo/contentserver/logger"
|
||||
"github.com/foomo/contentserver/metrics"
|
||||
"github.com/foomo/contentserver/server"
|
||||
"github.com/foomo/contentserver/status"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
const (
|
||||
@ -16,30 +23,22 @@ const (
|
||||
logLevelWarning = "warning"
|
||||
logLevelRecord = "record"
|
||||
logLevelError = "error"
|
||||
|
||||
ServiceName = "Content Server"
|
||||
DefaultHealthzHandlerAddress = ":8080"
|
||||
DefaultPrometheusListener = "127.0.0.1:9111"
|
||||
)
|
||||
|
||||
var (
|
||||
uniqushPushVersion = "content-server 1.4.1"
|
||||
showVersionFlag = flag.Bool("version", false, "version info")
|
||||
address = flag.String("address", "", "address to bind socket server host:port")
|
||||
webserverAddress = flag.String("webserver-address", "", "address to bind web server host:port, when empty no webserver will be spawned")
|
||||
webserverPath = flag.String("webserver-path", "/contentserver", "path to export the webserver on - useful when behind a proxy")
|
||||
varDir = flag.String("var-dir", "/var/lib/contentserver", "where to put my data")
|
||||
logLevelOptions = []string{
|
||||
logLevelError,
|
||||
logLevelRecord,
|
||||
logLevelWarning,
|
||||
logLevelNotice,
|
||||
logLevelDebug,
|
||||
}
|
||||
logLevel = flag.String(
|
||||
"log-level",
|
||||
logLevelRecord,
|
||||
fmt.Sprintf(
|
||||
"one of %s",
|
||||
strings.Join(logLevelOptions, ", "),
|
||||
),
|
||||
)
|
||||
flagAddress = flag.String("address", "", "address to bind socket server host:port")
|
||||
flagWebserverAddress = flag.String("webserver-address", "", "address to bind web server host:port, when empty no webserver will be spawned")
|
||||
flagWebserverPath = flag.String("webserver-path", "/contentserver", "path to export the webserver on - useful when behind a proxy")
|
||||
flagVarDir = flag.String("var-dir", "/var/lib/contentserver", "where to put my data")
|
||||
|
||||
// debugging / profiling
|
||||
flagDebug = flag.Bool("debug", false, "toggle debug mode")
|
||||
flagFreeOSMem = flag.Int("free-os-mem", 0, "free OS mem every X minutes")
|
||||
flagHeapDump = flag.Int("heap-dump", 0, "dump heap every X minutes")
|
||||
)
|
||||
|
||||
func exitUsage(code int) {
|
||||
@ -50,27 +49,55 @@ func exitUsage(code int) {
|
||||
|
||||
func main() {
|
||||
flag.Parse()
|
||||
if *showVersionFlag {
|
||||
fmt.Printf("%v\n", uniqushPushVersion)
|
||||
return
|
||||
|
||||
SetupLogging(*flagDebug, "contentserver.log")
|
||||
|
||||
go func() {
|
||||
fmt.Println(http.ListenAndServe("localhost:6060", nil))
|
||||
}()
|
||||
|
||||
if *flagFreeOSMem > 0 {
|
||||
Log.Info("freeing OS memory every $interval minutes", zap.Int("interval", *flagFreeOSMem))
|
||||
go func() {
|
||||
for {
|
||||
select {
|
||||
case <-time.After(time.Duration(*flagFreeOSMem) * time.Minute):
|
||||
log.Info("FreeOSMemory")
|
||||
debug.FreeOSMemory()
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
if *flagHeapDump > 0 {
|
||||
Log.Info("dumping heap every $interval minutes", zap.Int("interval", *flagHeapDump))
|
||||
go func() {
|
||||
for {
|
||||
select {
|
||||
case <-time.After(time.Duration(*flagFreeOSMem) * time.Minute):
|
||||
log.Info("HeapDump")
|
||||
f, err := os.Create("heapdump")
|
||||
if err != nil {
|
||||
panic("failed to create heap dump file")
|
||||
}
|
||||
debug.WriteHeapDump(f.Fd())
|
||||
err = f.Close()
|
||||
if err != nil {
|
||||
panic("failed to create heap dump file")
|
||||
}
|
||||
}
|
||||
}
|
||||
}()
|
||||
}
|
||||
|
||||
if len(flag.Args()) == 1 {
|
||||
fmt.Println(*address, flag.Arg(0))
|
||||
level := log.LevelRecord
|
||||
switch *logLevel {
|
||||
case logLevelError:
|
||||
level = log.LevelError
|
||||
case logLevelRecord:
|
||||
level = log.LevelRecord
|
||||
case logLevelWarning:
|
||||
level = log.LevelWarning
|
||||
case logLevelNotice:
|
||||
level = log.LevelNotice
|
||||
case logLevelDebug:
|
||||
level = log.LevelDebug
|
||||
}
|
||||
log.SelectedLevel = level
|
||||
err := server.RunServerSocketAndWebServer(flag.Arg(0), *address, *webserverAddress, *webserverPath, *varDir)
|
||||
fmt.Println(*flagAddress, flag.Arg(0))
|
||||
|
||||
// kickoff metric handlers
|
||||
go metrics.RunPrometheusHandler(DefaultPrometheusListener)
|
||||
go status.RunHealthzHandlerListener(DefaultHealthzHandlerAddress, ServiceName)
|
||||
|
||||
err := server.RunServerSocketAndWebServer(flag.Arg(0), *flagAddress, *flagWebserverAddress, *flagWebserverPath, *flagVarDir)
|
||||
if err != nil {
|
||||
fmt.Println("exiting with error", err)
|
||||
os.Exit(1)
|
||||
|
||||
BIN
contentserver.graffle
Normal file
BIN
contentserver.graffle
Normal file
Binary file not shown.
14
go.mod
Normal file
14
go.mod
Normal file
@ -0,0 +1,14 @@
|
||||
module github.com/foomo/contentserver
|
||||
|
||||
require (
|
||||
github.com/apex/log v1.1.0
|
||||
github.com/davecgh/go-spew v1.1.1
|
||||
github.com/json-iterator/go v1.1.6
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.1 // indirect
|
||||
github.com/pkg/errors v0.8.1 // indirect
|
||||
github.com/prometheus/client_golang v0.9.2
|
||||
go.uber.org/atomic v1.4.0 // indirect
|
||||
go.uber.org/multierr v1.1.0 // indirect
|
||||
go.uber.org/zap v1.10.0
|
||||
)
|
||||
34
go.sum
Normal file
34
go.sum
Normal file
@ -0,0 +1,34 @@
|
||||
github.com/apex/log v1.1.0 h1:J5rld6WVFi6NxA6m8GJ1LJqu3+GiTFIt3mYv27gdQWI=
|
||||
github.com/apex/log v1.1.0/go.mod h1:yA770aXIDQrhVOIGurT/pVdfCpSq1GQV/auzMN5fzvY=
|
||||
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973 h1:xJ4a3vCFaGF/jqvzLMYoU8P317H5OQ+Via4RmuPwCS0=
|
||||
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/json-iterator/go v1.1.6 h1:MrUvLMLTMxbqFJ9kzlvat/rYZqZnW3u4wkLzWTaFwKs=
|
||||
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg=
|
||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
|
||||
github.com/modern-go/reflect2 v1.0.1 h1:9f412s+6RmYXLWZSEzVVgPGK7C2PphHj5RJrvfx9AWI=
|
||||
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
|
||||
github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740=
|
||||
github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM=
|
||||
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8=
|
||||
github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
|
||||
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jOqq0gIVUe6Yk0/QMZ640k6NvkxcBf+8=
|
||||
github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
|
||||
github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE=
|
||||
github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
|
||||
go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU=
|
||||
go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
|
||||
go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI=
|
||||
go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0=
|
||||
go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM=
|
||||
go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
|
||||
golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
327
graphics/Horizontal Update.svg
Normal file
327
graphics/Horizontal Update.svg
Normal file
@ -0,0 +1,327 @@
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
|
||||
<svg xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns="http://www.w3.org/2000/svg" xmlns:xl="http://www.w3.org/1999/xlink" version="1.1" viewBox="133 101 1838 2500" width="1838" height="2500">
|
||||
<defs>
|
||||
<font-face font-family="Helvetica Neue" font-size="16" panose-1="2 0 5 3 0 0 0 2 0 4" units-per-em="1000" underline-position="-100" underline-thickness="50" slope="0" x-height="517" cap-height="714" ascent="951.9958" descent="-212.99744" font-weight="400">
|
||||
<font-face-src>
|
||||
<font-face-name name="HelveticaNeue"/>
|
||||
</font-face-src>
|
||||
</font-face>
|
||||
<marker orient="auto" overflow="visible" markerUnits="strokeWidth" id="FilledArrow_Marker" stroke-linejoin="miter" stroke-miterlimit="10" viewBox="-1 -3 7 6" markerWidth="7" markerHeight="6" color="black">
|
||||
<g>
|
||||
<path d="M 4.8 0 L 0 -1.8 L 0 1.8 Z" fill="currentColor" stroke="currentColor" stroke-width="1"/>
|
||||
</g>
|
||||
</marker>
|
||||
<font-face font-family="Futura" font-size="16" panose-1="2 11 6 2 2 2 4 2 3 3" units-per-em="1000" underline-position="-97.65625" underline-thickness="78.125" slope="0" x-height="482.4219" cap-height="761.2305" ascent="1038.5742" descent="-259.76562" font-weight="500">
|
||||
<font-face-src>
|
||||
<font-face-name name="Futura-Medium"/>
|
||||
</font-face-src>
|
||||
</font-face>
|
||||
<font-face font-family="Helvetica Neue" font-size="16" panose-1="2 0 8 3 0 0 0 9 0 4" units-per-em="1000" underline-position="-100" underline-thickness="50" slope="0" x-height="517" cap-height="714" ascent="975.0061" descent="-216.99524" font-weight="700">
|
||||
<font-face-src>
|
||||
<font-face-name name="HelveticaNeue-Bold"/>
|
||||
</font-face-src>
|
||||
</font-face>
|
||||
<marker orient="auto" overflow="visible" markerUnits="strokeWidth" id="FilledArrow_Marker_2" stroke-linejoin="miter" stroke-miterlimit="10" viewBox="-1 -3 7 6" markerWidth="7" markerHeight="6" color="#ff2600">
|
||||
<g>
|
||||
<path d="M 4.8 0 L 0 -1.8 L 0 1.8 Z" fill="currentColor" stroke="currentColor" stroke-width="1"/>
|
||||
</g>
|
||||
</marker>
|
||||
<marker orient="auto" overflow="visible" markerUnits="strokeWidth" id="FilledArrow_Marker_3" stroke-linejoin="miter" stroke-miterlimit="10" viewBox="-1 -3 7 6" markerWidth="7" markerHeight="6" color="#ff2600">
|
||||
<g>
|
||||
<path d="M 4.8 0 L 0 -1.8 L 0 1.8 Z" fill="currentColor" stroke="currentColor" stroke-width="1"/>
|
||||
</g>
|
||||
</marker>
|
||||
<font-face font-family="Futura" font-size="80" panose-1="2 11 6 2 2 2 4 2 3 3" units-per-em="1000" underline-position="-97.65625" underline-thickness="78.125" slope="0" x-height="482.4219" cap-height="761.2305" ascent="1038.5742" descent="-259.76562" font-weight="500">
|
||||
<font-face-src>
|
||||
<font-face-name name="Futura-Medium"/>
|
||||
</font-face-src>
|
||||
</font-face>
|
||||
</defs>
|
||||
<metadata> Produced by OmniGraffle 7.10.2
|
||||
<dc:date>2019-05-29 10:21:17 +0000</dc:date>
|
||||
</metadata>
|
||||
<g id="Horizontal_Update" stroke-opacity="1" fill="none" stroke="none" stroke-dasharray="none" fill-opacity="1">
|
||||
<title>Horizontal Update</title>
|
||||
<rect fill="white" x="133" y="101" width="1838" height="2500"/>
|
||||
<g id="Horizontal_Update: Layer 1">
|
||||
<title>Layer 1</title>
|
||||
<g id="Graphic_4">
|
||||
<rect x="526" y="282" width="539" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(531 303.5)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="209.828" y="15">*Repo.Update()</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_5">
|
||||
<rect x="526" y="425" width="539" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(531 446.5)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="200.644" y="15">*Repo.tryUpdate()</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_7">
|
||||
<rect x="134" y="589" width="298" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(139 610.5)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="23.344" y="15">if updateErr != errUpdateRejected</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Line_8">
|
||||
<line x1="693.9375" y1="489" x2="396.84877" y2="584.0684" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Graphic_12">
|
||||
<rect x="134" y="770" width="298" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(139 791.5)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="43.848" y="15">*Repo.tryToRestoreCurrent()</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Line_13">
|
||||
<line x1="283" y1="653" x2="283" y2="756.1" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Line_15">
|
||||
<line x1="795.5" y1="346" x2="795.5" y2="411.1" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Graphic_16">
|
||||
<rect x="1137" y="589" width="473" height="63" fill="white"/>
|
||||
<rect x="1137" y="589" width="473" height="63" stroke="#ff2600" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(1142 610.5)" fill="#ff2600">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="#ff2600" x="162.756" y="15">*Repo.history.lock()</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_18">
|
||||
<rect x="828" y="589" width="222" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(833 610.5)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="15.032" y="15">return errUpdateRejected</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Line_20">
|
||||
<line x1="823.9375" y1="489" x2="902.0678" y2="578.29176" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Line_21">
|
||||
<line x1="910.0427" y1="489" x2="1246.5472" y2="584.4788" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Graphic_22">
|
||||
<rect x="824.0671" y="519.599" width="81.28906" height="32" fill="white"/>
|
||||
<text transform="translate(829.0671 524.599)" fill="black">
|
||||
<tspan font-family="Futura" font-size="16" font-weight="500" fill="black" x="0" y="17">queue full</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_23">
|
||||
<rect x="1033.0312" y="521.1068" width="93.11719" height="32" fill="white"/>
|
||||
<text transform="translate(1038.0312 526.1068)" fill="black">
|
||||
<tspan font-family="Futura" font-size="16" font-weight="500" fill="black" x="2.46875" y="17">queue free </tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_24">
|
||||
<rect x="134" y="966" width="298" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(139 987.5)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="59.832" y="15">return updateResponse</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Line_25">
|
||||
<line x1="283" y1="834" x2="283" y2="952.1" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Graphic_28">
|
||||
<rect x="982.5" y="924" width="782" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(987.5 945.5)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="700" fill="black" x="299.496" y="16">*Repo.updateRoutine()</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_29">
|
||||
<rect x="982.5" y="987" width="782" height="1072" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Graphic_31">
|
||||
<rect x="1013.5" y="1023" width="359.1709" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(1018.5 1044.5)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="13.305457" y="15">resChan <- *Repo.updateInProgressChannel:</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_32">
|
||||
<rect x="1062.5" y="1110" width="359.1709" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(1067.5 1131.5)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="121.24146" y="15">*Repo.update()</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_33">
|
||||
<rect x="1124.9145" y="1197" width="359.1709" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(1129.9145 1218.5)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="134.88146" y="15">*Repo.get()</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_34">
|
||||
<rect x="1195.5" y="1284" width="359.1709" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(1200.5 1305.5)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="67.75346" y="15">*Repo.loadNodesFromJSON()</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_35">
|
||||
<rect x="1259.5" y="1371" width="359.1709" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(1264.5 1392.5)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="107.47346" y="15">*Repo.loadNodes()</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Line_36">
|
||||
<line x1="1373.5" y1="826" x2="1373.5" y2="910.1" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-dasharray="8.0,8.0" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Graphic_37">
|
||||
<rect x="1302.5" y="1458" width="409.1709" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(1307.5 1461.828)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="0" y="15">for dimension, newNode := range nodes {</tspan>
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="0" y="33.448"> *Repo.updateDimension(dimension, newNode)</tspan>
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="0" y="51.895996">}</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_38">
|
||||
<rect x="1352.5" y="1549" width="359.1709" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(1357.5 1570.5)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="700" fill="black" x="47.369457" y="16">*Repo.dimensionUpdateRoutine()</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_39">
|
||||
<rect x="1352.5" y="1612" width="359.1709" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(1357.5 1633.5)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="79.44946" y="15">*Repo._updateDimension()</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_40">
|
||||
<rect x="1352.5" y="1675" width="359.1709" height="343" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Graphic_41">
|
||||
<rect x="1373.5" y="1689" width="298" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(1378.5 1710.5)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="60.152" y="15">newNode.WireParents()</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_42">
|
||||
<rect x="1373.5" y="1772.5" width="298" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(1378.5 1794)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="90.072" y="15">buildDirectory()</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_43">
|
||||
<rect x="1373.5" y="1856" width="298" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(1378.5 1877.5)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="99.872" y="15">wireAliases()</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_44">
|
||||
<rect x="1373.5" y="1939.5" width="298" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(1378.5 1961)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="20.8" y="15">dimensionUpdateDoneChan <- err</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_45">
|
||||
<rect x="982.5" y="2186" width="359.1709" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(987.5 2207.5)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="71.76146" y="15">*Repo.history.add(jsonBytes)</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Line_46">
|
||||
<line x1="1210.0304" y1="2060" x2="1175.7356" y2="2172.6591" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-dasharray="8.0,8.0" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Graphic_48">
|
||||
<rect x="982.5" y="2537" width="359.1709" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(987.5 2558.5)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="62.79346" y="15">resultChan <- updateResponse</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Line_47">
|
||||
<line x1="1162.0855" y1="2488" x2="1162.0855" y2="2523.1" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Line_49">
|
||||
<line x1="1265.4012" y1="1174" x2="1273.6646" y2="1185.5183" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Line_50">
|
||||
<line x1="1330.8681" y1="1261" x2="1340.5898" y2="1272.9824" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Line_51">
|
||||
<line x1="1398.9935" y1="1348" x2="1407.5333" y2="1359.6088" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Line_52">
|
||||
<line x1="1464.4878" y1="1435" x2="1473.739" y2="1446.8363" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Line_53">
|
||||
<line x1="1522.5" y1="1753" x2="1522.5" y2="1758.6" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Line_54">
|
||||
<line x1="1522.5" y1="1836.5" x2="1522.5" y2="1842.1" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Line_55">
|
||||
<line x1="1522.5" y1="1920" x2="1522.5" y2="1925.6" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Graphic_57">
|
||||
<rect x="485.6393" y="523.2829" width="102.32812" height="32" fill="white"/>
|
||||
<text transform="translate(490.6393 528.2829)" fill="black">
|
||||
<tspan font-family="Futura" font-size="16" font-weight="500" fill="black" x="0" y="17">update error</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_60">
|
||||
<rect x="519" y="589" width="222" height="63" fill="white"/>
|
||||
<rect x="519" y="589" width="222" height="63" stroke="#ff2600" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(524 610.5)" fill="#ff2600">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="#ff2600" x="15.032" y="15">return errUpdateRejected</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Line_59">
|
||||
<line x1="762.7027" y1="489" x2="671.9604" y2="578.91994" marker-end="url(#FilledArrow_Marker_2)" stroke="#ff2600" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Graphic_58">
|
||||
<rect x="663.3299" y="519.599" width="104.69531" height="32" fill="white"/>
|
||||
<text transform="translate(668.3299 524.599)" fill="#ff2600">
|
||||
<tspan font-family="Futura" font-size="16" font-weight="500" fill="#ff2600" x="0" y="17">lockfile exists</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_61">
|
||||
<rect x="1137" y="762" width="473" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(1142 783.5)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="8.42" y="15">*Repo.updateInProgressChan <- make(chan updateResponse)</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Line_62">
|
||||
<line x1="1373.5" y1="653" x2="1373.5" y2="748.1" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-dasharray="8.0,8.0" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Graphic_63">
|
||||
<rect x="982.5" y="2305" width="359.1709" height="63" fill="white"/>
|
||||
<rect x="982.5" y="2305" width="359.1709" height="63" stroke="#ff2600" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(987.5 2326.5)" fill="#ff2600">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="#ff2600" x="96.94546" y="15">*Repo.history.unlock()</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Line_64">
|
||||
<line x1="1162.0855" y1="2250" x2="1162.0855" y2="2291.1" marker-end="url(#FilledArrow_Marker_2)" stroke="#ff2600" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Graphic_66">
|
||||
<rect x="982.5" y="2424" width="359.1709" height="63" fill="white"/>
|
||||
<rect x="982.5" y="2424" width="359.1709" height="63" stroke="#ff2600" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(987.5 2445.5)" fill="#ff2600">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="#ff2600" x="58.27346" y="15">*Repo.history.</tspan>
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="#ff2600" y="15">broadcastUpdate()</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Line_67">
|
||||
<line x1="1162.0855" y1="2369" x2="1162.0855" y2="2410.1" marker-end="url(#FilledArrow_Marker_3)" stroke="#ff2600" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Graphic_68">
|
||||
<text transform="translate(139 106.61719)" fill="black">
|
||||
<tspan font-family="Futura" font-size="80" font-weight="500" fill="black" x="0" y="83">Contentserver Horizontal Scaling: Update Flow</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_69">
|
||||
<path d="M 1341.6709 2305 L 1700.8418 2305 L 1700.8418 2368 L 1341.6709 2368 Z" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-dasharray="2.0,8.0" stroke-width="2"/>
|
||||
<text transform="translate(1346.6709 2326.5)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="117.84146" y="15">Remove lockfile</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_70">
|
||||
<path d="M 1341.6709 2424 L 1700.8418 2424 L 1700.8418 2487 L 1341.6709 2487 Z" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-dasharray="2.0,8.0" stroke-width="2"/>
|
||||
<text transform="translate(1346.6709 2445.5)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="76.04946" y="15">Broadcast update via NATS</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_71">
|
||||
<path d="M 1610 589 L 1969.171 589 L 1969.171 652 L 1610 652 Z" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-dasharray="2.0,8.0" stroke-width="2"/>
|
||||
<text transform="translate(1615 610.5)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="123.62546" y="15">Create lockfile</tspan>
|
||||
</text>
|
||||
</g>
|
||||
</g>
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 22 KiB |
391
graphics/Horizontal.svg
Normal file
391
graphics/Horizontal.svg
Normal file
@ -0,0 +1,391 @@
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
|
||||
<svg xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns="http://www.w3.org/2000/svg" xmlns:xl="http://www.w3.org/1999/xlink" version="1.1" viewBox="-263 -474 2433 1740" width="2433" height="1740">
|
||||
<defs>
|
||||
<font-face font-family="Helvetica Neue" font-size="25" panose-1="2 0 8 3 0 0 0 9 0 4" units-per-em="1000" underline-position="-100" underline-thickness="50" slope="0" x-height="517" cap-height="714" ascent="975.0061" descent="-216.99524" font-weight="700">
|
||||
<font-face-src>
|
||||
<font-face-name name="HelveticaNeue-Bold"/>
|
||||
</font-face-src>
|
||||
</font-face>
|
||||
<font-face font-family="Helvetica Neue" font-size="16" panose-1="2 0 5 3 0 0 0 2 0 4" units-per-em="1000" underline-position="-100" underline-thickness="50" slope="0" x-height="517" cap-height="714" ascent="951.9958" descent="-212.99744" font-weight="400">
|
||||
<font-face-src>
|
||||
<font-face-name name="HelveticaNeue"/>
|
||||
</font-face-src>
|
||||
</font-face>
|
||||
<font-face font-family="Futura" font-size="16" panose-1="2 11 6 2 2 2 4 2 3 3" units-per-em="1000" underline-position="-97.65625" underline-thickness="78.125" slope="0" x-height="482.4219" cap-height="761.2305" ascent="1038.5742" descent="-259.76562" font-weight="500">
|
||||
<font-face-src>
|
||||
<font-face-name name="Futura-Medium"/>
|
||||
</font-face-src>
|
||||
</font-face>
|
||||
<marker orient="auto" overflow="visible" markerUnits="strokeWidth" id="FilledArrow_Marker" stroke-linejoin="miter" stroke-miterlimit="10" viewBox="-1 -3 7 6" markerWidth="7" markerHeight="6" color="black">
|
||||
<g>
|
||||
<path d="M 4.8 0 L 0 -1.8 L 0 1.8 Z" fill="currentColor" stroke="currentColor" stroke-width="1"/>
|
||||
</g>
|
||||
</marker>
|
||||
</defs>
|
||||
<metadata> Produced by OmniGraffle 7.10.2
|
||||
<dc:date>2019-05-29 10:21:17 +0000</dc:date>
|
||||
</metadata>
|
||||
<g id="Horizontal" stroke-opacity="1" fill="none" stroke="none" stroke-dasharray="none" fill-opacity="1">
|
||||
<title>Horizontal</title>
|
||||
<rect fill="white" x="-263" y="-474" width="2433" height="1740"/>
|
||||
<g id="Horizontal: Layer 1">
|
||||
<title>Layer 1</title>
|
||||
<g id="Graphic_35">
|
||||
<rect x="-65" y="991" width="341" height="273" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Graphic_32">
|
||||
<rect x="775" y="984" width="554" height="231" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Graphic_20">
|
||||
<rect x="-65" y="195" width="956" height="619" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Graphic_36">
|
||||
<rect x="-262" y="342" width="384" height="408" fill="white"/>
|
||||
<rect x="-262" y="342" width="384" height="408" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Graphic_21">
|
||||
<rect x="465" y="361.8998" width="384" height="398.1002" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Graphic_2">
|
||||
<rect x="-65" y="121.5" width="956" height="73.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(-60 142.75)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="25" font-weight="700" fill="black" x="390.1" y="24">contentserver</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_3">
|
||||
<rect x="140.5" y="425.4524" width="306" height="84" fill="white"/>
|
||||
<rect x="140.5" y="425.4524" width="306" height="84" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(145.5 457.4524)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="107.864" y="15">Web server</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_4">
|
||||
<rect x="140.5" y="567.5476" width="306" height="84" fill="white"/>
|
||||
<rect x="140.5" y="567.5476" width="306" height="84" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(145.5 599.5476)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="81.04" y="15">TCP Socket server</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_5">
|
||||
<rect x="886" y="-368.5" width="306" height="84" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(891 -336.5)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="87.992" y="15">CONTENT.JSON</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_6">
|
||||
<rect x="847" y="-473" width="384" height="67" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(852 -455)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="25" font-weight="700" fill="black" x="21.475" y="24">Content Source Webservice</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_7">
|
||||
<rect x="465" y="285" width="384" height="76.89978" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(470 307.9499)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="25" font-weight="700" fill="black" x="155.525" y="24">Repo</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_9">
|
||||
<rect x="670" y="396.94934" width="156" height="87.88546" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(675 430.89207)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="34.48" y="15">RepoNode</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_11">
|
||||
<rect x="495" y="396.94934" width="156" height="87.88546" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(500 430.89207)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="35.208" y="15">Dimension</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_22">
|
||||
<rect x="670" y="509.94493" width="156" height="87.88546" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(675 543.88767)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="34.48" y="15">RepoNode</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_23">
|
||||
<rect x="495" y="509.94493" width="156" height="87.88546" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(500 543.88767)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="35.208" y="15">Dimension</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_24">
|
||||
<rect x="847" y="-406" width="384" height="162.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Graphic_25">
|
||||
<rect x="-223" y="446.06663" width="306" height="54.961924" fill="white"/>
|
||||
<rect x="-223" y="446.06663" width="306" height="54.961924" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(-218 463.5476)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="106.512" y="15">GetContent</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_26">
|
||||
<rect x="-223" y="379" width="306" height="54.961924" fill="white"/>
|
||||
<rect x="-223" y="379" width="306" height="54.961924" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(-218 396.48096)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="117.784" y="15">GetURIs</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_27">
|
||||
<rect x="-223" y="513.13327" width="306" height="54.961924" fill="white"/>
|
||||
<rect x="-223" y="513.13327" width="306" height="54.961924" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(-218 530.6142)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="111.704" y="15">GetNodes</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_28">
|
||||
<rect x="-223" y="580.1999" width="306" height="54.961924" fill="white"/>
|
||||
<rect x="-223" y="580.1999" width="306" height="54.961924" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(-218 597.68086)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="121.624" y="15">Update</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_29">
|
||||
<rect x="-223" y="650.5381" width="306" height="54.961924" fill="white"/>
|
||||
<rect x="-223" y="650.5381" width="306" height="54.961924" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(-218 668.019)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="116" y="15">GetRepo</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_30">
|
||||
<rect x="831.2656" y="1071" width="441.46875" height="46" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(836.2656 1084)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="102.52637" y="15">contentserver-repo-current.json</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_31">
|
||||
<rect x="775" y="917" width="554" height="67" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(780 935)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="25" font-weight="700" fill="black" x="115.975" y="24">Var Directory (aka History)</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_34">
|
||||
<rect x="-65" y="924" width="341" height="67" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(-60 942)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="25" font-weight="700" fill="black" x="36.7875" y="24">Logfile (JSON or TXT)</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_37">
|
||||
<rect x="-262" y="275" width="384" height="67" fill="white"/>
|
||||
<rect x="-262" y="275" width="384" height="67" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(-257 293)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="25" font-weight="700" fill="black" x="166.4125" y="24">API</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_39">
|
||||
<rect x="670" y="623.10984" width="156" height="87.88546" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(675 657.0526)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="65" y="15">…</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_40">
|
||||
<rect x="495" y="623.10984" width="156" height="87.88546" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(500 657.0526)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="65" y="15">…</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_43">
|
||||
<rect x="831.2656" y="1011.5" width="441.46875" height="46" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(836.2656 1024.5)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="97.70237" y="15">contentserver-repo-[Timestamp].json</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_45">
|
||||
<rect x="-42" y="1011.5" width="301" height="46" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(-37 1023.5)" fill="black">
|
||||
<tspan font-family="Futura" font-size="16" font-weight="500" fill="black" x="111.19531" y="17">Log entry</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_44">
|
||||
<rect x="-42" y="1071" width="301" height="46" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(-37 1083)" fill="black">
|
||||
<tspan font-family="Futura" font-size="16" font-weight="500" fill="black" x="111.19531" y="17">Log entry</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_46">
|
||||
<rect x="-42" y="1130.5" width="301" height="46" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(-37 1142.5)" fill="black">
|
||||
<tspan font-family="Futura" font-size="16" font-weight="500" fill="black" x="111.19531" y="17">Log entry</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_47">
|
||||
<rect x="-42" y="1190" width="301" height="46" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(-37 1202)" fill="black">
|
||||
<tspan font-family="Futura" font-size="16" font-weight="500" fill="black" x="111.19531" y="17">Log entry</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Line_48">
|
||||
<line x1="461.9265" y1="120.5" x2="922.1852" y2="-234.61976" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Line_56">
|
||||
<line x1="202.23013" y1="815" x2="136.16397" y2="912.3267" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Line_57">
|
||||
<line x1="857.8643" y1="815" x2="991.9924" y2="908.6168" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Graphic_59">
|
||||
<rect x="1213" y="187" width="956" height="619" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Graphic_60">
|
||||
<rect x="1016" y="334" width="384" height="408" fill="white"/>
|
||||
<rect x="1016" y="334" width="384" height="408" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Graphic_61">
|
||||
<rect x="1743" y="353.8998" width="384" height="398.1002" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Graphic_62">
|
||||
<rect x="1213" y="113.5" width="956" height="73.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(1218 134.75)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="25" font-weight="700" fill="black" x="390.1" y="24">contentserver</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_63">
|
||||
<rect x="1418.5" y="417.4524" width="306" height="84" fill="white"/>
|
||||
<rect x="1418.5" y="417.4524" width="306" height="84" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(1423.5 449.4524)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="107.864" y="15">Web server</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_64">
|
||||
<rect x="1418.5" y="559.5476" width="306" height="84" fill="white"/>
|
||||
<rect x="1418.5" y="559.5476" width="306" height="84" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(1423.5 591.5476)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="81.04" y="15">TCP Socket server</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_65">
|
||||
<rect x="1743" y="277" width="384" height="76.89978" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(1748 299.9499)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="25" font-weight="700" fill="black" x="155.525" y="24">Repo</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_66">
|
||||
<rect x="1948" y="388.94934" width="156" height="87.88546" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(1953 422.89207)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="34.48" y="15">RepoNode</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_67">
|
||||
<rect x="1773" y="388.94934" width="156" height="87.88546" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(1778 422.89207)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="35.208" y="15">Dimension</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_68">
|
||||
<rect x="1948" y="501.94493" width="156" height="87.88546" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(1953 535.88767)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="34.48" y="15">RepoNode</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_69">
|
||||
<rect x="1773" y="501.94493" width="156" height="87.88546" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(1778 535.88767)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="35.208" y="15">Dimension</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_70">
|
||||
<rect x="1055" y="438.06663" width="306" height="54.961924" fill="white"/>
|
||||
<rect x="1055" y="438.06663" width="306" height="54.961924" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(1060 455.5476)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="106.512" y="15">GetContent</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_71">
|
||||
<rect x="1055" y="371" width="306" height="54.961924" fill="white"/>
|
||||
<rect x="1055" y="371" width="306" height="54.961924" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(1060 388.48096)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="117.784" y="15">GetURIs</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_72">
|
||||
<rect x="1055" y="505.13327" width="306" height="54.961924" fill="white"/>
|
||||
<rect x="1055" y="505.13327" width="306" height="54.961924" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(1060 522.6142)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="111.704" y="15">GetNodes</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_73">
|
||||
<rect x="1055" y="572.1999" width="306" height="54.961924" fill="white"/>
|
||||
<rect x="1055" y="572.1999" width="306" height="54.961924" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(1060 589.68086)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="121.624" y="15">Update</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_74">
|
||||
<rect x="1055" y="642.5381" width="306" height="54.961924" fill="white"/>
|
||||
<rect x="1055" y="642.5381" width="306" height="54.961924" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(1060 660.019)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="116" y="15">GetRepo</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_75">
|
||||
<rect x="1016" y="267" width="384" height="67" fill="white"/>
|
||||
<rect x="1016" y="267" width="384" height="67" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(1021 285)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="25" font-weight="700" fill="black" x="166.4125" y="24">API</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_76">
|
||||
<rect x="1948" y="615.10984" width="156" height="87.88546" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(1953 649.0526)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="65" y="15">…</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_77">
|
||||
<rect x="1773" y="615.10984" width="156" height="87.88546" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(1778 649.0526)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="65" y="15">…</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Line_78">
|
||||
<line x1="1903.2618" y1="807" x2="1967.6353" y2="901.1666" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Line_79">
|
||||
<line x1="1253.9747" y1="807" x2="1111.0744" y2="908.5285" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Line_84">
|
||||
<line x1="1639.1832" y1="112.5" x2="1162.3254" y2="-234.90403" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Graphic_85">
|
||||
<rect x="1828" y="979.816" width="341" height="273" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Graphic_86">
|
||||
<rect x="1828" y="912.816" width="341" height="67" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(1833 930.816)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="25" font-weight="700" fill="black" x="36.7875" y="24">Logfile (JSON or TXT)</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_87">
|
||||
<rect x="1851" y="1000.316" width="301" height="46" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(1856 1012.316)" fill="black">
|
||||
<tspan font-family="Futura" font-size="16" font-weight="500" fill="black" x="111.19531" y="17">Log entry</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_88">
|
||||
<rect x="1851" y="1059.816" width="301" height="46" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(1856 1071.816)" fill="black">
|
||||
<tspan font-family="Futura" font-size="16" font-weight="500" fill="black" x="111.19531" y="17">Log entry</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_89">
|
||||
<rect x="1851" y="1119.316" width="301" height="46" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(1856 1131.316)" fill="black">
|
||||
<tspan font-family="Futura" font-size="16" font-weight="500" fill="black" x="111.19531" y="17">Log entry</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_90">
|
||||
<rect x="1851" y="1178.816" width="301" height="46" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(1856 1190.816)" fill="black">
|
||||
<tspan font-family="Futura" font-size="16" font-weight="500" fill="black" x="111.19531" y="17">Log entry</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_91">
|
||||
<rect x="831.2656" y="1137" width="441.46875" height="46" fill="#ccc"/>
|
||||
<rect x="831.2656" y="1137" width="441.46875" height="46" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(836.2656 1150)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="135.42237" y="15">updateInProgress.lock</tspan>
|
||||
</text>
|
||||
</g>
|
||||
</g>
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 25 KiB |
250
graphics/Overview.svg
Normal file
250
graphics/Overview.svg
Normal file
@ -0,0 +1,250 @@
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
|
||||
<svg xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns="http://www.w3.org/2000/svg" xmlns:xl="http://www.w3.org/1999/xlink" version="1.1" viewBox="-549 -177 1441 1431" width="1441" height="1431">
|
||||
<defs>
|
||||
<font-face font-family="Helvetica Neue" font-size="16" panose-1="2 0 8 3 0 0 0 9 0 4" units-per-em="1000" underline-position="-100" underline-thickness="50" slope="0" x-height="517" cap-height="714" ascent="975.0061" descent="-216.99524" font-weight="700">
|
||||
<font-face-src>
|
||||
<font-face-name name="HelveticaNeue-Bold"/>
|
||||
</font-face-src>
|
||||
</font-face>
|
||||
<font-face font-family="Helvetica Neue" font-size="16" panose-1="2 0 5 3 0 0 0 2 0 4" units-per-em="1000" underline-position="-100" underline-thickness="50" slope="0" x-height="517" cap-height="714" ascent="951.9958" descent="-212.99744" font-weight="400">
|
||||
<font-face-src>
|
||||
<font-face-name name="HelveticaNeue"/>
|
||||
</font-face-src>
|
||||
</font-face>
|
||||
<font-face font-family="Futura" font-size="16" panose-1="2 11 6 2 2 2 4 2 3 3" units-per-em="1000" underline-position="-97.65625" underline-thickness="78.125" slope="0" x-height="482.4219" cap-height="761.2305" ascent="1038.5742" descent="-259.76562" font-weight="500">
|
||||
<font-face-src>
|
||||
<font-face-name name="Futura-Medium"/>
|
||||
</font-face-src>
|
||||
</font-face>
|
||||
<marker orient="auto" overflow="visible" markerUnits="strokeWidth" id="FilledArrow_Marker" stroke-linejoin="miter" stroke-miterlimit="10" viewBox="-1 -3 7 6" markerWidth="7" markerHeight="6" color="black">
|
||||
<g>
|
||||
<path d="M 4.8 0 L 0 -1.8 L 0 1.8 Z" fill="currentColor" stroke="currentColor" stroke-width="1"/>
|
||||
</g>
|
||||
</marker>
|
||||
</defs>
|
||||
<metadata> Produced by OmniGraffle 7.10.2
|
||||
<dc:date>2019-05-29 10:21:17 +0000</dc:date>
|
||||
</metadata>
|
||||
<g id="Overview" stroke-opacity="1" fill="none" stroke="none" stroke-dasharray="none" fill-opacity="1">
|
||||
<title>Overview</title>
|
||||
<rect fill="white" x="-549" y="-177" width="1441" height="1431"/>
|
||||
<g id="Overview: Layer 1">
|
||||
<title>Layer 1</title>
|
||||
<g id="Graphic_35">
|
||||
<rect x="546" y="979.7753" width="341" height="273" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Graphic_32">
|
||||
<rect x="-69" y="979.816" width="554" height="185.45933" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Graphic_20">
|
||||
<rect x="-65" y="195" width="956" height="619" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Graphic_36">
|
||||
<rect x="-262" y="342" width="384" height="408" fill="white"/>
|
||||
<rect x="-262" y="342" width="384" height="408" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Graphic_21">
|
||||
<rect x="465" y="361.8998" width="384" height="398.1002" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Graphic_2">
|
||||
<rect x="-65" y="121.5" width="956" height="73.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(-60 148.25)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="700" fill="black" x="419.944" y="16">contentserver</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_3">
|
||||
<rect x="140.5" y="425.4524" width="306" height="84" fill="white"/>
|
||||
<rect x="140.5" y="425.4524" width="306" height="84" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(145.5 457.4524)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="107.864" y="15">Web server</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_4">
|
||||
<rect x="140.5" y="567.5476" width="306" height="84" fill="white"/>
|
||||
<rect x="140.5" y="567.5476" width="306" height="84" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(145.5 599.5476)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="81.04" y="15">TCP Socket server</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_5">
|
||||
<rect x="260" y="-71.5" width="306" height="84" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(265 -39.5)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="87.992" y="15">CONTENT.JSON</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_6">
|
||||
<rect x="221" y="-176" width="384" height="67" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(226 -152.5)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="700" fill="black" x="81.064" y="16">Content Source Webservice</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_7">
|
||||
<rect x="465" y="285" width="384" height="76.89978" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(470 313.4499)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="700" fill="black" x="166.856" y="16">Repo</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_9">
|
||||
<rect x="670" y="396.94934" width="156" height="87.88546" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(675 430.89207)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="34.48" y="15">RepoNode</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_11">
|
||||
<rect x="495" y="396.94934" width="156" height="87.88546" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(500 430.89207)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="35.208" y="15">Dimension</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_22">
|
||||
<rect x="670" y="509.94493" width="156" height="87.88546" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(675 543.88767)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="34.48" y="15">RepoNode</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_23">
|
||||
<rect x="495" y="509.94493" width="156" height="87.88546" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(500 543.88767)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="35.208" y="15">Dimension</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_24">
|
||||
<rect x="221" y="-109" width="384" height="162.5" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Graphic_25">
|
||||
<rect x="-223" y="446.06663" width="306" height="54.961924" fill="white"/>
|
||||
<rect x="-223" y="446.06663" width="306" height="54.961924" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(-218 463.5476)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="106.512" y="15">GetContent</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_26">
|
||||
<rect x="-223" y="379" width="306" height="54.961924" fill="white"/>
|
||||
<rect x="-223" y="379" width="306" height="54.961924" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(-218 396.48096)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="117.784" y="15">GetURIs</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_27">
|
||||
<rect x="-223" y="513.13327" width="306" height="54.961924" fill="white"/>
|
||||
<rect x="-223" y="513.13327" width="306" height="54.961924" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(-218 530.6142)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="111.704" y="15">GetNodes</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_28">
|
||||
<rect x="-223" y="580.1999" width="306" height="54.961924" fill="white"/>
|
||||
<rect x="-223" y="580.1999" width="306" height="54.961924" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(-218 597.68086)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="121.624" y="15">Update</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_29">
|
||||
<rect x="-223" y="650.5381" width="306" height="54.961924" fill="white"/>
|
||||
<rect x="-223" y="650.5381" width="306" height="54.961924" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(-218 668.019)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="116" y="15">GetRepo</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_30">
|
||||
<rect x="-12.734375" y="1088.724" width="441.46875" height="46" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(-7.734375 1101.724)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="102.52637" y="15">contentserver-repo-current.json</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_31">
|
||||
<rect x="-69" y="912.816" width="554" height="67" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(-64 936.316)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="700" fill="black" x="172.144" y="16">Var Directory (aka History)</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_34">
|
||||
<rect x="546" y="912.7753" width="341" height="67" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(551 936.2753)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="700" fill="black" x="83.124" y="16">Logfile (JSON or TXT)</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_37">
|
||||
<rect x="-262" y="275" width="384" height="67" fill="white"/>
|
||||
<rect x="-262" y="275" width="384" height="67" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(-257 298.5)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="700" fill="black" x="173.824" y="16">API</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_39">
|
||||
<rect x="670" y="623.10984" width="156" height="87.88546" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(675 657.0526)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="65" y="15">…</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_40">
|
||||
<rect x="495" y="623.10984" width="156" height="87.88546" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(500 657.0526)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="65" y="15">…</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_41">
|
||||
<rect x="-12.734375" y="1010.3673" width="441.46875" height="46" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(-7.734375 1023.3673)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="97.70237" y="15">contentserver-repo-[Timestamp].json</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_45">
|
||||
<rect x="569" y="1000.2753" width="301" height="46" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(574 1012.2753)" fill="black">
|
||||
<tspan font-family="Futura" font-size="16" font-weight="500" fill="black" x="111.19531" y="17">Log entry</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_44">
|
||||
<rect x="569" y="1059.7753" width="301" height="46" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(574 1071.7753)" fill="black">
|
||||
<tspan font-family="Futura" font-size="16" font-weight="500" fill="black" x="111.19531" y="17">Log entry</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_46">
|
||||
<rect x="569" y="1119.2753" width="301" height="46" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(574 1131.2753)" fill="black">
|
||||
<tspan font-family="Futura" font-size="16" font-weight="500" fill="black" x="111.19531" y="17">Log entry</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_47">
|
||||
<rect x="569" y="1178.7753" width="301" height="46" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(574 1190.7753)" fill="black">
|
||||
<tspan font-family="Futura" font-size="16" font-weight="500" fill="black" x="111.19531" y="17">Log entry</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Line_48">
|
||||
<line x1="413" y1="120.5" x2="413" y2="67.4" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Line_49">
|
||||
<line x1="-469.47585" y1="537.66146" x2="-237.4888" y2="419.19996" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Line_50">
|
||||
<line x1="-469.47585" y1="537.66146" x2="-238.52136" y2="480.43586" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Line_51">
|
||||
<line x1="-469.47585" y1="537.66146" x2="-236.89965" y2="539.3806" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Line_53">
|
||||
<line x1="-464.51305" y1="540.5356" x2="-238.39468" y2="605.75816" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Line_52">
|
||||
<line x1="-469.47585" y1="537.66146" x2="-237.1631" y2="672.2017" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Line_56">
|
||||
<line x1="626.31374" y1="815" x2="685.49386" y2="901.1427" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Line_57">
|
||||
<line x1="268.92984" y1="815" x2="229.43732" y2="900.1143" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Graphic_58">
|
||||
<rect x="-548" y="500.94756" width="119.31591" height="84" fill="white"/>
|
||||
<rect x="-548" y="500.94756" width="119.31591" height="84" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(-543 532.94756)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="34.065957" y="15">Client</tspan>
|
||||
</text>
|
||||
</g>
|
||||
</g>
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 16 KiB |
260
graphics/Update-Flow.svg
Normal file
260
graphics/Update-Flow.svg
Normal file
@ -0,0 +1,260 @@
|
||||
<?xml version="1.0" encoding="UTF-8" standalone="no"?>
|
||||
<!DOCTYPE svg PUBLIC "-//W3C//DTD SVG 1.1//EN" "http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd">
|
||||
<svg xmlns:dc="http://purl.org/dc/elements/1.1/" xmlns="http://www.w3.org/2000/svg" xmlns:xl="http://www.w3.org/1999/xlink" version="1.1" viewBox="314 98 1205 2185" width="1205" height="2185">
|
||||
<defs>
|
||||
<font-face font-family="Helvetica Neue" font-size="16" panose-1="2 0 5 3 0 0 0 2 0 4" units-per-em="1000" underline-position="-100" underline-thickness="50" slope="0" x-height="517" cap-height="714" ascent="951.9958" descent="-212.99744" font-weight="400">
|
||||
<font-face-src>
|
||||
<font-face-name name="HelveticaNeue"/>
|
||||
</font-face-src>
|
||||
</font-face>
|
||||
<marker orient="auto" overflow="visible" markerUnits="strokeWidth" id="FilledArrow_Marker" stroke-linejoin="miter" stroke-miterlimit="10" viewBox="-1 -3 7 6" markerWidth="7" markerHeight="6" color="black">
|
||||
<g>
|
||||
<path d="M 4.8 0 L 0 -1.8 L 0 1.8 Z" fill="currentColor" stroke="currentColor" stroke-width="1"/>
|
||||
</g>
|
||||
</marker>
|
||||
<font-face font-family="Futura" font-size="16" panose-1="2 11 6 2 2 2 4 2 3 3" units-per-em="1000" underline-position="-97.65625" underline-thickness="78.125" slope="0" x-height="482.4219" cap-height="761.2305" ascent="1038.5742" descent="-259.76562" font-weight="500">
|
||||
<font-face-src>
|
||||
<font-face-name name="Futura-Medium"/>
|
||||
</font-face-src>
|
||||
</font-face>
|
||||
<font-face font-family="Helvetica Neue" font-size="16" panose-1="2 0 8 3 0 0 0 9 0 4" units-per-em="1000" underline-position="-100" underline-thickness="50" slope="0" x-height="517" cap-height="714" ascent="975.0061" descent="-216.99524" font-weight="700">
|
||||
<font-face-src>
|
||||
<font-face-name name="HelveticaNeue-Bold"/>
|
||||
</font-face-src>
|
||||
</font-face>
|
||||
<font-face font-family="Futura" font-size="80" panose-1="2 11 6 2 2 2 4 2 3 3" units-per-em="1000" underline-position="-97.65625" underline-thickness="78.125" slope="0" x-height="482.4219" cap-height="761.2305" ascent="1038.5742" descent="-259.76562" font-weight="500">
|
||||
<font-face-src>
|
||||
<font-face-name name="Futura-Medium"/>
|
||||
</font-face-src>
|
||||
</font-face>
|
||||
</defs>
|
||||
<metadata> Produced by OmniGraffle 7.10.2
|
||||
<dc:date>2019-05-29 10:21:17 +0000</dc:date>
|
||||
</metadata>
|
||||
<g id="Update-Flow" stroke-opacity="1" fill="none" stroke="none" stroke-dasharray="none" fill-opacity="1">
|
||||
<title>Update-Flow</title>
|
||||
<rect fill="white" x="314" y="98" width="1205" height="2185"/>
|
||||
<g id="Update-Flow: Layer 1">
|
||||
<title>Layer 1</title>
|
||||
<g id="Graphic_4">
|
||||
<rect x="526" y="282" width="298" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(531 303.5)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="89.328" y="15">*Repo.Update()</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_5">
|
||||
<rect x="526" y="425" width="298" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(531 446.5)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="80.144" y="15">*Repo.tryUpdate()</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_7">
|
||||
<rect x="315" y="587" width="298" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(320 608.5)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="23.344" y="15">if updateErr != errUpdateRejected</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Line_8">
|
||||
<line x1="632.66975" y1="489" x2="516.5623" y2="578.1441" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Graphic_10">
|
||||
<rect x="545.27846" y="518.599" width="56" height="32" fill="white"/>
|
||||
<text transform="translate(550.27846 523.599)" fill="black">
|
||||
<tspan font-family="Futura" font-size="16" font-weight="500" fill="black" x="0" y="17">failure</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_12">
|
||||
<rect x="315" y="768" width="298" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(320 789.5)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="43.848" y="15">*Repo.tryToRestoreCurrent()</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Line_13">
|
||||
<line x1="464" y1="651" x2="464" y2="754.1" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Line_15">
|
||||
<line x1="675" y1="346" x2="675" y2="411.1" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Graphic_16">
|
||||
<rect x="1045" y="768" width="473" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(1050 789.5)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="8.42" y="15">*Repo.updateInProgressChan <- make(chan updateResponse)</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_18">
|
||||
<rect x="736" y="768" width="222" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(741 789.5)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="15.032" y="15">return errUpdateRejected</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_19">
|
||||
<rect x="522.1144" y="518.599" width="102.32812" height="32" fill="white"/>
|
||||
<text transform="translate(527.1144 523.599)" fill="black">
|
||||
<tspan font-family="Futura" font-size="16" font-weight="500" fill="black" x="0" y="17">update error</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Line_20">
|
||||
<line x1="691.2974" y1="489" x2="824.9201" y2="755.4686" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Line_21">
|
||||
<line x1="732.4672" y1="489" x2="1212.8041" y2="760.6497" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Graphic_22">
|
||||
<rect x="716.2705" y="603.8538" width="81.28906" height="32" fill="white"/>
|
||||
<text transform="translate(721.2705 608.8538)" fill="black">
|
||||
<tspan font-family="Futura" font-size="16" font-weight="500" fill="black" x="0" y="17">queue full</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_23">
|
||||
<rect x="924.7738" y="608.0878" width="93.11719" height="32" fill="white"/>
|
||||
<text transform="translate(929.7738 613.0878)" fill="black">
|
||||
<tspan font-family="Futura" font-size="16" font-weight="500" fill="black" x="2.46875" y="17">queue free </tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_24">
|
||||
<rect x="315" y="964" width="298" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(320 985.5)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="59.832" y="15">return updateResponse</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Line_25">
|
||||
<line x1="464" y1="832" x2="464" y2="950.1" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Graphic_28">
|
||||
<rect x="736" y="883" width="782" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(741 904.5)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="700" fill="black" x="299.496" y="16">*Repo.updateRoutine()</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_29">
|
||||
<rect x="736" y="946" width="782" height="1072" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Graphic_31">
|
||||
<rect x="767" y="982" width="359.1709" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(772 1003.5)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="13.305457" y="15">resChan <- *Repo.updateInProgressChannel:</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_32">
|
||||
<rect x="816" y="1069" width="359.1709" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(821 1090.5)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="121.24146" y="15">*Repo.update()</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_33">
|
||||
<rect x="878.4145" y="1156" width="359.1709" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(883.4145 1177.5)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="134.88146" y="15">*Repo.get()</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_34">
|
||||
<rect x="949" y="1243" width="359.1709" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(954 1264.5)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="67.75346" y="15">*Repo.loadNodesFromJSON()</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_35">
|
||||
<rect x="1013" y="1330" width="359.1709" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(1018 1351.5)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="107.47346" y="15">*Repo.loadNodes()</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Line_36">
|
||||
<line x1="1237.837" y1="832" x2="1181.0111" y2="874.2976" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-dasharray="8.0,8.0" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Graphic_37">
|
||||
<rect x="1056" y="1417" width="409.1709" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(1061 1420.828)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="0" y="15">for dimension, newNode := range nodes {</tspan>
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="0" y="33.448"> *Repo.updateDimension(dimension, newNode)</tspan>
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="0" y="51.895996">}</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_38">
|
||||
<rect x="1106" y="1508" width="359.1709" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(1111 1529.5)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="700" fill="black" x="47.369457" y="16">*Repo.dimensionUpdateRoutine()</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_39">
|
||||
<rect x="1106" y="1571" width="359.1709" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(1111 1592.5)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="79.44946" y="15">*Repo._updateDimension()</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_40">
|
||||
<rect x="1106" y="1634" width="359.1709" height="343" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Graphic_41">
|
||||
<rect x="1127" y="1648" width="298" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(1132 1669.5)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="60.152" y="15">newNode.WireParents()</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_42">
|
||||
<rect x="1127" y="1731.5" width="298" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(1132 1753)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="90.072" y="15">buildDirectory()</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_43">
|
||||
<rect x="1127" y="1815" width="298" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(1132 1836.5)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="99.872" y="15">wireAliases()</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_44">
|
||||
<rect x="1127" y="1898.5" width="298" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(1132 1920)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="20.8" y="15">dimensionUpdateDoneChan <- err</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Graphic_45">
|
||||
<rect x="736" y="2078" width="359.1709" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(741 2099.5)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="71.76146" y="15">*Repo.history.add(jsonBytes)</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Line_46">
|
||||
<line x1="946.0763" y1="2019" x2="930.6539" y2="2064.7752" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-dasharray="8.0,8.0" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Graphic_48">
|
||||
<rect x="766.5855" y="2219" width="298" height="63" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
<text transform="translate(771.5855 2240.5)" fill="black">
|
||||
<tspan font-family="Helvetica Neue" font-size="16" font-weight="400" fill="black" x="32.208" y="15">resultChan <- updateResponse</tspan>
|
||||
</text>
|
||||
</g>
|
||||
<g id="Line_47">
|
||||
<line x1="915.5855" y1="2142" x2="915.5855" y2="2205.1" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Line_49">
|
||||
<line x1="1018.9012" y1="1133" x2="1027.1646" y2="1144.5183" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Line_50">
|
||||
<line x1="1084.3681" y1="1220" x2="1094.0898" y2="1231.9824" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Line_51">
|
||||
<line x1="1152.4935" y1="1307" x2="1161.0333" y2="1318.6088" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Line_52">
|
||||
<line x1="1217.9878" y1="1394" x2="1227.2391" y2="1405.8363" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Line_53">
|
||||
<line x1="1276" y1="1712" x2="1276" y2="1717.6" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Line_54">
|
||||
<line x1="1276" y1="1795.5" x2="1276" y2="1801.1" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Line_55">
|
||||
<line x1="1276" y1="1879" x2="1276" y2="1884.6" marker-end="url(#FilledArrow_Marker)" stroke="black" stroke-linecap="round" stroke-linejoin="round" stroke-width="2"/>
|
||||
</g>
|
||||
<g id="Graphic_57">
|
||||
<text transform="translate(411 103)" fill="black">
|
||||
<tspan font-family="Futura" font-size="80" font-weight="500" fill="black" x="0" y="83">Contentserver</tspan>
|
||||
<tspan font-family="Futura" font-size="80" font-weight="500" fill="black" y="83">:</tspan>
|
||||
<tspan font-family="Futura" font-size="80" font-weight="500" fill="black" y="83"> Update Flow</tspan>
|
||||
</text>
|
||||
</g>
|
||||
</g>
|
||||
</g>
|
||||
</svg>
|
||||
|
After Width: | Height: | Size: 17 KiB |
79
log/log.go
79
log/log.go
@ -1,79 +0,0 @@
|
||||
package log
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
// Level logging level enum
|
||||
type Level int
|
||||
|
||||
const (
|
||||
// LevelError an error - as bad as it gets
|
||||
LevelError Level = 0
|
||||
// LevelRecord put this to the logs in any case
|
||||
LevelRecord Level = 1
|
||||
// LevelWarning not that bad
|
||||
LevelWarning Level = 2
|
||||
// LevelNotice almost on debug level
|
||||
LevelNotice Level = 3
|
||||
// LevelDebug we are debugging
|
||||
LevelDebug Level = 4
|
||||
)
|
||||
|
||||
// SelectedLevel selected log level
|
||||
var SelectedLevel = LevelDebug
|
||||
|
||||
var prefices = map[Level]string{
|
||||
LevelRecord: "record : ",
|
||||
LevelError: "error : ",
|
||||
LevelWarning: "warning : ",
|
||||
LevelNotice: "notice : ",
|
||||
LevelDebug: "debug : ",
|
||||
}
|
||||
|
||||
func log(msg string, level Level) string {
|
||||
if level <= SelectedLevel {
|
||||
prefix := time.Now().Format(time.RFC3339Nano) + " " + prefices[level]
|
||||
lines := strings.Split(msg, "\n")
|
||||
for i := 0; i < len(lines); i++ {
|
||||
fmt.Println(level, prefix+lines[i])
|
||||
}
|
||||
}
|
||||
return msg
|
||||
}
|
||||
|
||||
func logThings(msgs []interface{}, level Level) string {
|
||||
r := ""
|
||||
for _, msg := range msgs {
|
||||
r += "\n" + fmt.Sprint(msg)
|
||||
}
|
||||
r = strings.Trim(r, "\n")
|
||||
return log(r, level)
|
||||
}
|
||||
|
||||
// Debug write debug messages to the log
|
||||
func Debug(msgs ...interface{}) string {
|
||||
return logThings(msgs, LevelDebug)
|
||||
}
|
||||
|
||||
// Notice write notice messages to the log
|
||||
func Notice(msgs ...interface{}) string {
|
||||
return logThings(msgs, LevelNotice)
|
||||
}
|
||||
|
||||
// Warning write warning messages to the log
|
||||
func Warning(msgs ...interface{}) string {
|
||||
return logThings(msgs, LevelWarning)
|
||||
}
|
||||
|
||||
// Record write record messages to the log
|
||||
func Record(msgs ...interface{}) string {
|
||||
return logThings(msgs, LevelRecord)
|
||||
}
|
||||
|
||||
// Error write error messages to the log
|
||||
func Error(msgs ...interface{}) string {
|
||||
return logThings(msgs, LevelError)
|
||||
}
|
||||
43
logger/log.go
Normal file
43
logger/log.go
Normal file
@ -0,0 +1,43 @@
|
||||
package logger
|
||||
|
||||
import (
|
||||
"log"
|
||||
"os"
|
||||
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
var (
|
||||
// Log is the logger instance exposed by this package
|
||||
// call Setup() prior to using it
|
||||
// want JSON output? Set LOG_JSON env var to 1!
|
||||
Log *zap.Logger
|
||||
)
|
||||
|
||||
// SetupLogging configures the logger
|
||||
func SetupLogging(debug bool, outputPath string) {
|
||||
|
||||
var (
|
||||
zc zap.Config
|
||||
err error
|
||||
)
|
||||
|
||||
if debug {
|
||||
zc = zap.NewDevelopmentConfig()
|
||||
zc.OutputPaths = append(zc.OutputPaths, outputPath)
|
||||
} else {
|
||||
zc = zap.NewProductionConfig()
|
||||
zc.OutputPaths = append(zc.OutputPaths, outputPath)
|
||||
}
|
||||
|
||||
if os.Getenv("LOG_JSON") == "1" {
|
||||
zc.Encoding = "json"
|
||||
} else {
|
||||
zc.Encoding = "console"
|
||||
}
|
||||
|
||||
Log, err = zc.Build()
|
||||
if err != nil {
|
||||
log.Fatalf("can't initialize zap logger: %v", err)
|
||||
}
|
||||
}
|
||||
21
metrics/prometheus.go
Normal file
21
metrics/prometheus.go
Normal file
@ -0,0 +1,21 @@
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
|
||||
. "github.com/foomo/contentserver/logger"
|
||||
"github.com/prometheus/client_golang/prometheus/promhttp"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
const metricsRoute = "/metrics"
|
||||
|
||||
func RunPrometheusHandler(listener string) {
|
||||
Log.Info("starting prometheus handler on",
|
||||
zap.String("address", listener),
|
||||
zap.String("route", metricsRoute),
|
||||
)
|
||||
Log.Error("prometheus listener failed",
|
||||
zap.Error(http.ListenAndServe(listener, promhttp.Handler())),
|
||||
)
|
||||
}
|
||||
14
prometheus/prometheus.yml
Normal file
14
prometheus/prometheus.yml
Normal file
@ -0,0 +1,14 @@
|
||||
# reference: https://prometheus.io/docs/prometheus/latest/configuration/configuration/
|
||||
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
scrape_timeout: 15s
|
||||
#evaluation_interval: 15s
|
||||
|
||||
scrape_configs:
|
||||
- job_name: contentserver
|
||||
metrics_path: /metrics
|
||||
scheme: http
|
||||
static_configs:
|
||||
- targets:
|
||||
- 127.0.0.1:9111
|
||||
@ -1,19 +1,28 @@
|
||||
package repo
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"path"
|
||||
"sort"
|
||||
"strings"
|
||||
"time"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
. "github.com/foomo/contentserver/logger"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
const historyRepoJSONPrefix = "contentserver-repo-"
|
||||
const historyRepoJSONSuffix = ".json"
|
||||
const maxHistoryVersions = 20
|
||||
const (
|
||||
historyRepoJSONPrefix = "contentserver-repo-"
|
||||
historyRepoJSONSuffix = ".json"
|
||||
)
|
||||
|
||||
var flagMaxHistoryVersions = flag.Int("max-history", 2, "set the maximum number of content backup files")
|
||||
|
||||
type history struct {
|
||||
varDir string
|
||||
@ -26,18 +35,34 @@ func newHistory(varDir string) *history {
|
||||
}
|
||||
|
||||
func (h *history) add(jsonBytes []byte) error {
|
||||
// historic file name
|
||||
filename := path.Join(h.varDir, historyRepoJSONPrefix+time.Now().Format(time.RFC3339Nano)+historyRepoJSONSuffix)
|
||||
err := ioutil.WriteFile(filename, jsonBytes, 0644)
|
||||
|
||||
var (
|
||||
// historiy file name
|
||||
filename = path.Join(h.varDir, historyRepoJSONPrefix+time.Now().Format(time.RFC3339Nano)+historyRepoJSONSuffix)
|
||||
err = ioutil.WriteFile(filename, jsonBytes, 0644)
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
Log.Info("adding content backup", zap.String("file", filename))
|
||||
|
||||
// current filename
|
||||
return ioutil.WriteFile(h.getCurrentFilename(), jsonBytes, 0644)
|
||||
err = ioutil.WriteFile(h.getCurrentFilename(), jsonBytes, 0644)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = h.cleanup()
|
||||
if err != nil {
|
||||
Log.Error("an error occured while cleaning up my history", zap.Error(err))
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (h *history) getHistory() (files []string, err error) {
|
||||
files = []string{}
|
||||
fileInfos, err := ioutil.ReadDir(h.varDir)
|
||||
if err != nil {
|
||||
return
|
||||
@ -56,14 +81,16 @@ func (h *history) getHistory() (files []string, err error) {
|
||||
}
|
||||
|
||||
func (h *history) cleanup() error {
|
||||
files, err := h.getFilesForCleanup(maxHistoryVersions)
|
||||
files, err := h.getFilesForCleanup(*flagMaxHistoryVersions)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, f := range files {
|
||||
Log.Info("removing outdated backup", zap.String("file", f))
|
||||
err := os.Remove(f)
|
||||
if err != nil {
|
||||
return errors.New(fmt.Sprintf("could not remove file %s : %s", f, err.Error()))
|
||||
return fmt.Errorf("could not remove file %s : %s", f, err.Error())
|
||||
}
|
||||
}
|
||||
|
||||
@ -75,8 +102,16 @@ func (h *history) getFilesForCleanup(historyVersions int) (files []string, err e
|
||||
if err != nil {
|
||||
return nil, errors.New("could not generate file cleanup list: " + err.Error())
|
||||
}
|
||||
if len(contentFiles) > historyVersions {
|
||||
for i := historyVersions; i < len(contentFiles); i++ {
|
||||
|
||||
// fmt.Println("contentFiles:")
|
||||
// for _, f := range contentFiles {
|
||||
// fmt.Println(f)
|
||||
// }
|
||||
|
||||
// -1 to remove the current backup file from the number of items
|
||||
// so that only files with a timestamp are compared
|
||||
if len(contentFiles)-1 > historyVersions {
|
||||
for i := historyVersions + 1; i < len(contentFiles); i++ {
|
||||
// ignore current repository file to fall back on
|
||||
if contentFiles[i] == h.getCurrentFilename() {
|
||||
continue
|
||||
@ -91,6 +126,12 @@ func (h *history) getCurrentFilename() string {
|
||||
return path.Join(h.varDir, historyRepoJSONPrefix+"current"+historyRepoJSONSuffix)
|
||||
}
|
||||
|
||||
func (h *history) getCurrent() (jsonBytes []byte, err error) {
|
||||
return ioutil.ReadFile(h.getCurrentFilename())
|
||||
func (h *history) getCurrent(buf *bytes.Buffer) (err error) {
|
||||
f, err := os.Open(h.getCurrentFilename())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
_, err = io.Copy(buf, f)
|
||||
return err
|
||||
}
|
||||
|
||||
@ -18,31 +18,45 @@ func testHistory() *history {
|
||||
}
|
||||
|
||||
func TestHistoryCurrent(t *testing.T) {
|
||||
h := testHistory()
|
||||
test := []byte("test")
|
||||
h.add(test)
|
||||
current, err := h.getCurrent()
|
||||
var (
|
||||
h = testHistory()
|
||||
test = []byte("test")
|
||||
b bytes.Buffer
|
||||
)
|
||||
err := h.add(test)
|
||||
if err != nil {
|
||||
t.Fatal("failed to add: ", err)
|
||||
}
|
||||
err = h.getCurrent(&b)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if bytes.Compare(current, test) != 0 {
|
||||
t.Fatal(fmt.Sprintf("expected %q, got %q", string(test), string(current)))
|
||||
if !bytes.Equal(b.Bytes(), test) {
|
||||
t.Fatal(fmt.Sprintf("expected %q, got %q", string(test), string(b.Bytes())))
|
||||
}
|
||||
}
|
||||
|
||||
func TestHistoryCleanup(t *testing.T) {
|
||||
h := testHistory()
|
||||
for i := 0; i < 50; i++ {
|
||||
h.add([]byte(fmt.Sprint(i)))
|
||||
err := h.add([]byte(fmt.Sprint(i)))
|
||||
if err != nil {
|
||||
t.Fatal("failed to add: ", err)
|
||||
}
|
||||
time.Sleep(time.Millisecond * 5)
|
||||
}
|
||||
h.cleanup()
|
||||
err := h.cleanup()
|
||||
if err != nil {
|
||||
t.Fatal("failed to run cleanup: ", err)
|
||||
}
|
||||
files, err := h.getHistory()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if len(files) != maxHistoryVersions {
|
||||
t.Fatal("history too long", len(files), "instead of", maxHistoryVersions)
|
||||
|
||||
// -1 for ignoring the current content backup file
|
||||
if len(files)-1 != *flagMaxHistoryVersions {
|
||||
t.Fatal("history too long", len(files), "instead of", *flagMaxHistoryVersions)
|
||||
}
|
||||
}
|
||||
|
||||
@ -51,7 +65,6 @@ func TestHistoryOrder(t *testing.T) {
|
||||
h.varDir = "testdata/order"
|
||||
|
||||
files, err := h.getHistory()
|
||||
|
||||
if err != nil {
|
||||
t.Fatal("error not expected")
|
||||
}
|
||||
@ -69,11 +82,9 @@ func TestGetFilesForCleanup(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal("error not expected")
|
||||
}
|
||||
assertStringEqual(t, "testdata/order/contentserver-repo-2017-10-22.json", files[0])
|
||||
assertStringEqual(t, "testdata/order/contentserver-repo-2017-10-21.json", files[1])
|
||||
assertStringEqual(t, "testdata/order/contentserver-repo-2017-10-21.json", files[0])
|
||||
}
|
||||
|
||||
|
||||
func assertStringEqual(t *testing.T, expected, actual string) {
|
||||
if expected != actual {
|
||||
t.Errorf("expected string %s differs from the actual %s", expected, actual)
|
||||
|
||||
194
repo/loader.go
194
repo/loader.go
@ -1,50 +1,86 @@
|
||||
package repo
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"io"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/foomo/contentserver/content"
|
||||
"github.com/foomo/contentserver/log"
|
||||
. "github.com/foomo/contentserver/logger"
|
||||
"github.com/foomo/contentserver/status"
|
||||
jsoniter "github.com/json-iterator/go"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
var (
|
||||
json = jsoniter.ConfigCompatibleWithStandardLibrary
|
||||
errUpdateRejected = errors.New("update rejected: queue full")
|
||||
)
|
||||
|
||||
type updateResponse struct {
|
||||
repoRuntime int64
|
||||
err error
|
||||
}
|
||||
|
||||
func (repo *Repo) updateRoutine() {
|
||||
go func() {
|
||||
for {
|
||||
log.Debug("update routine is about to select")
|
||||
select {
|
||||
case newDimension := <-repo.updateChannel:
|
||||
log.Debug("update routine received a new dimension: " + newDimension.Dimension)
|
||||
err := repo._updateDimension(newDimension.Dimension, newDimension.Node)
|
||||
log.Debug("update routine received result")
|
||||
if err != nil {
|
||||
log.Debug(" update routine error: " + err.Error())
|
||||
}
|
||||
repo.updateDoneChannel <- err
|
||||
for {
|
||||
select {
|
||||
case resChan := <-repo.updateInProgressChannel:
|
||||
Log.Info("waiting for update to complete", zap.String("chan", fmt.Sprintf("%p", resChan)))
|
||||
start := time.Now()
|
||||
|
||||
repoRuntime, errUpdate := repo.update()
|
||||
if errUpdate != nil {
|
||||
status.M.UpdatesFailedCounter.WithLabelValues(errUpdate.Error()).Inc()
|
||||
}
|
||||
|
||||
resChan <- updateResponse{
|
||||
repoRuntime: repoRuntime,
|
||||
err: errUpdate,
|
||||
}
|
||||
|
||||
duration := time.Since(start)
|
||||
Log.Info("update completed", zap.Duration("duration", duration), zap.String("chan", fmt.Sprintf("%p", resChan)))
|
||||
status.M.UpdatesCompletedCounter.WithLabelValues().Inc()
|
||||
status.M.UpdateDuration.WithLabelValues().Observe(duration.Seconds())
|
||||
}
|
||||
}()
|
||||
}
|
||||
}
|
||||
|
||||
func (repo *Repo) dimensionUpdateRoutine() {
|
||||
for newDimension := range repo.dimensionUpdateChannel {
|
||||
Log.Info("dimensionUpdateRoutine received a new dimension", zap.String("dimension", newDimension.Dimension))
|
||||
|
||||
err := repo._updateDimension(newDimension.Dimension, newDimension.Node)
|
||||
Log.Info("dimensionUpdateRoutine received result")
|
||||
if err != nil {
|
||||
Log.Debug("update dimension failed", zap.Error(err))
|
||||
}
|
||||
repo.dimensionUpdateDoneChannel <- err
|
||||
}
|
||||
}
|
||||
|
||||
func (repo *Repo) updateDimension(dimension string, node *content.RepoNode) error {
|
||||
repo.updateChannel <- &repoDimension{
|
||||
Log.Debug("trying to push dimension into update channel", zap.String("dimension", dimension), zap.String("nodeName", node.Name))
|
||||
repo.dimensionUpdateChannel <- &repoDimension{
|
||||
Dimension: dimension,
|
||||
Node: node,
|
||||
}
|
||||
return <-repo.updateDoneChannel
|
||||
Log.Debug("waiting for done signal")
|
||||
return <-repo.dimensionUpdateDoneChannel
|
||||
}
|
||||
|
||||
// do not call directly, but only through channel
|
||||
func (repo *Repo) _updateDimension(dimension string, newNode *content.RepoNode) error {
|
||||
newNode.WireParents()
|
||||
newDirectory := make(map[string]*content.RepoNode)
|
||||
newURIDirectory := make(map[string]*content.RepoNode)
|
||||
|
||||
err := builDirectory(newNode, newDirectory, newURIDirectory)
|
||||
var (
|
||||
newDirectory = make(map[string]*content.RepoNode)
|
||||
newURIDirectory = make(map[string]*content.RepoNode)
|
||||
err = buildDirectory(newNode, newDirectory, newURIDirectory)
|
||||
)
|
||||
if err != nil {
|
||||
return errors.New("update dimension \"" + dimension + "\" failed when building its directory:: " + err.Error())
|
||||
}
|
||||
@ -53,23 +89,43 @@ func (repo *Repo) _updateDimension(dimension string, newNode *content.RepoNode)
|
||||
return err
|
||||
}
|
||||
|
||||
// ---------------------------------------------
|
||||
|
||||
// copy old datastructure to prevent concurrent map access
|
||||
// collect other dimension in the Directory
|
||||
newRepoDirectory := map[string]*Dimension{}
|
||||
for d, D := range repo.Directory {
|
||||
if d != dimension {
|
||||
newRepoDirectory[d] = D
|
||||
}
|
||||
}
|
||||
|
||||
// add the new dimension
|
||||
newRepoDirectory[dimension] = &Dimension{
|
||||
Node: newNode,
|
||||
Directory: newDirectory,
|
||||
URIDirectory: newURIDirectory,
|
||||
}
|
||||
repo.Directory = newRepoDirectory
|
||||
|
||||
// ---------------------------------------------
|
||||
|
||||
// @TODO: why not update only the dimension that has changed instead?
|
||||
// repo.Directory[dimension] = &Dimension{
|
||||
// Node: newNode,
|
||||
// Directory: newDirectory,
|
||||
// URIDirectory: newURIDirectory,
|
||||
// }
|
||||
|
||||
// ---------------------------------------------
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func builDirectory(dirNode *content.RepoNode, directory map[string]*content.RepoNode, uRIDirectory map[string]*content.RepoNode) error {
|
||||
log.Debug("repo.buildDirectory: " + dirNode.ID)
|
||||
func buildDirectory(dirNode *content.RepoNode, directory map[string]*content.RepoNode, uRIDirectory map[string]*content.RepoNode) error {
|
||||
|
||||
// Log.Debug("buildDirectory", zap.String("ID", dirNode.ID))
|
||||
|
||||
existingNode, ok := directory[dirNode.ID]
|
||||
if ok {
|
||||
return errors.New("duplicate node with id:" + existingNode.ID)
|
||||
@ -81,7 +137,7 @@ func builDirectory(dirNode *content.RepoNode, directory map[string]*content.Repo
|
||||
}
|
||||
uRIDirectory[dirNode.URI] = dirNode
|
||||
for _, childNode := range dirNode.Nodes {
|
||||
err := builDirectory(childNode, directory, uRIDirectory)
|
||||
err := buildDirectory(childNode, directory, uRIDirectory)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -102,74 +158,98 @@ func wireAliases(directory map[string]*content.RepoNode) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func loadNodesFromJSON(jsonBytes []byte) (nodes map[string]*content.RepoNode, err error) {
|
||||
func (repo *Repo) loadNodesFromJSON() (nodes map[string]*content.RepoNode, err error) {
|
||||
nodes = make(map[string]*content.RepoNode)
|
||||
err = json.Unmarshal(jsonBytes, &nodes)
|
||||
err = json.Unmarshal(repo.jsonBuf.Bytes(), &nodes)
|
||||
return nodes, err
|
||||
}
|
||||
|
||||
func (repo *Repo) tryToRestoreCurrent() error {
|
||||
currentJSONBytes, err := repo.history.getCurrent()
|
||||
func (repo *Repo) tryToRestoreCurrent() (err error) {
|
||||
err = repo.history.getCurrent(&repo.jsonBuf)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return repo.loadJSONBytes(currentJSONBytes)
|
||||
return repo.loadJSONBytes()
|
||||
}
|
||||
|
||||
func get(URL string) (data []byte, err error) {
|
||||
func (repo *Repo) get(URL string) (err error) {
|
||||
response, err := http.Get(URL)
|
||||
if err != nil {
|
||||
return data, err
|
||||
return err
|
||||
}
|
||||
defer response.Body.Close()
|
||||
if response.StatusCode != http.StatusOK {
|
||||
return data, fmt.Errorf("Bad HTTP Response: %q", response.Status)
|
||||
return fmt.Errorf("Bad HTTP Response: %q", response.Status)
|
||||
}
|
||||
return ioutil.ReadAll(response.Body)
|
||||
|
||||
// Log.Info(ansi.Red + "RESETTING BUFFER" + ansi.Reset)
|
||||
repo.jsonBuf.Reset()
|
||||
|
||||
// Log.Info(ansi.Green + "LOADING DATA INTO BUFFER" + ansi.Reset)
|
||||
_, err = io.Copy(&repo.jsonBuf, response.Body)
|
||||
return err
|
||||
}
|
||||
|
||||
func (repo *Repo) update() (repoRuntime int64, jsonBytes []byte, err error) {
|
||||
func (repo *Repo) update() (repoRuntime int64, err error) {
|
||||
startTimeRepo := time.Now().UnixNano()
|
||||
jsonBytes, err = get(repo.server)
|
||||
err = repo.get(repo.server)
|
||||
repoRuntime = time.Now().UnixNano() - startTimeRepo
|
||||
if err != nil {
|
||||
// we have no json to load - the repo server did not reply
|
||||
log.Debug("we have no json to load - the repo server did not reply", err)
|
||||
return repoRuntime, jsonBytes, err
|
||||
Log.Debug("failed to load json", zap.Error(err))
|
||||
return repoRuntime, err
|
||||
}
|
||||
log.Debug("loading json from: "+repo.server, string(jsonBytes))
|
||||
nodes, err := loadNodesFromJSON(jsonBytes)
|
||||
Log.Debug("loading json", zap.String("server", repo.server), zap.Int("length", len(repo.jsonBuf.Bytes())))
|
||||
nodes, err := repo.loadNodesFromJSON()
|
||||
if err != nil {
|
||||
// could not load nodes from json
|
||||
return repoRuntime, jsonBytes, err
|
||||
return repoRuntime, err
|
||||
}
|
||||
err = repo.loadNodes(nodes)
|
||||
if err != nil {
|
||||
// repo failed to load nodes
|
||||
return repoRuntime, jsonBytes, err
|
||||
return repoRuntime, err
|
||||
}
|
||||
return repoRuntime, jsonBytes, nil
|
||||
return repoRuntime, nil
|
||||
}
|
||||
|
||||
func (repo *Repo) loadJSONBytes(jsonBytes []byte) error {
|
||||
nodes, err := loadNodesFromJSON(jsonBytes)
|
||||
// limit ressources and allow only one update request at once
|
||||
func (repo *Repo) tryUpdate() (repoRuntime int64, err error) {
|
||||
c := make(chan updateResponse)
|
||||
select {
|
||||
case repo.updateInProgressChannel <- c:
|
||||
Log.Info("update request added to queue")
|
||||
ur := <-c
|
||||
return ur.repoRuntime, ur.err
|
||||
default:
|
||||
Log.Info("update request rejected, queue is full")
|
||||
status.M.UpdatesRejectedCounter.WithLabelValues().Inc()
|
||||
return 0, errUpdateRejected
|
||||
}
|
||||
}
|
||||
|
||||
func (repo *Repo) loadJSONBytes() error {
|
||||
nodes, err := repo.loadNodesFromJSON()
|
||||
if err != nil {
|
||||
log.Debug("could not parse json", string(jsonBytes))
|
||||
data := repo.jsonBuf.Bytes()
|
||||
|
||||
if len(data) > 10 {
|
||||
Log.Debug("could not parse json",
|
||||
zap.String("jsonStart", string(data[:10])),
|
||||
zap.String("jsonStart", string(data[len(data)-10:])),
|
||||
)
|
||||
}
|
||||
return err
|
||||
}
|
||||
|
||||
err = repo.loadNodes(nodes)
|
||||
if err == nil {
|
||||
historyErr := repo.history.add(jsonBytes)
|
||||
historyErr := repo.history.add(repo.jsonBuf.Bytes())
|
||||
if historyErr != nil {
|
||||
log.Warning("could not add valid json to history:" + historyErr.Error())
|
||||
Log.Error("could not add valid json to history", zap.Error(historyErr))
|
||||
status.M.HistoryPersistFailedCounter.WithLabelValues(historyErr.Error()).Inc()
|
||||
} else {
|
||||
log.Record("added valid json to history")
|
||||
}
|
||||
cleanUpErr := repo.history.cleanup()
|
||||
if cleanUpErr != nil {
|
||||
log.Warning("an error occured while cleaning up my history:", cleanUpErr)
|
||||
} else {
|
||||
log.Record("cleaned up history")
|
||||
Log.Info("added valid json to history")
|
||||
}
|
||||
}
|
||||
return err
|
||||
@ -179,10 +259,10 @@ func (repo *Repo) loadNodes(newNodes map[string]*content.RepoNode) error {
|
||||
newDimensions := []string{}
|
||||
for dimension, newNode := range newNodes {
|
||||
newDimensions = append(newDimensions, dimension)
|
||||
log.Debug("loading nodes for dimension " + dimension)
|
||||
Log.Debug("loading nodes for dimension", zap.String("dimension", dimension))
|
||||
loadErr := repo.updateDimension(dimension, newNode)
|
||||
if loadErr != nil {
|
||||
log.Debug(" failed to load " + dimension + ": " + loadErr.Error())
|
||||
Log.Debug("failed to load", zap.String("dimension", dimension), zap.Error(loadErr))
|
||||
return loadErr
|
||||
}
|
||||
}
|
||||
@ -197,7 +277,7 @@ func (repo *Repo) loadNodes(newNodes map[string]*content.RepoNode) error {
|
||||
// we need to throw away orphaned dimensions
|
||||
for dimension := range repo.Directory {
|
||||
if !dimensionIsValid(dimension) {
|
||||
log.Notice("removing orphaned dimension:" + dimension)
|
||||
Log.Info("removing orphaned dimension", zap.String("dimension", dimension))
|
||||
delete(repo.Directory, dimension)
|
||||
}
|
||||
}
|
||||
|
||||
@ -9,13 +9,12 @@ import (
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/foomo/contentserver/log"
|
||||
"github.com/foomo/contentserver/requests"
|
||||
)
|
||||
|
||||
// GetMockData mock data to run a repo
|
||||
func GetMockData(t testing.TB) (server *httptest.Server, varDir string) {
|
||||
log.SelectedLevel = log.LevelError
|
||||
|
||||
_, filename, _, _ := runtime.Caller(0)
|
||||
mockDir := path.Dir(filename)
|
||||
|
||||
|
||||
179
repo/repo.go
179
repo/repo.go
@ -1,17 +1,25 @@
|
||||
package repo
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/foomo/contentserver/status"
|
||||
|
||||
"github.com/foomo/contentserver/content"
|
||||
"github.com/foomo/contentserver/log"
|
||||
. "github.com/foomo/contentserver/logger"
|
||||
"github.com/foomo/contentserver/requests"
|
||||
"github.com/foomo/contentserver/responses"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
const maxGetURIForNodeRecursionLevel = 1000
|
||||
|
||||
// Dimension dimension in a repo
|
||||
type Dimension struct {
|
||||
Directory map[string]*content.RepoNode
|
||||
@ -21,11 +29,17 @@ type Dimension struct {
|
||||
|
||||
// Repo content repositiory
|
||||
type Repo struct {
|
||||
server string
|
||||
Directory map[string]*Dimension
|
||||
updateChannel chan *repoDimension
|
||||
updateDoneChannel chan error
|
||||
history *history
|
||||
server string
|
||||
Directory map[string]*Dimension
|
||||
// updateLock sync.Mutex
|
||||
dimensionUpdateChannel chan *repoDimension
|
||||
dimensionUpdateDoneChannel chan error
|
||||
|
||||
history *history
|
||||
updateInProgressChannel chan chan updateResponse
|
||||
|
||||
// jsonBytes []byte
|
||||
jsonBuf bytes.Buffer
|
||||
}
|
||||
|
||||
type repoDimension struct {
|
||||
@ -35,22 +49,29 @@ type repoDimension struct {
|
||||
|
||||
// NewRepo constructor
|
||||
func NewRepo(server string, varDir string) *Repo {
|
||||
log.Notice("creating new repo for " + server)
|
||||
log.Notice(" using var dir:" + varDir)
|
||||
|
||||
Log.Info("creating new repo",
|
||||
zap.String("server", server),
|
||||
zap.String("varDir", varDir),
|
||||
)
|
||||
repo := &Repo{
|
||||
server: server,
|
||||
Directory: map[string]*Dimension{},
|
||||
history: newHistory(varDir),
|
||||
updateChannel: make(chan *repoDimension),
|
||||
updateDoneChannel: make(chan error),
|
||||
server: server,
|
||||
Directory: map[string]*Dimension{},
|
||||
history: newHistory(varDir),
|
||||
dimensionUpdateChannel: make(chan *repoDimension),
|
||||
dimensionUpdateDoneChannel: make(chan error),
|
||||
updateInProgressChannel: make(chan chan updateResponse, 0),
|
||||
}
|
||||
|
||||
go repo.updateRoutine()
|
||||
log.Record("trying to restore pervious state")
|
||||
go repo.dimensionUpdateRoutine()
|
||||
|
||||
Log.Info("trying to restore previous state")
|
||||
restoreErr := repo.tryToRestoreCurrent()
|
||||
if restoreErr != nil {
|
||||
log.Record(" could not restore previous repo content:" + restoreErr.Error())
|
||||
Log.Error(" could not restore previous repo content", zap.Error(restoreErr))
|
||||
} else {
|
||||
log.Record(" restored previous repo content")
|
||||
Log.Info("restored previous repo content")
|
||||
}
|
||||
return repo
|
||||
}
|
||||
@ -70,10 +91,18 @@ func (repo *Repo) GetNodes(r *requests.Nodes) map[string]*content.Node {
|
||||
}
|
||||
|
||||
func (repo *Repo) getNodes(nodeRequests map[string]*requests.Node, env *requests.Env) map[string]*content.Node {
|
||||
nodes := map[string]*content.Node{}
|
||||
path := []*content.Item{}
|
||||
|
||||
var (
|
||||
nodes = map[string]*content.Node{}
|
||||
path = []*content.Item{}
|
||||
)
|
||||
for nodeName, nodeRequest := range nodeRequests {
|
||||
log.Debug(" adding node " + nodeName + " " + nodeRequest.ID)
|
||||
|
||||
if nodeName == "" || nodeRequest.ID == "" {
|
||||
Log.Info("invalid node request", zap.Error(errors.New("nodeName or nodeRequest.ID empty")))
|
||||
continue
|
||||
}
|
||||
Log.Debug("adding node", zap.String("name", nodeName), zap.String("requestID", nodeRequest.ID))
|
||||
|
||||
groups := env.Groups
|
||||
if len(nodeRequest.Groups) > 0 {
|
||||
@ -84,26 +113,29 @@ func (repo *Repo) getNodes(nodeRequests map[string]*requests.Node, env *requests
|
||||
nodes[nodeName] = nil
|
||||
|
||||
if !ok && nodeRequest.Dimension == "" {
|
||||
log.Debug(" could not get dimension root node for dimension " + nodeRequest.Dimension)
|
||||
Log.Debug("could not get dimension root node", zap.String("dimension", nodeRequest.Dimension))
|
||||
for _, dimension := range env.Dimensions {
|
||||
dimensionNode, ok = repo.Directory[dimension]
|
||||
if ok {
|
||||
log.Debug(" searched for root node in env.dimension " + dimension + " with success")
|
||||
Log.Debug("found root node in env.Dimensions", zap.String("dimension", dimension))
|
||||
break
|
||||
}
|
||||
log.Debug(" searched for root node in env.dimension " + dimension + " without success")
|
||||
Log.Debug("could NOT find root node in env.Dimensions", zap.String("dimension", dimension))
|
||||
}
|
||||
}
|
||||
|
||||
if !ok {
|
||||
log.Warning("could not get dimension root node for nodeRequest.Dimension: " + nodeRequest.Dimension)
|
||||
Log.Error("could not get dimension root node", zap.String("nodeRequest.Dimension", nodeRequest.Dimension))
|
||||
continue
|
||||
}
|
||||
treeNode, ok := dimensionNode.Directory[nodeRequest.ID]
|
||||
if ok {
|
||||
nodes[nodeName] = repo.getNode(treeNode, nodeRequest.Expand, nodeRequest.MimeTypes, path, 0, groups, nodeRequest.DataFields)
|
||||
} else {
|
||||
log.Warning("you are requesting an invalid tree node for " + nodeName + " : " + nodeRequest.ID)
|
||||
Log.Error("an invalid tree node was requested",
|
||||
zap.String("nodeName", nodeName),
|
||||
zap.String("ID", nodeRequest.ID),
|
||||
)
|
||||
}
|
||||
}
|
||||
return nodes
|
||||
@ -122,18 +154,18 @@ func (repo *Repo) GetContent(r *requests.Content) (c *content.SiteContent, err e
|
||||
// add more input validation
|
||||
err = repo.validateContentRequest(r)
|
||||
if err != nil {
|
||||
log.Debug("repo.GetContent invalid request", err)
|
||||
Log.Error("repo.GetContent invalid request", zap.Error(err))
|
||||
return
|
||||
}
|
||||
log.Debug("repo.GetContent: ", r.URI)
|
||||
Log.Debug("repo.GetContent", zap.String("URI", r.URI))
|
||||
c = content.NewSiteContent()
|
||||
resolved, resolvedURI, resolvedDimension, node := repo.resolveContent(r.Env.Dimensions, r.URI)
|
||||
if resolved {
|
||||
if !node.CanBeAccessedByGroups(r.Env.Groups) {
|
||||
log.Notice("401 for " + r.URI)
|
||||
Log.Warn("resolvecontent got status 401", zap.String("URI", r.URI))
|
||||
c.Status = content.StatusForbidden
|
||||
} else {
|
||||
log.Notice("200 for " + r.URI)
|
||||
Log.Info("resolvecontent got status 200", zap.String("URI", r.URI))
|
||||
c.Status = content.StatusOk
|
||||
c.Data = node.Data
|
||||
}
|
||||
@ -149,15 +181,22 @@ func (repo *Repo) GetContent(r *requests.Content) (c *content.SiteContent, err e
|
||||
}
|
||||
c.URIs = uris
|
||||
} else {
|
||||
log.Notice("404 for " + r.URI)
|
||||
Log.Info("resolvecontent got status 404", zap.String("URI", r.URI))
|
||||
c.Status = content.StatusNotFound
|
||||
c.Dimension = r.Env.Dimensions[0]
|
||||
}
|
||||
if log.SelectedLevel == log.LevelDebug {
|
||||
log.Debug(fmt.Sprintf("resolved: %v, uri: %v, dim: %v, n: %v", resolved, resolvedURI, resolvedDimension, node))
|
||||
}
|
||||
if resolved == false {
|
||||
log.Debug("repo.GetContent", r.URI, "could not be resolved falling back to default dimension", r.Env.Dimensions[0])
|
||||
|
||||
Log.Debug("got content",
|
||||
zap.Bool("resolved", resolved),
|
||||
zap.String("resolvedURI", resolvedURI),
|
||||
zap.String("resolvedDimension", resolvedDimension),
|
||||
zap.String("nodeName", node.Name),
|
||||
)
|
||||
if !resolved {
|
||||
Log.Debug("failed to resolve, falling back to default dimension",
|
||||
zap.String("URI", r.URI),
|
||||
zap.String("defaultDimension", r.Env.Dimensions[0]),
|
||||
)
|
||||
// r.Env.Dimensions is validated => we can access it
|
||||
resolvedDimension = r.Env.Dimensions[0]
|
||||
}
|
||||
@ -180,13 +219,35 @@ func (repo *Repo) GetRepo() map[string]*content.RepoNode {
|
||||
return response
|
||||
}
|
||||
|
||||
// WriteRepoBytes get the whole repo in all dimensions
|
||||
// reads the JSON history file from the Filesystem and copies it directly in to the supplied buffer
|
||||
// the result is wrapped as service response, e.g: {"reply": <contentData>}
|
||||
func (repo *Repo) WriteRepoBytes(w io.Writer) {
|
||||
|
||||
f, err := os.Open(repo.history.getCurrentFilename())
|
||||
if err != nil {
|
||||
Log.Error("failed to serve Repo JSON", zap.Error(err))
|
||||
}
|
||||
|
||||
w.Write([]byte("{\"reply\":"))
|
||||
_, err = io.Copy(w, f)
|
||||
if err != nil {
|
||||
Log.Error("failed to serve Repo JSON", zap.Error(err))
|
||||
}
|
||||
w.Write([]byte("}"))
|
||||
}
|
||||
|
||||
// Update - reload contents of repository with json from repo.server
|
||||
func (repo *Repo) Update() (updateResponse *responses.Update) {
|
||||
floatSeconds := func(nanoSeconds int64) float64 {
|
||||
return float64(float64(nanoSeconds) / float64(1000000000.0))
|
||||
}
|
||||
|
||||
Log.Info("Update triggered")
|
||||
// Log.Info(ansi.Yellow + "BUFFER LENGTH BEFORE tryUpdate(): " + strconv.Itoa(len(repo.jsonBuf.Bytes())) + ansi.Reset)
|
||||
|
||||
startTime := time.Now().UnixNano()
|
||||
updateRepotime, jsonBytes, updateErr := repo.update()
|
||||
updateRepotime, updateErr := repo.tryUpdate()
|
||||
updateResponse = &responses.Update{}
|
||||
updateResponse.Stats.RepoRuntime = floatSeconds(updateRepotime)
|
||||
|
||||
@ -195,20 +256,26 @@ func (repo *Repo) Update() (updateResponse *responses.Update) {
|
||||
updateResponse.Stats.NumberOfNodes = -1
|
||||
updateResponse.Stats.NumberOfURIs = -1
|
||||
// let us try to restore the world from a file
|
||||
log.Error("could not update repository:" + updateErr.Error())
|
||||
Log.Error("could not update repository:", zap.Error(updateErr))
|
||||
// Log.Info(ansi.Yellow + "BUFFER LENGTH AFTER ERROR: " + strconv.Itoa(len(repo.jsonBuf.Bytes())) + ansi.Reset)
|
||||
updateResponse.ErrorMessage = updateErr.Error()
|
||||
restoreErr := repo.tryToRestoreCurrent()
|
||||
if restoreErr != nil {
|
||||
log.Error("failed to restore preceding repo version: " + restoreErr.Error())
|
||||
} else {
|
||||
log.Record("restored current repo from local history")
|
||||
|
||||
// only try to restore if the update failed during processing
|
||||
if updateErr != errUpdateRejected {
|
||||
restoreErr := repo.tryToRestoreCurrent()
|
||||
if restoreErr != nil {
|
||||
Log.Error("failed to restore preceding repo version", zap.Error(restoreErr))
|
||||
} else {
|
||||
Log.Info("restored current repo from local history")
|
||||
}
|
||||
}
|
||||
} else {
|
||||
updateResponse.Success = true
|
||||
// persist the currently loaded one
|
||||
historyErr := repo.history.add(jsonBytes)
|
||||
historyErr := repo.history.add(repo.jsonBuf.Bytes())
|
||||
if historyErr != nil {
|
||||
log.Warning("could not persist current repo in history: " + historyErr.Error())
|
||||
Log.Error("could not persist current repo in history", zap.Error(historyErr))
|
||||
status.M.HistoryPersistFailedCounter.WithLabelValues(historyErr.Error()).Inc()
|
||||
}
|
||||
// add some stats
|
||||
for dimension := range repo.Directory {
|
||||
@ -223,11 +290,7 @@ func (repo *Repo) Update() (updateResponse *responses.Update) {
|
||||
// resolveContent find content in a repository
|
||||
func (repo *Repo) resolveContent(dimensions []string, URI string) (resolved bool, resolvedURI string, resolvedDimension string, repoNode *content.RepoNode) {
|
||||
parts := strings.Split(URI, content.PathSeparator)
|
||||
resolved = false
|
||||
resolvedURI = ""
|
||||
resolvedDimension = ""
|
||||
repoNode = nil
|
||||
log.Debug("repo.ResolveContent: " + URI)
|
||||
Log.Debug("repo.ResolveContent", zap.String("URI", URI))
|
||||
for i := len(parts); i > 0; i-- {
|
||||
testURI := strings.Join(parts[0:i], content.PathSeparator)
|
||||
if testURI == "" {
|
||||
@ -235,11 +298,13 @@ func (repo *Repo) resolveContent(dimensions []string, URI string) (resolved bool
|
||||
}
|
||||
for _, dimension := range dimensions {
|
||||
if d, ok := repo.Directory[dimension]; ok {
|
||||
log.Debug(" testing[" + dimension + "]: " + testURI)
|
||||
Log.Debug("checking",
|
||||
zap.String("dimension", dimension),
|
||||
zap.String("URI", testURI),
|
||||
)
|
||||
if repoNode, ok := d.URIDirectory[testURI]; ok {
|
||||
resolved = true
|
||||
log.Debug(" found => " + testURI)
|
||||
log.Debug(" destination " + fmt.Sprint(repoNode.DestinationID))
|
||||
Log.Debug("found node", zap.String("URI", testURI), zap.String("destination", repoNode.DestinationID))
|
||||
if len(repoNode.DestinationID) > 0 {
|
||||
if destionationNode, destinationNodeOk := d.Directory[repoNode.DestinationID]; destinationNodeOk {
|
||||
repoNode = destionationNode
|
||||
@ -253,8 +318,6 @@ func (repo *Repo) resolveContent(dimensions []string, URI string) (resolved bool
|
||||
return
|
||||
}
|
||||
|
||||
const maxGetURIForNodeRecursionLevel = 1000
|
||||
|
||||
func (repo *Repo) getURIForNode(dimension string, repoNode *content.RepoNode, recursionLevel int64) (uri string) {
|
||||
if len(repoNode.LinkID) == 0 {
|
||||
uri = repoNode.URI
|
||||
@ -263,7 +326,7 @@ func (repo *Repo) getURIForNode(dimension string, repoNode *content.RepoNode, re
|
||||
linkedNode, ok := repo.Directory[dimension].Directory[repoNode.LinkID]
|
||||
if ok {
|
||||
if recursionLevel > maxGetURIForNodeRecursionLevel {
|
||||
log.Error("maxGetURIForNodeRecursionLevel reached for", repoNode.ID, "link id", repoNode.LinkID, "in dimension", dimension)
|
||||
Log.Error("maxGetURIForNodeRecursionLevel reached", zap.String("repoNode.ID", repoNode.ID), zap.String("linkID", repoNode.LinkID), zap.String("dimension", dimension))
|
||||
return ""
|
||||
}
|
||||
return repo.getURIForNode(dimension, linkedNode, recursionLevel+1)
|
||||
@ -282,7 +345,7 @@ func (repo *Repo) getURI(dimension string, id string) string {
|
||||
func (repo *Repo) getNode(repoNode *content.RepoNode, expanded bool, mimeTypes []string, path []*content.Item, level int, groups []string, dataFields []string) *content.Node {
|
||||
node := content.NewNode()
|
||||
node.Item = repoNode.ToItem(dataFields)
|
||||
log.Debug("repo.GetNode: " + repoNode.ID)
|
||||
Log.Debug("getNode", zap.String("ID", repoNode.ID))
|
||||
for _, childID := range repoNode.Index {
|
||||
childNode := repoNode.Nodes[childID]
|
||||
if (level == 0 || expanded || !expanded && childNode.InPath(path)) && !childNode.Hidden && childNode.CanBeAccessedByGroups(groups) && childNode.IsOneOfTheseMimeTypes(mimeTypes) {
|
||||
@ -327,6 +390,6 @@ func (repo *Repo) hasDimension(d string) bool {
|
||||
return hasDimension
|
||||
}
|
||||
|
||||
func uriKeyForState(state string, uri string) string {
|
||||
return state + "-" + uri
|
||||
}
|
||||
// func uriKeyForState(state string, uri string) string {
|
||||
// return state + "-" + uri
|
||||
// }
|
||||
|
||||
@ -3,11 +3,18 @@ package repo
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
. "github.com/foomo/contentserver/logger"
|
||||
_ "github.com/foomo/contentserver/logger"
|
||||
"github.com/foomo/contentserver/repo/mock"
|
||||
"github.com/foomo/contentserver/requests"
|
||||
)
|
||||
|
||||
func init() {
|
||||
SetupLogging(true, "contentserver_repo_test.log")
|
||||
}
|
||||
|
||||
func assertRepoIsEmpty(t *testing.T, r *Repo, empty bool) {
|
||||
if empty {
|
||||
if len(r.Directory) > 0 {
|
||||
@ -15,15 +22,18 @@ func assertRepoIsEmpty(t *testing.T, r *Repo, empty bool) {
|
||||
}
|
||||
} else {
|
||||
if len(r.Directory) == 0 {
|
||||
t.Fatal("directory should not have been empty, but it is")
|
||||
t.Fatal("directory is empty, but should have been not")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoad404(t *testing.T) {
|
||||
mockServer, varDir := mock.GetMockData(t)
|
||||
server := mockServer.URL + "/repo-no-have"
|
||||
r := NewRepo(server, varDir)
|
||||
var (
|
||||
mockServer, varDir = mock.GetMockData(t)
|
||||
server = mockServer.URL + "/repo-no-have"
|
||||
r = NewRepo(server, varDir)
|
||||
)
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
response := r.Update()
|
||||
if response.Success {
|
||||
t.Fatal("can not get a repo, if the server responds with a 404")
|
||||
@ -31,9 +41,12 @@ func TestLoad404(t *testing.T) {
|
||||
}
|
||||
|
||||
func TestLoadBrokenRepo(t *testing.T) {
|
||||
mockServer, varDir := mock.GetMockData(t)
|
||||
server := mockServer.URL + "/repo-broken-json.json"
|
||||
r := NewRepo(server, varDir)
|
||||
var (
|
||||
mockServer, varDir = mock.GetMockData(t)
|
||||
server = mockServer.URL + "/repo-broken-json.json"
|
||||
r = NewRepo(server, varDir)
|
||||
)
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
response := r.Update()
|
||||
if response.Success {
|
||||
t.Fatal("how could we load a broken json")
|
||||
@ -42,13 +55,17 @@ func TestLoadBrokenRepo(t *testing.T) {
|
||||
|
||||
func TestLoadRepo(t *testing.T) {
|
||||
|
||||
mockServer, varDir := mock.GetMockData(t)
|
||||
server := mockServer.URL + "/repo-ok.json"
|
||||
r := NewRepo(server, varDir)
|
||||
var (
|
||||
mockServer, varDir = mock.GetMockData(t)
|
||||
server = mockServer.URL + "/repo-ok.json"
|
||||
r = NewRepo(server, varDir)
|
||||
)
|
||||
assertRepoIsEmpty(t, r, true)
|
||||
|
||||
response := r.Update()
|
||||
assertRepoIsEmpty(t, r, false)
|
||||
if response.Success == false {
|
||||
|
||||
if !response.Success {
|
||||
t.Fatal("could not load valid repo")
|
||||
}
|
||||
if response.Stats.OwnRuntime > response.Stats.RepoRuntime {
|
||||
@ -63,23 +80,61 @@ func TestLoadRepo(t *testing.T) {
|
||||
assertRepoIsEmpty(t, nr, false)
|
||||
}
|
||||
|
||||
func BenchmarkLoadRepo(b *testing.B) {
|
||||
|
||||
var (
|
||||
t = &testing.T{}
|
||||
mockServer, varDir = mock.GetMockData(t)
|
||||
server = mockServer.URL + "/repo-ok.json"
|
||||
r = NewRepo(server, varDir)
|
||||
)
|
||||
if len(r.Directory) > 0 {
|
||||
b.Fatal("directory should have been empty, but is not")
|
||||
}
|
||||
|
||||
b.ReportAllocs()
|
||||
b.ResetTimer()
|
||||
for n := 0; n < b.N; n++ {
|
||||
response := r.Update()
|
||||
if len(r.Directory) == 0 {
|
||||
b.Fatal("directory is empty, but should have been not")
|
||||
}
|
||||
|
||||
if !response.Success {
|
||||
b.Fatal("could not load valid repo")
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoadRepoDuplicateUris(t *testing.T) {
|
||||
mockServer, varDir := mock.GetMockData(t)
|
||||
server := mockServer.URL + "/repo-duplicate-uris.json"
|
||||
r := NewRepo(server, varDir)
|
||||
|
||||
var (
|
||||
mockServer, varDir = mock.GetMockData(t)
|
||||
server = mockServer.URL + "/repo-duplicate-uris.json"
|
||||
r = NewRepo(server, varDir)
|
||||
)
|
||||
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
|
||||
response := r.Update()
|
||||
if response.Success {
|
||||
t.Fatal("there are duplicates, this repo update should have failed")
|
||||
}
|
||||
if !strings.Contains(response.ErrorMessage, "update dimension") {
|
||||
t.Fatal("error message not as expected")
|
||||
t.Fatal("error message not as expected: " + response.ErrorMessage)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDimensionHygiene(t *testing.T) {
|
||||
mockServer, varDir := mock.GetMockData(t)
|
||||
server := mockServer.URL + "/repo-two-dimensions.json"
|
||||
r := NewRepo(server, varDir)
|
||||
|
||||
var (
|
||||
mockServer, varDir = mock.GetMockData(t)
|
||||
server = mockServer.URL + "/repo-two-dimensions.json"
|
||||
r = NewRepo(server, varDir)
|
||||
)
|
||||
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
|
||||
response := r.Update()
|
||||
if !response.Success {
|
||||
t.Fatal("well those two dimension should be fine")
|
||||
@ -98,6 +153,7 @@ func getTestRepo(path string, t *testing.T) *Repo {
|
||||
mockServer, varDir := mock.GetMockData(t)
|
||||
server := mockServer.URL + path
|
||||
r := NewRepo(server, varDir)
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
response := r.Update()
|
||||
if !response.Success {
|
||||
t.Fatal("well those two dimension should be fine")
|
||||
@ -138,6 +194,7 @@ func TestLinkIds(t *testing.T) {
|
||||
mockServer, varDir := mock.GetMockData(t)
|
||||
server := mockServer.URL + "/repo-link-ok.json"
|
||||
r := NewRepo(server, varDir)
|
||||
time.Sleep(500 * time.Millisecond)
|
||||
response := r.Update()
|
||||
if !response.Success {
|
||||
t.Fatal("those links should have been fine")
|
||||
|
||||
@ -1,31 +1,38 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/foomo/contentserver/log"
|
||||
"go.uber.org/zap"
|
||||
|
||||
. "github.com/foomo/contentserver/logger"
|
||||
"github.com/foomo/contentserver/repo"
|
||||
"github.com/foomo/contentserver/requests"
|
||||
"github.com/foomo/contentserver/responses"
|
||||
"github.com/foomo/contentserver/status"
|
||||
)
|
||||
|
||||
func handleRequest(r *repo.Repo, handler Handler, jsonBytes []byte) (replyBytes []byte, err error) {
|
||||
func handleRequest(r *repo.Repo, handler Handler, jsonBytes []byte, source string) (replyBytes []byte, err error) {
|
||||
|
||||
var reply interface{}
|
||||
var apiErr error
|
||||
var jsonErr error
|
||||
|
||||
processIfJSONIsOk := func(err error, processingFunc func()) {
|
||||
if err != nil {
|
||||
jsonErr = err
|
||||
return
|
||||
var (
|
||||
reply interface{}
|
||||
apiErr error
|
||||
jsonErr error
|
||||
start = time.Now()
|
||||
processIfJSONIsOk = func(err error, processingFunc func()) {
|
||||
if err != nil {
|
||||
jsonErr = err
|
||||
return
|
||||
}
|
||||
processingFunc()
|
||||
}
|
||||
processingFunc()
|
||||
}
|
||||
)
|
||||
status.M.ContentRequestCounter.WithLabelValues(source).Inc()
|
||||
|
||||
// handle and process
|
||||
switch handler {
|
||||
// case HandlerGetRepo: // This case is handled prior to handleRequest being called.
|
||||
// since the resulting bytes are written directly in to the http.ResponseWriter / net.Connection
|
||||
case HandlerGetURIs:
|
||||
getURIRequest := &requests.URIs{}
|
||||
processIfJSONIsOk(json.Unmarshal(jsonBytes, &getURIRequest), func() {
|
||||
@ -46,38 +53,46 @@ func handleRequest(r *repo.Repo, handler Handler, jsonBytes []byte) (replyBytes
|
||||
processIfJSONIsOk(json.Unmarshal(jsonBytes, &updateRequest), func() {
|
||||
reply = r.Update()
|
||||
})
|
||||
case HandlerGetRepo:
|
||||
repoRequest := &requests.Repo{}
|
||||
processIfJSONIsOk(json.Unmarshal(jsonBytes, &repoRequest), func() {
|
||||
reply = r.GetRepo()
|
||||
})
|
||||
|
||||
default:
|
||||
err = errors.New(log.Error(" can not handle this one " + handler))
|
||||
errorResponse := responses.NewError(1, "unknown handler")
|
||||
reply = errorResponse
|
||||
reply = responses.NewError(1, "unknown handler: "+string(handler))
|
||||
}
|
||||
addMetrics(handler, start, jsonErr, apiErr, source)
|
||||
|
||||
// error handling
|
||||
if jsonErr != nil {
|
||||
err = jsonErr
|
||||
log.Error(" could not read incoming json:", jsonErr)
|
||||
errorResponse := responses.NewError(2, "could not read incoming json "+jsonErr.Error())
|
||||
reply = errorResponse
|
||||
Log.Error("could not read incoming json", zap.Error(jsonErr))
|
||||
reply = responses.NewError(2, "could not read incoming json "+jsonErr.Error())
|
||||
} else if apiErr != nil {
|
||||
log.Error(" an API error occured:", apiErr)
|
||||
err = apiErr
|
||||
Log.Error("an API error occured", zap.Error(apiErr))
|
||||
reply = responses.NewError(3, "internal error "+apiErr.Error())
|
||||
}
|
||||
|
||||
return encodeReply(reply)
|
||||
}
|
||||
|
||||
func encodeReply(reply interface{}) (replyBytes []byte, err error) {
|
||||
encodedBytes, jsonReplyErr := json.MarshalIndent(map[string]interface{}{
|
||||
"reply": reply,
|
||||
}, "", " ")
|
||||
if jsonReplyErr != nil {
|
||||
err = jsonReplyErr
|
||||
log.Error(" could not encode reply " + fmt.Sprint(jsonReplyErr))
|
||||
} else {
|
||||
replyBytes = encodedBytes
|
||||
func addMetrics(handlerName Handler, start time.Time, errJSON error, errAPI error, source string) {
|
||||
|
||||
var (
|
||||
duration = time.Since(start)
|
||||
s = "succeeded"
|
||||
)
|
||||
if errJSON != nil || errAPI != nil {
|
||||
s = "failed"
|
||||
}
|
||||
return replyBytes, err
|
||||
|
||||
status.M.ServiceRequestCounter.WithLabelValues(string(handlerName), s, source).Inc()
|
||||
status.M.ServiceRequestDuration.WithLabelValues(string(handlerName), s, source).Observe(float64(duration.Seconds()))
|
||||
}
|
||||
|
||||
// encodeReply takes an interface and encodes it as JSON
|
||||
// it returns the resulting JSON and a marshalling error
|
||||
func encodeReply(reply interface{}) (replyBytes []byte, err error) {
|
||||
replyBytes, err = json.Marshal(map[string]interface{}{
|
||||
"reply": reply,
|
||||
})
|
||||
if err != nil {
|
||||
Log.Error("could not encode reply", zap.Error(err))
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
@ -5,9 +5,16 @@ import (
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
|
||||
"github.com/foomo/contentserver/log"
|
||||
. "github.com/foomo/contentserver/logger"
|
||||
"github.com/foomo/contentserver/repo"
|
||||
jsoniter "github.com/json-iterator/go"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
var (
|
||||
json = jsoniter.ConfigCompatibleWithStandardLibrary
|
||||
)
|
||||
|
||||
// Handler type
|
||||
@ -34,23 +41,42 @@ func Run(server string, address string, varDir string) error {
|
||||
func RunServerSocketAndWebServer(
|
||||
server string,
|
||||
address string,
|
||||
webserverAdresss string,
|
||||
webserverAddress string,
|
||||
webserverPath string,
|
||||
varDir string,
|
||||
) error {
|
||||
if address == "" && webserverAdresss == "" {
|
||||
if address == "" && webserverAddress == "" {
|
||||
return errors.New("one of the addresses needs to be set")
|
||||
}
|
||||
log.Record("building repo with content from " + server)
|
||||
Log.Info("building repo with content", zap.String("server", server))
|
||||
|
||||
r := repo.NewRepo(server, varDir)
|
||||
go r.Update()
|
||||
|
||||
// start initial update and handle error
|
||||
go func() {
|
||||
resp := r.Update()
|
||||
if !resp.Success {
|
||||
Log.Error("failed to update",
|
||||
zap.String("error", resp.ErrorMessage),
|
||||
zap.Int("NumberOfNodes", resp.Stats.NumberOfNodes),
|
||||
zap.Int("NumberOfURIs", resp.Stats.NumberOfURIs),
|
||||
zap.Float64("OwnRuntime", resp.Stats.OwnRuntime),
|
||||
zap.Float64("RepoRuntime", resp.Stats.RepoRuntime),
|
||||
)
|
||||
os.Exit(1)
|
||||
}
|
||||
}()
|
||||
|
||||
// update can run in bg
|
||||
chanErr := make(chan error)
|
||||
|
||||
if address != "" {
|
||||
Log.Info("starting socketserver", zap.String("address", address))
|
||||
go runSocketServer(r, address, chanErr)
|
||||
}
|
||||
if webserverAdresss != "" {
|
||||
go runWebserver(r, webserverAdresss, webserverPath, chanErr)
|
||||
if webserverAddress != "" {
|
||||
Log.Info("starting webserver", zap.String("webserverAddress", webserverAddress))
|
||||
go runWebserver(r, webserverAddress, webserverPath, chanErr)
|
||||
}
|
||||
return <-chanErr
|
||||
}
|
||||
@ -61,12 +87,7 @@ func runWebserver(
|
||||
path string,
|
||||
chanErr chan error,
|
||||
) {
|
||||
s, errNew := NewWebServer(path, r)
|
||||
if errNew != nil {
|
||||
chanErr <- errNew
|
||||
return
|
||||
}
|
||||
chanErr <- http.ListenAndServe(address, s)
|
||||
chanErr <- http.ListenAndServe(address, NewWebServer(path, r))
|
||||
}
|
||||
|
||||
func runSocketServer(
|
||||
@ -74,31 +95,32 @@ func runSocketServer(
|
||||
address string,
|
||||
chanErr chan error,
|
||||
) {
|
||||
s := &socketServer{
|
||||
stats: newStats(),
|
||||
repo: repo,
|
||||
}
|
||||
ln, err := net.Listen("tcp", address)
|
||||
if err != nil {
|
||||
err = errors.New("RunSocketServer: could not start the on \"" + address + "\" - error: " + fmt.Sprint(err))
|
||||
// failed to create socket
|
||||
log.Error(err)
|
||||
chanErr <- err
|
||||
// create socket server
|
||||
s := newSocketServer(repo)
|
||||
|
||||
// listen on socket
|
||||
ln, errListen := net.Listen("tcp", address)
|
||||
if errListen != nil {
|
||||
Log.Error("runSocketServer: could not start",
|
||||
zap.String("address", address),
|
||||
zap.Error(errListen),
|
||||
)
|
||||
chanErr <- errors.New("runSocketServer: could not start the on \"" + address + "\" - error: " + fmt.Sprint(errListen))
|
||||
return
|
||||
}
|
||||
// there we go
|
||||
log.Record("RunSocketServer: started to listen on " + address)
|
||||
|
||||
Log.Info("runSocketServer: started listening", zap.String("address", address))
|
||||
for {
|
||||
// this blocks until connection or error
|
||||
conn, err := ln.Accept()
|
||||
if err != nil {
|
||||
log.Error("RunSocketServer: could not accept connection" + fmt.Sprint(err))
|
||||
Log.Error("runSocketServer: could not accept connection", zap.Error(err))
|
||||
continue
|
||||
}
|
||||
log.Debug("new connection")
|
||||
|
||||
// a goroutine handles conn so that the loop can accept other connections
|
||||
go func() {
|
||||
log.Debug("accepted connection")
|
||||
Log.Debug("accepted connection", zap.String("source", conn.RemoteAddr().String()))
|
||||
s.handleConnection(conn)
|
||||
conn.Close()
|
||||
// log.Debug("connection closed")
|
||||
|
||||
@ -1,48 +1,34 @@
|
||||
package server
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/foomo/contentserver/log"
|
||||
"go.uber.org/zap"
|
||||
|
||||
. "github.com/foomo/contentserver/logger"
|
||||
"github.com/foomo/contentserver/repo"
|
||||
"github.com/foomo/contentserver/responses"
|
||||
"github.com/foomo/contentserver/status"
|
||||
)
|
||||
|
||||
// simple internal request counter
|
||||
type stats struct {
|
||||
requests int64
|
||||
chanCount chan int
|
||||
}
|
||||
|
||||
func newStats() *stats {
|
||||
s := &stats{
|
||||
requests: 0,
|
||||
chanCount: make(chan int),
|
||||
}
|
||||
go func() {
|
||||
for {
|
||||
select {
|
||||
case <-s.chanCount:
|
||||
s.requests++
|
||||
s.chanCount <- 1
|
||||
}
|
||||
}
|
||||
}()
|
||||
return s
|
||||
}
|
||||
|
||||
func (s *stats) countRequest() {
|
||||
s.chanCount <- 1
|
||||
<-s.chanCount
|
||||
}
|
||||
const sourceSocketServer = "socketserver"
|
||||
|
||||
type socketServer struct {
|
||||
stats *stats
|
||||
repo *repo.Repo
|
||||
repo *repo.Repo
|
||||
metrics *status.Metrics
|
||||
}
|
||||
|
||||
// newSocketServer returns a shiny new socket server
|
||||
func newSocketServer(repo *repo.Repo) *socketServer {
|
||||
return &socketServer{
|
||||
repo: repo,
|
||||
}
|
||||
}
|
||||
|
||||
func extractHandlerAndJSONLentgh(header string) (handler Handler, jsonLength int, err error) {
|
||||
@ -58,14 +44,22 @@ func extractHandlerAndJSONLentgh(header string) (handler Handler, jsonLength int
|
||||
}
|
||||
|
||||
func (s *socketServer) execute(handler Handler, jsonBytes []byte) (reply []byte) {
|
||||
s.stats.countRequest()
|
||||
log.Notice("socketServer.execute: ", s.stats.requests, ", ", handler)
|
||||
if log.SelectedLevel == log.LevelDebug {
|
||||
log.Debug(" incoming json buffer:", string(jsonBytes))
|
||||
Log.Debug("incoming json buffer", zap.Int("length", len(jsonBytes)))
|
||||
|
||||
if handler == HandlerGetRepo {
|
||||
|
||||
var (
|
||||
b bytes.Buffer
|
||||
start = time.Now()
|
||||
)
|
||||
s.repo.WriteRepoBytes(&b)
|
||||
addMetrics(handler, start, nil, nil, sourceSocketServer)
|
||||
return b.Bytes()
|
||||
}
|
||||
reply, handlingError := handleRequest(s.repo, handler, jsonBytes)
|
||||
|
||||
reply, handlingError := handleRequest(s.repo, handler, jsonBytes, sourceSocketServer)
|
||||
if handlingError != nil {
|
||||
log.Error("socketServer.execute handlingError :", handlingError)
|
||||
Log.Error("socketServer.execute failed", zap.Error(handlingError))
|
||||
}
|
||||
return reply
|
||||
}
|
||||
@ -73,32 +67,39 @@ func (s *socketServer) execute(handler Handler, jsonBytes []byte) (reply []byte)
|
||||
func (s *socketServer) writeResponse(conn net.Conn, reply []byte) {
|
||||
headerBytes := []byte(strconv.Itoa(len(reply)))
|
||||
reply = append(headerBytes, reply...)
|
||||
log.Debug(" replying: " + string(reply))
|
||||
Log.Debug("replying", zap.String("reply", string(reply)))
|
||||
n, writeError := conn.Write(reply)
|
||||
if writeError != nil {
|
||||
log.Error("socketServer.writeResponse: could not write my reply: " + fmt.Sprint(writeError))
|
||||
Log.Error("socketServer.writeResponse: could not write reply", zap.Error(writeError))
|
||||
return
|
||||
}
|
||||
if n < len(reply) {
|
||||
log.Error(fmt.Sprintf("socketServer.writeResponse: write too short %q instead of %q", n, len(reply)))
|
||||
Log.Error("socketServer.writeResponse: write too short",
|
||||
zap.Int("got", n),
|
||||
zap.Int("expected", len(reply)),
|
||||
)
|
||||
return
|
||||
}
|
||||
log.Debug(" replied. waiting for next request on open connection")
|
||||
|
||||
Log.Debug("replied. waiting for next request on open connection")
|
||||
}
|
||||
|
||||
func (s *socketServer) handleConnection(conn net.Conn) {
|
||||
log.Debug("socketServer.handleConnection")
|
||||
var headerBuffer [1]byte
|
||||
header := ""
|
||||
i := 0
|
||||
Log.Debug("socketServer.handleConnection")
|
||||
status.M.NumSocketsGauge.WithLabelValues(conn.RemoteAddr().String()).Inc()
|
||||
|
||||
var (
|
||||
headerBuffer [1]byte
|
||||
header = ""
|
||||
i = 0
|
||||
)
|
||||
for {
|
||||
i++
|
||||
// fmt.Println("---->", i)
|
||||
// let us read with 1 byte steps on conn until we find "{"
|
||||
_, readErr := conn.Read(headerBuffer[0:])
|
||||
if readErr != nil {
|
||||
log.Debug(" looks like the client closed the connection: ", readErr)
|
||||
Log.Debug("looks like the client closed the connection", zap.Error(readErr))
|
||||
status.M.NumSocketsGauge.WithLabelValues(conn.RemoteAddr().String()).Dec()
|
||||
return
|
||||
}
|
||||
// read next byte
|
||||
@ -109,44 +110,54 @@ func (s *socketServer) handleConnection(conn net.Conn) {
|
||||
// reset header
|
||||
header = ""
|
||||
if headerErr != nil {
|
||||
log.Error("invalid request could not read header", headerErr)
|
||||
Log.Error("invalid request could not read header", zap.Error(headerErr))
|
||||
encodedErr, encodingErr := encodeReply(responses.NewError(4, "invalid header "+headerErr.Error()))
|
||||
if encodingErr == nil {
|
||||
s.writeResponse(conn, encodedErr)
|
||||
} else {
|
||||
log.Error("could not respond to invalid request", encodingErr)
|
||||
Log.Error("could not respond to invalid request", zap.Error(encodingErr))
|
||||
}
|
||||
return
|
||||
}
|
||||
log.Debug(fmt.Sprintf(" found json with %d bytes", jsonLength))
|
||||
Log.Debug("found json", zap.Int("length", jsonLength))
|
||||
if jsonLength > 0 {
|
||||
// let us try to read some json
|
||||
jsonBytes := make([]byte, jsonLength)
|
||||
|
||||
var (
|
||||
// let us try to read some json
|
||||
jsonBytes = make([]byte, jsonLength)
|
||||
jsonLengthCurrent = 1
|
||||
readRound = 0
|
||||
)
|
||||
|
||||
// that is "{"
|
||||
jsonBytes[0] = 123
|
||||
jsonLengthCurrent := 1
|
||||
readRound := 0
|
||||
|
||||
for jsonLengthCurrent < jsonLength {
|
||||
readRound++
|
||||
readLength, jsonReadErr := conn.Read(jsonBytes[jsonLengthCurrent:jsonLength])
|
||||
if jsonReadErr != nil {
|
||||
//@fixme we need to force a read timeout (SetReadDeadline?), if expected jsonLength is lower than really sent bytes (e.g. if client implements protocol wrong)
|
||||
//@todo should we check for io.EOF here
|
||||
log.Error(" could not read json - giving up with this client connection" + fmt.Sprint(jsonReadErr))
|
||||
Log.Error("could not read json - giving up with this client connection", zap.Error(jsonReadErr))
|
||||
status.M.NumSocketsGauge.WithLabelValues(conn.RemoteAddr().String()).Dec()
|
||||
return
|
||||
}
|
||||
jsonLengthCurrent += readLength
|
||||
log.Debug(fmt.Sprintf(" read so far %d of %d bytes in read cycle %d", jsonLengthCurrent, jsonLength, readRound))
|
||||
Log.Debug("read cycle status",
|
||||
zap.Int("jsonLengthCurrent", jsonLengthCurrent),
|
||||
zap.Int("jsonLength", jsonLength),
|
||||
zap.Int("readRound", readRound),
|
||||
)
|
||||
}
|
||||
|
||||
if log.SelectedLevel == log.LevelDebug {
|
||||
log.Debug(" read json: " + string(jsonBytes))
|
||||
}
|
||||
Log.Debug("read json", zap.Int("length", len(jsonBytes)))
|
||||
|
||||
s.writeResponse(conn, s.execute(handler, jsonBytes))
|
||||
// note: connection remains open
|
||||
continue
|
||||
}
|
||||
log.Error("can not read empty json")
|
||||
Log.Error("can not read empty json")
|
||||
status.M.NumSocketsGauge.WithLabelValues(conn.RemoteAddr().String()).Dec()
|
||||
return
|
||||
}
|
||||
// adding to header byte by byte
|
||||
|
||||
@ -4,22 +4,27 @@ import (
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"go.uber.org/zap"
|
||||
|
||||
. "github.com/foomo/contentserver/logger"
|
||||
"github.com/foomo/contentserver/repo"
|
||||
)
|
||||
|
||||
const sourceWebserver = "webserver"
|
||||
|
||||
type webServer struct {
|
||||
r *repo.Repo
|
||||
path string
|
||||
r *repo.Repo
|
||||
}
|
||||
|
||||
// NewWebServer returns a shiny new web server
|
||||
func NewWebServer(path string, r *repo.Repo) (s http.Handler, err error) {
|
||||
s = &webServer{
|
||||
r: r,
|
||||
func NewWebServer(path string, r *repo.Repo) http.Handler {
|
||||
return &webServer{
|
||||
path: path,
|
||||
r: r,
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
func (s *webServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
@ -33,10 +38,21 @@ func (s *webServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
http.Error(w, "failed to read incoming request", http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
reply, errReply := handleRequest(s.r, Handler(strings.TrimPrefix(r.URL.Path, s.path+"/")), jsonBytes)
|
||||
h := Handler(strings.TrimPrefix(r.URL.Path, s.path+"/"))
|
||||
if h == HandlerGetRepo {
|
||||
start := time.Now()
|
||||
s.r.WriteRepoBytes(w)
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
addMetrics(h, start, nil, nil, sourceWebserver)
|
||||
return
|
||||
}
|
||||
reply, errReply := handleRequest(s.r, h, jsonBytes, "webserver")
|
||||
if errReply != nil {
|
||||
http.Error(w, errReply.Error(), http.StatusInternalServerError)
|
||||
return
|
||||
}
|
||||
w.Write(reply)
|
||||
_, err := w.Write(reply)
|
||||
if err != nil {
|
||||
Log.Error("failed to write webServer reply", zap.Error(err))
|
||||
}
|
||||
}
|
||||
|
||||
37
status/healthz.go
Normal file
37
status/healthz.go
Normal file
@ -0,0 +1,37 @@
|
||||
package status
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
. "github.com/foomo/contentserver/logger"
|
||||
jsoniter "github.com/json-iterator/go"
|
||||
"go.uber.org/zap"
|
||||
)
|
||||
|
||||
var (
|
||||
json = jsoniter.ConfigCompatibleWithStandardLibrary
|
||||
)
|
||||
|
||||
func RunHealthzHandlerListener(address string, serviceName string) {
|
||||
Log.Info(fmt.Sprintf("starting healthz handler on '%s'" + address))
|
||||
Log.Error("healthz server failed", zap.Error(http.ListenAndServe(address, HealthzHandler(serviceName))))
|
||||
}
|
||||
|
||||
func HealthzHandler(serviceName string) http.Handler {
|
||||
var (
|
||||
data = map[string]string{
|
||||
"service": serviceName,
|
||||
}
|
||||
status, _ = json.Marshal(data)
|
||||
h = http.NewServeMux()
|
||||
)
|
||||
h.Handle("/healthz", http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
|
||||
_, err := w.Write(status)
|
||||
if err != nil {
|
||||
Log.Error("failed to write healthz status", zap.Error(err))
|
||||
}
|
||||
}))
|
||||
|
||||
return h
|
||||
}
|
||||
119
status/metrics.go
Normal file
119
status/metrics.go
Normal file
@ -0,0 +1,119 @@
|
||||
package status
|
||||
|
||||
import (
|
||||
"github.com/prometheus/client_golang/prometheus"
|
||||
)
|
||||
|
||||
// M is the Metrics instance
|
||||
var M = newMetrics()
|
||||
|
||||
const (
|
||||
namespace = "contentserver"
|
||||
|
||||
metricLabelHandler = "handler"
|
||||
metricLabelStatus = "status"
|
||||
metricLabelSource = "source"
|
||||
metricLabelRemote = "remote"
|
||||
metricLabelError = "error"
|
||||
)
|
||||
|
||||
// Metrics is the structure that holds all prometheus metrics
|
||||
type Metrics struct {
|
||||
ServiceRequestCounter *prometheus.CounterVec // count the number of requests for each service function
|
||||
ServiceRequestDuration *prometheus.SummaryVec // observe the duration of requests for each service function
|
||||
UpdatesRejectedCounter *prometheus.CounterVec // count the number of completed updates
|
||||
UpdatesCompletedCounter *prometheus.CounterVec // count the number of rejected updates
|
||||
UpdatesFailedCounter *prometheus.CounterVec // count the number of updates that had an error
|
||||
UpdateDuration *prometheus.SummaryVec // observe the duration of each repo.update() call
|
||||
ContentRequestCounter *prometheus.CounterVec // count the total number of content requests
|
||||
NumSocketsGauge *prometheus.GaugeVec // keep track of the total number of open sockets
|
||||
HistoryPersistFailedCounter *prometheus.CounterVec // count the number of failed attempts to persist the content history
|
||||
}
|
||||
|
||||
// newMetrics can be used to instantiate a metrics instance
|
||||
// since this function will also register each metric and metrics should only be registered once
|
||||
// it is private
|
||||
// the package exposes the initialized Metrics instance as the variable M.
|
||||
func newMetrics() *Metrics {
|
||||
return &Metrics{
|
||||
ServiceRequestCounter: newCounterVec(
|
||||
"service_request_count",
|
||||
"Count of requests for each handler",
|
||||
metricLabelHandler, metricLabelStatus, metricLabelSource,
|
||||
),
|
||||
ServiceRequestDuration: newSummaryVec(
|
||||
"service_request_duration_seconds",
|
||||
"Seconds to unmarshal requests, execute a service function and marshal its reponses",
|
||||
metricLabelHandler, metricLabelStatus, metricLabelSource,
|
||||
),
|
||||
UpdatesRejectedCounter: newCounterVec(
|
||||
"updates_rejected_count",
|
||||
"Number of updates that were rejected because the queue was full",
|
||||
),
|
||||
UpdatesCompletedCounter: newCounterVec(
|
||||
"updates_completed_count",
|
||||
"Number of updates that were successfully completed",
|
||||
),
|
||||
UpdatesFailedCounter: newCounterVec(
|
||||
"updates_failed_count",
|
||||
"Number of updates that failed due to an error",
|
||||
metricLabelError,
|
||||
),
|
||||
UpdateDuration: newSummaryVec(
|
||||
"update_duration_seconds",
|
||||
"Duration in seconds for each successful repo.update() call",
|
||||
),
|
||||
ContentRequestCounter: newCounterVec(
|
||||
"content_request_count",
|
||||
"Number of requests for content",
|
||||
metricLabelSource,
|
||||
),
|
||||
NumSocketsGauge: newGaugeVec(
|
||||
"num_sockets_total",
|
||||
"Total number of currently open socket connections",
|
||||
metricLabelRemote,
|
||||
),
|
||||
HistoryPersistFailedCounter: newCounterVec(
|
||||
"history_persist_failed_count",
|
||||
"Number of failures to store the content history on the filesystem",
|
||||
metricLabelError,
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Metric constructors
|
||||
*/
|
||||
|
||||
func newSummaryVec(name, help string, labels ...string) *prometheus.SummaryVec {
|
||||
vec := prometheus.NewSummaryVec(
|
||||
prometheus.SummaryOpts{
|
||||
Namespace: namespace,
|
||||
Name: name,
|
||||
Help: help,
|
||||
}, labels)
|
||||
prometheus.MustRegister(vec)
|
||||
return vec
|
||||
}
|
||||
|
||||
func newCounterVec(name, help string, labels ...string) *prometheus.CounterVec {
|
||||
vec := prometheus.NewCounterVec(
|
||||
prometheus.CounterOpts{
|
||||
Namespace: namespace,
|
||||
Name: name,
|
||||
Help: help,
|
||||
}, labels)
|
||||
prometheus.MustRegister(vec)
|
||||
return vec
|
||||
}
|
||||
|
||||
func newGaugeVec(name, help string, labels ...string) *prometheus.GaugeVec {
|
||||
vec := prometheus.NewGaugeVec(
|
||||
prometheus.GaugeOpts{
|
||||
Namespace: namespace,
|
||||
Name: name,
|
||||
Help: help,
|
||||
}, labels)
|
||||
prometheus.MustRegister(vec)
|
||||
return vec
|
||||
}
|
||||
59
testing/client/client.go
Normal file
59
testing/client/client.go
Normal file
@ -0,0 +1,59 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"github.com/davecgh/go-spew/spew"
|
||||
"github.com/foomo/contentserver/client"
|
||||
)
|
||||
|
||||
var (
|
||||
flagAddr = flag.String("addr", "http://127.0.0.1:9191/contentserver", "set addr")
|
||||
flagGetRepo = flag.Bool("getRepo", false, "get repo")
|
||||
flagUpdate = flag.Bool("update", true, "trigger content update")
|
||||
flagNum = flag.Int("num", 100, "num repititions")
|
||||
flagDelay = flag.Int("delay", 2, "delay in seconds")
|
||||
)
|
||||
|
||||
func main() {
|
||||
|
||||
flag.Parse()
|
||||
|
||||
c, errClient := client.NewHTTPClient(*flagAddr)
|
||||
if errClient != nil {
|
||||
log.Fatal(errClient)
|
||||
}
|
||||
|
||||
for i := 1; i <= *flagNum; i++ {
|
||||
|
||||
if *flagUpdate {
|
||||
go func(num int) {
|
||||
log.Println("start update")
|
||||
resp, errUpdate := c.Update()
|
||||
if errUpdate != nil {
|
||||
spew.Dump(resp)
|
||||
log.Fatal(errUpdate)
|
||||
}
|
||||
log.Println(num, "update done", resp)
|
||||
}(i)
|
||||
}
|
||||
|
||||
if *flagGetRepo {
|
||||
go func(num int) {
|
||||
log.Println("GetRepo", num)
|
||||
resp, err := c.GetRepo()
|
||||
if err != nil {
|
||||
// spew.Dump(resp)
|
||||
log.Fatal("failed to get repo")
|
||||
}
|
||||
log.Println(num, "GetRepo done, got", len(resp), "dimensions")
|
||||
}(i)
|
||||
}
|
||||
|
||||
time.Sleep(time.Duration(*flagDelay) * time.Second)
|
||||
}
|
||||
|
||||
log.Println("done!")
|
||||
}
|
||||
35
testing/server/server.go
Normal file
35
testing/server/server.go
Normal file
@ -0,0 +1,35 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"log"
|
||||
"net/http"
|
||||
)
|
||||
|
||||
type testServer struct {
|
||||
file string
|
||||
}
|
||||
|
||||
func main() {
|
||||
|
||||
var (
|
||||
flagJSONFile = flag.String("json-file", "", "provide a json source file")
|
||||
flagAddress = flag.String("addr", ":1234", "set the webserver address")
|
||||
)
|
||||
flag.Parse()
|
||||
|
||||
if *flagJSONFile == "" {
|
||||
log.Fatal("js source file must be provided")
|
||||
}
|
||||
|
||||
ts := &testServer{
|
||||
file: *flagJSONFile,
|
||||
}
|
||||
|
||||
log.Println("start test server at", *flagAddress, "serving file:", ts.file)
|
||||
log.Fatal(http.ListenAndServe(*flagAddress, ts))
|
||||
}
|
||||
|
||||
func (ts *testServer) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
http.ServeFile(w, r, ts.file)
|
||||
}
|
||||
Loading…
Reference in New Issue
Block a user