diff --git a/Dockerfile b/Dockerfile index f9ba173..a0151be 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,16 +1,13 @@ # ----------------------------------------------------------------------------- # Builder Base # ----------------------------------------------------------------------------- -FROM golang:alpine as base +FROM golang:1.13-alpine as base -RUN apk add --no-cache git glide upx \ +RUN apk add --no-cache git upx \ && rm -rf /var/cache/apk/* WORKDIR /go/src/github.com/foomo/neosproxy -COPY glide.yaml glide.lock ./ -RUN glide install - # ----------------------------------------------------------------------------- @@ -21,7 +18,7 @@ FROM base as builder COPY . ./ # Build the binary -RUN glide install +RUN go mod vendor RUN CGO_ENABLED=0 go build -o /go/bin/neosproxy cmd/neosproxy/main.go # Compress the binary diff --git a/cache/content/constructor.go b/cache/content/constructor.go index 6abe193..546eb5d 100644 --- a/cache/content/constructor.go +++ b/cache/content/constructor.go @@ -1,23 +1,52 @@ package content import ( + "container/list" "time" "github.com/foomo/neosproxy/cache/content/store" "github.com/foomo/neosproxy/client/cms" + "github.com/foomo/neosproxy/logging" "golang.org/x/sync/singleflight" ) // New will return a newly created content cache -func New(cacheLifetime time.Duration, store store.CacheStore, loader cms.ContentLoader, observer Observer) *Cache { +func New(cacheLifetime time.Duration, store store.CacheStore, loader cms.ContentLoader, observer Observer, log logging.Entry) *Cache { c := &Cache{ observer: observer, loader: loader, store: store, - // invalidationChannel: make(chan InvalidationRequest), + cacheDependencies: NewCacheDependencies(), + invalidationRequestGroup: &singleflight.Group{}, + invalidationChannel: make(chan InvalidationRequest), + invalidationRetryChannel: make(chan InvalidationRequest), + retryQueue: &list.List{}, lifetime: cacheLifetime, + log: log, } + + // load cache dependencies + cacheDependencies, errCacheDependencies := c.store.GetAllCacheDependencies() + if errCacheDependencies != nil { + c.log.WithError(errCacheDependencies).Error("unable to init cache dependencies") + } + + // update cache dependencies + for _, obj := range cacheDependencies { + for _, targetID := range obj.Dependencies { + c.cacheDependencies.Set(obj.ID, targetID, obj.Dimension, obj.Workspace) + } + } + + // initialize invalidation workers + for w := 1; w <= 10; w++ { + go c.invalidationWorker(w) + } + + // initialize retry worker + c.runRetryWorker() + return c } diff --git a/cache/content/dependencies.go b/cache/content/dependencies.go new file mode 100644 index 0000000..d55caad --- /dev/null +++ b/cache/content/dependencies.go @@ -0,0 +1,71 @@ +package content + +import "sync" + +//----------------------------------------------------------------------------- +// ~ CACHE DEPENDENCIES for all dimensions in all workspaces +//----------------------------------------------------------------------------- + +type cacheDependencies struct { + dependencies map[string]*cacheDependency +} + +func NewCacheDependencies() *cacheDependencies { + return &cacheDependencies{ + dependencies: make(map[string]*cacheDependency, 4), + } +} + +func (c *cacheDependencies) getHash(dimension, workspace string) string { + return workspace + "_" + dimension +} + +func (c *cacheDependencies) Get(id, dimension, workspace string) []string { + hash := c.getHash(dimension, workspace) + if cache, ok := c.dependencies[hash]; ok { + return cache.Get(id) + } + return nil +} + +func (c *cacheDependencies) Set(sourceID, targetID, dimension, workspace string) { + hash := c.getHash(dimension, workspace) + if _, ok := c.dependencies[hash]; !ok { + c.dependencies[hash] = &cacheDependency{} + } + cache := c.dependencies[hash] + cache.Set(sourceID, targetID) + return +} + +//----------------------------------------------------------------------------- +// ~ CACHE DEPENDENCY for a dimension in a workspace +//----------------------------------------------------------------------------- + +type cacheDependency struct { + lock sync.RWMutex + dependencies map[string][]string +} + +func (c *cacheDependency) Get(id string) []string { + c.lock.RLock() + if dependencies, ok := c.dependencies[id]; ok { + c.lock.RUnlock() + return dependencies + } + c.lock.RUnlock() + return nil +} + +func (c *cacheDependency) Set(sourceID, targetID string) { + c.lock.Lock() + if c.dependencies == nil || len(c.dependencies) == 0 { + c.dependencies = make(map[string][]string) + } + if _, ok := c.dependencies[targetID]; !ok { + c.dependencies[targetID] = []string{} + } + c.dependencies[targetID] = append(c.dependencies[targetID], sourceID) + c.lock.Unlock() + return +} diff --git a/cache/content/dependencies_test.go b/cache/content/dependencies_test.go new file mode 100644 index 0000000..e8a39f3 --- /dev/null +++ b/cache/content/dependencies_test.go @@ -0,0 +1,22 @@ +package content + +import ( + "fmt" + "testing" +) + +func TestDependencies(t *testing.T) { + deps := NewCacheDependencies() + deps.Set("abc", "123", "de", "live") + d := deps.Get("123", "de", "live") + + fmt.Println(d) + + if len(d) != 1 { + t.Fatal("unexpected length") + } + + if d[0] != "abc" { + t.Fatal("unexpected dependency") + } +} diff --git a/cache/content/invalidator.go b/cache/content/invalidator.go index 1256f32..a22891a 100644 --- a/cache/content/invalidator.go +++ b/cache/content/invalidator.go @@ -1,10 +1,12 @@ package content import ( + "context" "strings" "time" "github.com/foomo/neosproxy/cache/content/store" + "github.com/sirupsen/logrus" ) // RemoveAll will reset whole cache by dropping all items @@ -12,43 +14,36 @@ func (c *Cache) RemoveAll() (err error) { return c.store.RemoveAll() } -func (c *Cache) invalidator(id, dimension, workspace string) (item store.CacheItem, err error) { - - // timer - start := time.Now() - - // load item - cmsContent, errGetContent := c.loader.GetContent(id, dimension, workspace) - if errGetContent != nil { - err = errGetContent - return +// Invalidate creates an invalidation job and adds it to the queue +// serveral workers will take care of job execution +func (c *Cache) Invalidate(id, dimension, workspace string) { + c.log.WithFields(logrus.Fields{ + "id": id, + "dimension": dimension, + "workspace": workspace, + }).Info("content cache invalidation request added to queue") + req := InvalidationRequest{ + CreatedAt: time.Now(), + ID: id, + Dimension: dimension, + Workspace: workspace, + ExecutionCounter: 0, } - - // prepare cache item - item = store.NewCacheItem(id, dimension, workspace, cmsContent.HTML, c.validUntil(cmsContent.ValidUntil)) - - // write item to cache - errUpsert := c.store.Upsert(item) - if errUpsert != nil { - err = errUpsert - return - } - - // notify observer - c.observer.Notify(InvalidationResponse{ - Item: item, - Duration: time.Since(start), - }) - - return + c.invalidationChannel <- req } -// Invalidate cache item -func (c *Cache) Invalidate(id, dimension, workspace string) (item store.CacheItem, err error) { +// Load will immediately load content from NEOS and persist it as a cache item +// no retry if it fails +func (c *Cache) Load(id, dimension, workspace string) (item store.CacheItem, err error) { groupName := strings.Join([]string{"invalidate", id, dimension, workspace}, "-") itemInterfaced, errThrottled, _ := c.invalidationRequestGroup.Do(groupName, func() (i interface{}, e error) { - return c.invalidator(id, dimension, workspace) + return c.invalidate(InvalidationRequest{ + CreatedAt: time.Now(), + ID: id, + Dimension: dimension, + Workspace: workspace, + }) }) if errThrottled != nil { @@ -60,6 +55,72 @@ func (c *Cache) Invalidate(id, dimension, workspace string) (item store.CacheIte return } +// invalidate cache item, load fresh content from NEOS +func (c *Cache) invalidate(req InvalidationRequest) (item store.CacheItem, err error) { + + // timer + start := time.Now() + + timeout := 10 * time.Second + if req.ExecutionCounter >= 5 { + timeout = 30 * time.Second + } + + // context + ctx, cancel := context.WithTimeout(context.Background(), timeout) + defer cancel() + + // load item + cmsContent, errGetContent := c.loader.GetContent(req.ID, req.Dimension, req.Workspace, ctx) + if errGetContent != nil { + err = errGetContent + return + } + + // update cache dependencies + if len(cmsContent.CacheDependencies) > 0 { + for _, targetID := range cmsContent.CacheDependencies { + c.cacheDependencies.Set(req.ID, targetID, req.Dimension, req.Workspace) + } + } + + // invalidate dependencies + dependencies := c.cacheDependencies.Get(req.ID, req.Dimension, req.Workspace) + if len(dependencies) > 0 { + for _, nodeID := range dependencies { + c.Invalidate(nodeID, req.Dimension, req.Workspace) + } + } + + // prepare cache item + item = store.NewCacheItem(req.ID, req.Dimension, req.Workspace, cmsContent.HTML, cmsContent.CacheDependencies, c.validUntil(cmsContent.ValidUntil)) + + // write item to cache + errUpsert := c.store.Upsert(item) + if errUpsert != nil { + err = errUpsert + return + } + + // logging + c.log.WithFields(logrus.Fields{ + "id": req.ID, + "dimension": req.Dimension, + "workspace": req.Workspace, + "retry": req.ExecutionCounter, + "createdAt": req.CreatedAt, + "waitTime": time.Since(req.CreatedAt).Seconds(), + }).WithDuration(start).Info("content cache invalidated") + + // notify observer + c.observer.Notify(InvalidationResponse{ + Item: item, + Duration: time.Since(start), + }) + + return +} + func (c *Cache) validUntil(validUntil int64) time.Time { now := time.Now() diff --git a/cache/content/store/cache.go b/cache/content/store/cache.go index d8d4575..d0cb466 100644 --- a/cache/content/store/cache.go +++ b/cache/content/store/cache.go @@ -10,6 +10,8 @@ type CacheStore interface { GetEtag(hash string) (etag string, e error) GetAllEtags(workspace string) (etags map[string]string) + GetAllCacheDependencies() ([]CacheDependencies, error) + Count() (int, error) Remove(hash string) (e error) diff --git a/cache/content/store/fs/filesystem.go b/cache/content/store/fs/filesystem.go index eea2c48..caf71a9 100644 --- a/cache/content/store/fs/filesystem.go +++ b/cache/content/store/fs/filesystem.go @@ -96,6 +96,44 @@ func (f *fsCacheStore) Upsert(item store.CacheItem) (e error) { return nil } +func (f *fsCacheStore) GetAllCacheDependencies() ([]store.CacheDependencies, error) { + start := time.Now() + l := f.l.WithField(logging.FieldFunction, "GetAllCacheDependencies") + files, errReadDir := ioutil.ReadDir(f.CacheDir) + if errReadDir != nil { + l.WithError(errReadDir).Error("failed reading cache dir") + return nil, errReadDir + } + + dependencies := []store.CacheDependencies{} + + counter := 0 + for _, file := range files { + if !file.IsDir() { + filename := file.Name() + index := strings.Index(filename, ".") + if index >= 0 { + filename = filename[0:index] + } + item, errGet := f.Get(filename) + if errGet != nil { + l.WithError(errGet).Warn("could not load cache item") + continue + } + counter++ + dependencies = append(dependencies, store.CacheDependencies{ + ID: item.ID, + Dimension: item.Dimension, + Workspace: item.Workspace, + Dependencies: item.Dependencies, + }) + } + } + l.WithField("len", counter).WithDuration(start).Debug("all cache dependencies loaded") + + return dependencies, nil +} + func (f *fsCacheStore) GetAllEtags(workspace string) (etags map[string]string) { f.lockEtags.RLock() etags = make(map[string]string) diff --git a/cache/content/store/fs/filesystem_test.go b/cache/content/store/fs/filesystem_test.go new file mode 100644 index 0000000..761a911 --- /dev/null +++ b/cache/content/store/fs/filesystem_test.go @@ -0,0 +1,33 @@ +package fs + +import ( + "github.com/foomo/neosproxy/cache/content/store" + "github.com/stretchr/testify/assert" + "io/ioutil" + "os" + "testing" +) + +func TestNewCacheStore(t *testing.T) { + dir, err := ioutil.TempDir("", "") + assert.NoError(t, err) + defer os.RemoveAll(dir) + + hash := "derp" + + s := NewCacheStore(dir) + item := store.CacheItem{ + Hash: hash, + ID: "123", + Dimension: "de", + Workspace: "live", + HTML: "", + } + err = s.Upsert(item) + assert.NoError(t, err) + + cachedItem, err := s.Get(hash) + assert.NoError(t, err) + assert.Equal(t, item, cachedItem) + +} diff --git a/cache/content/store/fs/lock.go b/cache/content/store/fs/lock.go index 581dcf5..822049c 100644 --- a/cache/content/store/fs/lock.go +++ b/cache/content/store/fs/lock.go @@ -86,18 +86,17 @@ func (f *fsCacheStore) hashKey(key string) string { func (f *fsCacheStore) rwLock(hashKey string) *sync.RWMutex { f.lock.Lock() + defer f.lock.Unlock() if f.rw == nil { f.rw = make(map[string]*sync.RWMutex) } if result, ok := f.rw[hashKey]; ok { - f.lock.Unlock() return result } var result sync.RWMutex f.rw[hashKey] = &result - f.lock.Unlock() return &result } diff --git a/cache/content/store/memory/cachestore_test.go b/cache/content/store/memory/cachestore_test.go index 8423cbe..1991ad4 100644 --- a/cache/content/store/memory/cachestore_test.go +++ b/cache/content/store/memory/cachestore_test.go @@ -20,7 +20,9 @@ func TestCache(t *testing.T) { assert.NoError(t, countErr) assert.Equal(t, 0, count) - item := store.NewCacheItem(id, dimension, workspace, "

Test

", validUntil) + dependencies := []string{"foo", "bar"} + + item := store.NewCacheItem(id, dimension, workspace, "

Test

", dependencies, validUntil) hash := item.Hash errUpsert := s.Upsert(item) assert.NoError(t, errUpsert) @@ -37,6 +39,8 @@ func TestCache(t *testing.T) { assert.NoError(t, errGet) assert.NotNil(t, itemCached) + assert.Equal(t, 2, len(itemCached.Dependencies)) + errRemoveAll := s.RemoveAll() assert.NoError(t, errRemoveAll) diff --git a/cache/content/store/vo.go b/cache/content/store/vo.go index 24367b5..508c24b 100644 --- a/cache/content/store/vo.go +++ b/cache/content/store/vo.go @@ -20,21 +20,30 @@ type CacheItem struct { created time.Time validUntil time.Time - HTML string - Etag string // hashed fingerprint of html content + HTML string + Etag string // hashed fingerprint of html content + Dependencies []string +} + +type CacheDependencies struct { + ID string + Dimension string + Workspace string + Dependencies []string } // NewCacheItem will create a new cache item -func NewCacheItem(id string, dimension string, workspace string, html string, validUntil time.Time) CacheItem { +func NewCacheItem(id string, dimension string, workspace string, html string, dependencies []string, validUntil time.Time) CacheItem { return CacheItem{ - Hash: GetHash(id, dimension, workspace), - ID: id, - Dimension: dimension, - Workspace: workspace, - created: time.Now(), - validUntil: validUntil, - HTML: html, - Etag: generateFingerprint(html), + Hash: GetHash(id, dimension, workspace), + ID: id, + Dimension: dimension, + Workspace: workspace, + created: time.Now(), + validUntil: validUntil, + HTML: html, + Etag: generateFingerprint(html), + Dependencies: dependencies, } } diff --git a/cache/content/vo.go b/cache/content/vo.go index bac5b5d..0dd1170 100644 --- a/cache/content/vo.go +++ b/cache/content/vo.go @@ -1,30 +1,41 @@ package content import ( + "container/list" "time" "github.com/foomo/neosproxy/cache/content/store" "github.com/foomo/neosproxy/client/cms" + "github.com/foomo/neosproxy/logging" "golang.org/x/sync/singleflight" ) // Cache workspace items type Cache struct { - observer Observer - loader cms.ContentLoader - store store.CacheStore - invalidationChannel chan InvalidationRequest - invalidationRequestGroup *singleflight.Group + observer Observer + loader cms.ContentLoader + store store.CacheStore - lifetime time.Duration // time until an item must be re-invalidated (< 0 === never) + invalidationRequestGroup *singleflight.Group + invalidationChannel chan InvalidationRequest + invalidationRetryChannel chan InvalidationRequest + retryQueue *list.List + + cacheDependencies *cacheDependencies + lifetime time.Duration // time until an item must be re-invalidated (< 0 === never) + + log logging.Entry } // InvalidationRequest request VO type InvalidationRequest struct { - CreatedAt time.Time ID string Dimension string Workspace string + + CreatedAt time.Time + LastExecutedAt time.Time + ExecutionCounter int } // InvalidationResponse response VO diff --git a/cache/content/worker.go b/cache/content/worker.go new file mode 100644 index 0000000..8c68eaa --- /dev/null +++ b/cache/content/worker.go @@ -0,0 +1,125 @@ +package content + +import ( + "container/list" + "sync" + "time" + + "github.com/foomo/neosproxy/client/cms" + "github.com/sirupsen/logrus" +) + +var retryWorkerSingleton sync.Once + +// runRetryWorker will run a singleton of a retry worker +// it will slow down a job retry and add it to the invalidation queue after some criterias are met +// but it will ensure that a job will be executed until it succeeds or failed more then 50 times in a row +func (c *Cache) runRetryWorker() { + retryWorkerSingleton.Do(func() { + go func() { + tick := time.Tick(10 * time.Second) + for { + select { + case <-tick: + before := time.Now().Add(-5 * time.Minute) + + if c.retryQueue.Len() > 0 { + var markedForDeletion bool + var prev *list.Element + last := c.retryQueue.Back() // last element + + // loop over the whole queue + for e := c.retryQueue.Front(); e != nil; e = e.Next() { + + req := e.Value.(InvalidationRequest) + + prev = e.Prev() + if prev == nil { + prev = c.retryQueue.Front() + } + + // remove previous element if marked for deletion + // we cannot immediately remove it, otherwise we would saw on the branch we sit on + if prev != nil && markedForDeletion { + c.retryQueue.Remove(prev) + markedForDeletion = false + } + + // less then 5 executions, please try it again + if req.ExecutionCounter < 5 { + c.invalidationChannel <- req + markedForDeletion = true + continue + } + + // older then 5 minutes => slow down, but retry + if req.LastExecutedAt.Before(before) { + c.invalidationChannel <- req + markedForDeletion = true + continue + } + + // it's the end of the world ... + if e == last { + break + } + } + // remove last item if marked for deletion + if markedForDeletion { + c.retryQueue.Remove(c.retryQueue.Back()) + } + } + + case req := <-c.invalidationRetryChannel: + // add a new job to the end of the line (retry queue) + c.retryQueue.PushBack(req) + } + } + }() + }) +} + +// invalidationWorkers will take care of invalidating the jobs which are in the queue +func (c *Cache) invalidationWorker(id int) { + for job := range c.invalidationChannel { + + // invalidate + _, err := c.invalidate(job) + + // well done + if err == nil { + continue + } + + // logger + l := c.log.WithFields(logrus.Fields{ + "id": job.ID, + "dimension": job.Dimension, + "workspace": job.Workspace, + "retry": job.ExecutionCounter, + "createdAt": job.CreatedAt, + "modifiedAt": job.LastExecutedAt, + "waitTime": time.Since(job.CreatedAt).Seconds(), + }).WithError(err) + + // too many executions => cancel that job + if job.ExecutionCounter >= 10 { + // @todo: inform in slack channel? + l.Warn("content cache invalidation failed to often - request will be ignored") + continue + } + + // unresolvable error => cancel that job + if err == cms.ErrorNotFound || err == cms.ErrorBadRequest { + // @todo: inform in slack channel? + l.Warn("content cache invalidation failed - request will be ignored") + continue + } + + // retry + job.LastExecutedAt = time.Now() + job.ExecutionCounter++ + c.invalidationRetryChannel <- job + l.Warn("content cache invalidation failed, retry job added to queue") + } +} diff --git a/cache/invalidator.go b/cache/invalidator.go index 89c5d5f..a770ff3 100644 --- a/cache/invalidator.go +++ b/cache/invalidator.go @@ -18,10 +18,10 @@ func (c *Cache) Invalidate() bool { select { case c.invalidationChannel <- time.Now(): - log.Info("invalidation request added to queue") + log.Info("contentserver export invalidation request added to queue") return true default: - log.Info("invalidation request ignored, queue seems to be full") + log.Info("contentserver export invalidation request ignored, queue seems to be full") return false } diff --git a/cache/scheduler.go b/cache/scheduler.go index 3f75784..29318a9 100644 --- a/cache/scheduler.go +++ b/cache/scheduler.go @@ -30,7 +30,7 @@ func (c *Cache) scheduleInvalidation() { if errInvalidation := c.cacheNeosContentServerExport(); errInvalidation != nil { if errInvalidation == ErrorNoNewExort { - log.WithDuration(requestTime).Info("cache invalidation request processed - but export hash matches old one") + log.WithDuration(requestTime).Info("contentserver export cache invalidation request processed - but export hash matches old one") continue } @@ -40,7 +40,7 @@ func (c *Cache) scheduleInvalidation() { // @todo: notify observers - log.WithDuration(requestTime).Info("cache invalidation request processed") + log.WithDuration(requestTime).Info("contentserver export cache invalidation request processed") } } diff --git a/client/cms/client.go b/client/cms/client.go index ac5a646..fd7ea8b 100644 --- a/client/cms/client.go +++ b/client/cms/client.go @@ -2,6 +2,7 @@ package cms import ( "bytes" + "context" "encoding/json" "io/ioutil" "net/http" @@ -27,7 +28,7 @@ func New(endpoint string) (*Client, error) { endpointURL, err := url.Parse(endpoint) if err != nil { - return nil, errors.Wrap(err, "could not parse the url from 'endpoint'") + return nil, errors.Wrap(err, "could not parse url from 'endpoint'") } transport, transportErr := utils.GetDefaultTransportFor("CMS") @@ -36,7 +37,7 @@ func New(endpoint string) (*Client, error) { } httpClient := &http.Client{ - Timeout: time.Second * 5, + Timeout: time.Second * 30, Transport: transport, } @@ -76,11 +77,10 @@ func (c *Client) NewGetRequest(path string, body interface{}) (*http.Request, er } //Do will send an API request and returns the API response (decodes and serializes) -func (c *Client) Do(req *http.Request, v interface{}) *ClientError { - - resp, err := c.client.Do(req) +func (c *Client) Do(req *http.Request, ctx context.Context, v interface{}) *ClientError { + resp, err := c.client.Do(req.WithContext(ctx)) if err != nil { - return CreateClientError(errors.Wrap(err, "could not create a new request"), req, resp, nil) + return CreateClientError(errors.Wrap(err, "could not execute request"), req, resp, nil) } defer resp.Body.Close() diff --git a/client/cms/client_test.go b/client/cms/client_test.go index 4c4d0bf..55e461e 100644 --- a/client/cms/client_test.go +++ b/client/cms/client_test.go @@ -1,11 +1,13 @@ package cms import ( + "context" "encoding/json" "fmt" "net/http" "net/http/httptest" "testing" + "time" "github.com/stretchr/testify/assert" ) @@ -43,9 +45,57 @@ func TestClient(t *testing.T) { client, clientErr := New(ts.URL) assert.NoError(t, clientErr, "client must be initialised without errors") - content, contentErr := client.CMS.GetContent(id, dimension, workspace) - assert.NoError(t, contentErr) + // context + ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*500) + defer cancel() + content, contentErr := client.CMS.GetContent(id, dimension, workspace, ctx) + assert.NoError(t, contentErr) assert.NotEmpty(t, content) assert.Equal(t, "

Test

", content.HTML) } + +func TestClientTimeout(t *testing.T) { + + id := "a839f683-dc58-47aa-8000-72d5b6fdeb85" + dimension := "de" + workspace := "stage" + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + + path := fmt.Sprintf(pathContent, dimension, id, workspace) + + if r.RequestURI != path { + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte("error: invalid request uri: " + r.RequestURI)) + return + } + + time.Sleep(time.Second * 1) + + data := &Content{ + HTML: "

Test

", + } + + encoder := json.NewEncoder(w) + err := encoder.Encode(data) + if err != nil { + w.WriteHeader(http.StatusInternalServerError) + w.Write([]byte("error: " + err.Error())) + return + } + })) + defer ts.Close() + + client, clientErr := New(ts.URL) + assert.NoError(t, clientErr, "client must be initialised without errors") + + // context + ctx, cancel := context.WithTimeout(context.Background(), time.Millisecond*500) + defer cancel() + + content, contentErr := client.CMS.GetContent(id, dimension, workspace, ctx) + assert.Error(t, contentErr) + assert.Empty(t, content) + assert.Equal(t, ErrorResponseTimeout, contentErr) +} diff --git a/client/cms/cms.go b/client/cms/cms.go index 6ea91d3..fe1ba83 100644 --- a/client/cms/cms.go +++ b/client/cms/cms.go @@ -1,12 +1,14 @@ package cms import ( + "context" "fmt" "net/http" "sync" "github.com/foomo/neosproxy/cache/content/store" "github.com/foomo/neosproxy/logging" + "github.com/sirupsen/logrus" ) //----------------------------------------------------------------------------- @@ -72,9 +74,14 @@ func NewCMSService(defaultClient *Client) Service { //----------------------------------------------------------------------------- // GetContent from NEOS CMS as html string -func (s *cmsService) GetContent(id string, dimension string, workspace string) (content Content, e error) { +func (s *cmsService) GetContent(id string, dimension string, workspace string, ctx context.Context) (content Content, e error) { - l := s.logger.WithField(logging.FieldFunction, "GetContent") + l := s.logger.WithFields(logrus.Fields{ + logging.FieldFunction: "GetContent", + "id": id, + "dimension": dimension, + "workspace": workspace, + }) var clientErr error path := fmt.Sprintf(pathContent, dimension, id, workspace) @@ -86,7 +93,7 @@ func (s *cmsService) GetContent(id string, dimension string, workspace string) ( } content = Content{} - e, clientErr = s.convertClientErr(s.client.Do(req, &content)) + e, clientErr = s.convertClientErr(s.client.Do(req, ctx, &content), ctx) if clientErr != nil { l.WithError(clientErr).Error("unable to load html content from cms") return @@ -99,13 +106,27 @@ func (s *cmsService) GetContent(id string, dimension string, workspace string) ( // ~ PRIVATE METHODS //----------------------------------------------------------------------------- -func (s *cmsService) convertClientErr(clientErr *ClientError) (error, error) { +func (s *cmsService) convertClientErr(clientErr *ClientError, ctx context.Context) (error, error) { if clientErr == nil { return nil, nil } - if clientErr.Response.StatusCode == http.StatusServiceUnavailable { + if ctx != nil && ctx.Err() != nil && ctx.Err() == context.DeadlineExceeded { + return ErrorResponseTimeout, clientErr.Error + } + if ctx != nil && ctx.Err() != nil && ctx.Err() == context.Canceled { + return ErrorRequest, clientErr.Error + } + + switch clientErr.Response.StatusCode { + case http.StatusServiceUnavailable: return ErrorMaintenance, clientErr.Error + case http.StatusNotFound: + return ErrorNotFound, clientErr.Error + case http.StatusBadRequest: + return ErrorBadRequest, clientErr.Error + case http.StatusInternalServerError: + return ErrorInternalServerError, clientErr.Error } return ErrorResponse, clientErr.Error diff --git a/client/cms/errors.go b/client/cms/errors.go index 6a2f0c0..1d910f0 100644 --- a/client/cms/errors.go +++ b/client/cms/errors.go @@ -7,10 +7,14 @@ import ( ) var ( - ErrorUnknown = errors.New("unknown error occured") - ErrorRequest = errors.New("request error") - ErrorResponse = errors.New("response error") - ErrorMaintenance = errors.New("cms in maintenance mode") + ErrorUnknown = errors.New("unknown error occured") + ErrorRequest = errors.New("request error") + ErrorResponse = errors.New("response error") + ErrorResponseTimeout = errors.New("cms response timeout") + ErrorMaintenance = errors.New("cms in maintenance mode") + ErrorNotFound = errors.New("resource not found") + ErrorBadRequest = errors.New("bad request") + ErrorInternalServerError = errors.New("internal server error") ) type ClientError struct { diff --git a/client/cms/interfaces.go b/client/cms/interfaces.go index 7464624..0e7e296 100644 --- a/client/cms/interfaces.go +++ b/client/cms/interfaces.go @@ -1,12 +1,14 @@ package cms +import "context" + //----------------------------------------------------------------------------- // ~ INTERFACES //----------------------------------------------------------------------------- // Service to load type Service interface { - GetContent(id string, dimension string, workspace string) (content Content, e error) + GetContent(id string, dimension string, workspace string, ctx context.Context) (content Content, e error) // GetRepo(id string, dimension string) (html string, e error) // GetImage(id string, dimension string) (html string, e error) @@ -15,5 +17,5 @@ type Service interface { // ContentLoader interface type ContentLoader interface { - GetContent(id, dimension, workspace string) (content Content, e error) + GetContent(id, dimension, workspace string, ctx context.Context) (content Content, e error) } diff --git a/client/cms/vo.go b/client/cms/vo.go index 46dd610..364db5b 100644 --- a/client/cms/vo.go +++ b/client/cms/vo.go @@ -16,6 +16,7 @@ type Client struct { // Content returned on getContent type Content struct { - HTML string `json:"html"` - ValidUntil int64 `json:"validUntil"` + HTML string `json:"html"` + ValidUntil int64 `json:"validUntil"` + CacheDependencies []string `json:"cacheDependencies"` } diff --git a/go.mod b/go.mod index ac61d94..7a7914f 100644 --- a/go.mod +++ b/go.mod @@ -3,17 +3,22 @@ module github.com/foomo/neosproxy go 1.12 require ( + code.cloudfoundry.org/bytefmt v0.0.0-20190819182555-854d396b647c // indirect github.com/auth0/go-jwt-middleware v0.0.0-20170425171159-5493cabe49f7 github.com/cloudfoundry/bytefmt v0.0.0-20180906201452-2aa6f33b730c - github.com/dgrijalva/jwt-go v3.2.0+incompatible + github.com/codegangsta/negroni v1.0.0 // indirect + github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect github.com/foomo/shop v0.0.0-20190306093145-644b0b683ba1 + github.com/gorilla/context v1.1.1 // indirect github.com/gorilla/mux v1.6.2 + github.com/olebedev/emitter v0.0.0-20190110104742-e8d1457e6aee + github.com/onsi/ginkgo v1.10.2 // indirect + github.com/onsi/gomega v1.7.0 // indirect github.com/pkg/errors v0.0.0-20181023235946-059132a15dd0 github.com/sirupsen/logrus v1.2.0 + github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 // indirect github.com/stretchr/testify v1.2.2 - golang.org/x/crypto v0.0.0-20190103213133-ff983b9c42bc - golang.org/x/sync v0.0.0-20170517211232-f52d1811a629 - golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33 + golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce gopkg.in/yaml.v2 v2.2.2 ) diff --git a/go.sum b/go.sum index b822b8d..d358b59 100644 --- a/go.sum +++ b/go.sum @@ -1,30 +1,77 @@ +code.cloudfoundry.org/bytefmt v0.0.0-20190819182555-854d396b647c h1:2RuXx1+tSNWRjxhY0Bx52kjV2odJQ0a6MTbfTPhGAkg= +code.cloudfoundry.org/bytefmt v0.0.0-20190819182555-854d396b647c/go.mod h1:wN/zk7mhREp/oviagqUXY3EwuHhWyOvAdsn5Y4CzOrc= github.com/auth0/go-jwt-middleware v0.0.0-20170425171159-5493cabe49f7 h1:irR1cO6eek3n5uquIVaRAsQmZnlsfPuHNz31cXo4eyk= github.com/auth0/go-jwt-middleware v0.0.0-20170425171159-5493cabe49f7/go.mod h1:LWMyo4iOLWXHGdBki7NIht1kHru/0wM179h+d3g8ATM= github.com/cloudfoundry/bytefmt v0.0.0-20180906201452-2aa6f33b730c h1:zE9z4EZZwJTjOi9Q9WYM/81BuwOKyjhHagiNUDhDdnI= github.com/cloudfoundry/bytefmt v0.0.0-20180906201452-2aa6f33b730c/go.mod h1:4oo6ExqTPaBVBwSm814h6UO5Fels1kN2KvpNscaCcS0= +github.com/codegangsta/negroni v1.0.0 h1:+aYywywx4bnKXWvoWtRfJ91vC59NbEhEY03sZjQhbVY= +github.com/codegangsta/negroni v1.0.0/go.mod h1:v0y3T5G7Y1UlFfyxFn/QLRU4a2EuNau2iZY63YTKWo0= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/foomo/shop v0.0.0-20190306093145-644b0b683ba1 h1:BSbbitW3EfDmDVW4BOwB/xWCCFheimcBZHZPO9rSaRM= github.com/foomo/shop v0.0.0-20190306093145-644b0b683ba1/go.mod h1:+Y2nUdyvktarq9+F2B0tvk/BM7y+jwNJI1CW/KgM7as= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/context v1.1.1 h1:AWwleXJkX/nhcU9bZSnZoi3h/qGYqQAGhq6zZe/aQW8= +github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.6.2 h1:Pgr17XVTNXAk3q/r4CpKzC5xBM/qW1uVLV+IhRZpIIk= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= +github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/olebedev/emitter v0.0.0-20190110104742-e8d1457e6aee h1:IquUs3fIykn10zWDIyddanhpTqBvAHMaPnFhQuyYw5U= +github.com/olebedev/emitter v0.0.0-20190110104742-e8d1457e6aee/go.mod h1:eT2/Pcsim3XBjbvldGiJBvvgiqZkAFyiOJJsDKXs/ts= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.2 h1:uqH7bpe+ERSiDa34FDOF7RikN6RzXgduUF8yarlZp94= +github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.7.0 h1:XPnZz8VVBHjVsy1vzJmRwIcSwiUO+JFfrv/xGiigmME= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/pkg/errors v0.0.0-20181023235946-059132a15dd0 h1:R+lX9nKwNd1n7UE5SQAyoorREvRn3aLF6ZndXBoIWqY= github.com/pkg/errors v0.0.0-20181023235946-059132a15dd0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/sirupsen/logrus v1.2.0 h1:juTguoYk5qI21pwyTXY3B3Y5cOTH3ZUyZCg1v/mihuo= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337 h1:WN9BUFbdyOsSH/XohnWpXOlq9NBD5sGAB2FciQMUEe8= +github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190103213133-ff983b9c42bc h1:F5tKCVGp+MUAHhKp5MZtGqAlGX3+oCsiL1Q629FL90M= -golang.org/x/crypto v0.0.0-20190103213133-ff983b9c42bc/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/sync v0.0.0-20170517211232-f52d1811a629 h1:wqoYUzeICxRnvJCvfHTh0OY0VQ6xern7nYq+ccc19e4= -golang.org/x/sync v0.0.0-20170517211232-f52d1811a629/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20171031081856-95c657629925/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33 h1:I6FyU15t786LL7oL/hn43zqTuEGr4PN7F4XJ1p4E3Y8= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a h1:1BGLXjeY4akVXGgbC9HugT3Jv3hCI0z56oJR5vAMgBU= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce h1:xcEWjVhvbDy+nHP67nPDDpbYrY+ILlfndk4bRioVHaU= gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/proxy/api.go b/proxy/api.go index 2c94259..503e382 100644 --- a/proxy/api.go +++ b/proxy/api.go @@ -9,12 +9,12 @@ import ( "github.com/foomo/neosproxy/cache/content/store" - "github.com/sirupsen/logrus" "github.com/cloudfoundry/bytefmt" "github.com/foomo/neosproxy/cache" "github.com/foomo/neosproxy/client/cms" "github.com/foomo/neosproxy/logging" "github.com/gorilla/mux" + "github.com/sirupsen/logrus" "gopkg.in/yaml.v2" content_cache "github.com/foomo/neosproxy/cache/content" @@ -75,10 +75,10 @@ func (p *Proxy) getContent(w http.ResponseWriter, r *http.Request) { // invalidate content startInvalidation := time.Now() - itemInvalidated, errCacheInvalidate := p.contentCache.Invalidate(id, dimension, workspace) + itemInvalidated, errCacheInvalidate := p.contentCache.Load(id, dimension, workspace) if errCacheInvalidate != nil { w.WriteHeader(http.StatusInternalServerError) - log.WithError(errCacheInvalidate).Error("cache invalidation failed") + log.WithError(errCacheInvalidate).Error("serving uncached item failed") return } log.WithDuration(startInvalidation).WithField("len", p.contentCache.Len()).Debug("invalidated content item") @@ -88,7 +88,8 @@ func (p *Proxy) getContent(w http.ResponseWriter, r *http.Request) { // prepare response data data := &cms.Content{ - HTML: item.HTML, + HTML: item.HTML, + CacheDependencies: item.Dependencies, } w.Header().Set("ETag", item.GetEtag()) @@ -103,7 +104,8 @@ func (p *Proxy) getContent(w http.ResponseWriter, r *http.Request) { } // done - log.WithDuration(start).Debug("content served") + // log.WithDuration(start).Debug("content served") + p.servedStatsChan <- true return } @@ -144,13 +146,8 @@ func (p *Proxy) invalidateCache(w http.ResponseWriter, r *http.Request) { for _, workspace := range workspaces { for _, dimension := range p.config.Neos.Dimensions { - // @todo use channels and workers!!! - go func(id, dimension, workspace string) { - _, errInvalidate := p.contentCache.Invalidate(id, dimension, workspace) - if errInvalidate != nil { - log.WithError(errInvalidate).WithField(logging.FieldDimension, dimension).Error("invalidate content cache failed") - } - }(id, dimension, workspace) + // add invalidation request / job / task + p.contentCache.Invalidate(id, dimension, workspace) } // load workspace worker diff --git a/proxy/constructor.go b/proxy/constructor.go index 04b9227..50a3142 100644 --- a/proxy/constructor.go +++ b/proxy/constructor.go @@ -33,11 +33,26 @@ func New(cfg *config.Config, contentLoader cms.ContentLoader, contentStore store ProviderReports: map[string]model.Report{}, ConsumerReports: map[string]model.Report{}, }, - broker: notifier.NewBroker(), + broker: notifier.NewBroker(), + servedStatsChan: make(chan bool), + servedStatsCounter: uint(0), } + go func() { + for { + tick := time.Tick(time.Minute * 1) + select { + case <-tick: + p.log.WithField("requests", p.servedStatsCounter).Info("requests served in the last 60 seconds") + p.servedStatsCounter = uint(0) + case <-p.servedStatsChan: + p.servedStatsCounter++ + } + } + }() + // content cache for html from neos - p.contentCache = content_cache.New(cacheLifetime, contentStore, contentLoader, p.broker) + p.contentCache = content_cache.New(cacheLifetime, contentStore, contentLoader, p.broker, p.log) // sitemap / site structure cache for content servers for _, workspace := range cfg.Neos.Workspaces { diff --git a/proxy/vo.go b/proxy/vo.go index bd7a75b..a35d8bd 100644 --- a/proxy/vo.go +++ b/proxy/vo.go @@ -27,6 +27,9 @@ type Proxy struct { status *model.Status broker *notifier.Broker + + servedStatsChan chan bool + servedStatsCounter uint // served requests per minute } type basicAuth struct {