* add hashing of ids

* remove request-id-header from id creation

go.mod

@@ -9,10 +9,11 @@ require (
 
 require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
+	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
-	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
+	gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
+	gopkg.in/yaml.v2 v2.3.0 // indirect
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
-	github.com/foomo/keel v0.2.9
 )

go.sum

@@ -1,11 +1,16 @@
+github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
 github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
-github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
+github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
 github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -15,6 +20,7 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
 go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
 go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
+go.uber.org/goleak v1.1.11-0.20210813005559-691160354723 h1:sHOAIxRGBp443oHZIPB+HsUGaksVCXVQENPxwTfQdH4=
 go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
 go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
 go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
@@ -45,11 +51,13 @@ golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
 golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
+gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
 gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

proxy/cache.go

@@ -7,6 +7,8 @@ import (
 	"sort"
 	"strings"
 	"sync"
+	"crypto/md5"
+	"encoding/hex"
 )
 
 type cacheID string
@@ -56,6 +58,7 @@ func (c *cache) flush() {
 	c.RLock()
 	defer c.RUnlock()
 	c.cacheMap = cacheMap{}
+	c.l.Info("flushed the cache", zap.Int("length", len(c.cacheMap)))
 }
 
 func (c *cache) callWebHooks() {
@@ -78,8 +81,15 @@ func getCacheIDForRequest(r *http.Request) cacheID {
 	}
 	sort.Strings(keys)
 	for _, k := range keys {
+		// each cf request is signed by an uuid in the X-Request-Id header
+		// we have to remove this from the ID-creation
+		if k != "X-Request-Id" {
 			id += k + strings.Join(r.Header[k], "-")
 		}
+	}
 	// hash it here maybe, to keep it shorter
+	hash := md5.New()
+	hash.Write([]byte(id))
+	id = hex.EncodeToString(hash.Sum(nil))
 	return cacheID(id)
 }

proxy/proxy.go

@@ -13,11 +13,12 @@ type Proxy struct {
 	l              *zap.Logger
 	cache          *cache
 	backendURL     string
+	pathPrefix     string
 	chanRequestJob chan requestJob
 	chanFlushJob   chan requestFlush
 }
 
-func NewProxy(ctx context.Context, l *zap.Logger, backendURL string, webHooks []WebHookURL) *Proxy {
+func NewProxy(ctx context.Context, l *zap.Logger, backendURL string, pathPrefix string, webHooks []WebHookURL) (*Proxy, error) {
 	chanRequest := make(chan requestJob)
 	chanFlush := make(chan requestFlush)
 	c := &cache{
@@ -30,9 +31,10 @@ func NewProxy(ctx context.Context, l *zap.Logger, backendURL string, webHooks []
 		l:              l,
 		cache:          c,
 		backendURL:     backendURL,
+		pathPrefix:     pathPrefix,
 		chanRequestJob: chanRequest,
 		chanFlushJob:   chanFlush,
-	}
+	}, nil
 }
 
 func getLoop(
@@ -49,8 +51,8 @@ func getLoop(
 	for {
 		select {
 		case command := <-chanFlush:
-			c.flush()
 			l.Info("cache flush command coming in", zap.String("flushCommand", string(command)))
+			c.flush()
 			c.callWebHooks()
 		case nextJob := <-chanRequestJob:
 			id := getCacheIDForRequest(nextJob.request)
@@ -74,7 +76,7 @@ func getLoop(
 
 func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	switch r.URL.Path {
-	case "/api/flush":
+	case p.pathPrefix + "/update":
 		command := requestFlush("doit")
 		p.chanFlushJob <- command
 		return
@@ -93,13 +95,14 @@ func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 			}
 			jobDone := <-chanDone
 			if jobDone.err != nil {
+				p.l.Error("cache / job error", zap.String("url", r.RequestURI))
 				http.Error(w, "cache / job error", http.StatusInternalServerError)
 				return
 			}
 			cachedResponse = jobDone.cachedResponse
-			p.l.Info("serve response after cache creation", zap.String("url", r.RequestURI))
+			p.l.Info("serve response after cache creation", zap.String("url", r.RequestURI), zap.String("cache id", string(cacheID)))
 		} else {
-			p.l.Info("serve response from cache", zap.String("url", r.RequestURI))
+			p.l.Info("serve response from cache", zap.String("url", r.RequestURI), zap.String("cache id", string(cacheID)))
 		}
 		for key, values := range cachedResponse.header {
 			for _, value := range values {
@@ -108,7 +111,7 @@ func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		}
 		_, err := w.Write(cachedResponse.response)
 		if err != nil {
-			p.l.Info("writing cached response failed", zap.String("url", r.RequestURI))
+			p.l.Info("writing cached response failed", zap.String("url", r.RequestURI), zap.String("cache id", string(cacheID)))
 		}
 	default:
 		http.Error(w, "method not allowed", http.StatusMethodNotAllowed)

proxy/proxy_test.go

@@ -91,10 +91,11 @@ func getTestServer(t *testing.T) (gs func(path string) int, ws func(path string)
 	backendServer := httptest.NewServer(backendHandler)
 	webHookServer := httptest.NewServer(webHookHandler)
 
-	p := NewProxy(
+	p, _ := NewProxy(
 		context.Background(),
 		l,
 		backendServer.URL,
+		"",
 		[]WebHookURL{
 			WebHookURL(webHookServer.URL + "/update"),
 			WebHookURL(webHookServer.URL + "/flush"),