foomo/contentfulproxy
1
0
Fork 0
mirror of https://github.com/foomo/contentfulproxy.git synced 2025-10-16 12:25:37 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

* add hashing of ids

Browse Source 
* remove request-id-header from id creation
This commit is contained in:
Daniel Thomas 2021-10-02 17:40:09 +02:00
parent b527b1273a
commit 51f2acff85
5 changed files with 38 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
go.mod
View File

@ -9,10 +9,11 @@ require (
require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/pkg/errors v0.9.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
go.uber.org/atomic v1.7.0 // indirect
go.uber.org/multierr v1.6.0 // indirect
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect
gopkg.in/yaml.v2 v2.3.0 // indirect
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b // indirect
github.com/foomo/keel v0.2.9
)

16
go.sum
View File

@ -1,11 +1,16 @@
github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA=
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@ -15,6 +20,7 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
go.uber.org/atomic v1.7.0 h1:ADUqmZGgLDDfbSL9ZmPxKTybcoEYHgpYfELNoN+7hsw=
go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc=
go.uber.org/goleak v1.1.11-0.20210813005559-691160354723 h1:sHOAIxRGBp443oHZIPB+HsUGaksVCXVQENPxwTfQdH4=
go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ=
go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
@ -45,11 +51,13 @@ golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b h1:h8qDotaEPuJATrMmW04NCwg7v22aHH28wwpauUhK9Oo=
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

12
proxy/cache.go
View File

@ -7,6 +7,8 @@ import (
"sort"
"strings"
"sync"
"crypto/md5"
"encoding/hex"
)
type cacheID string
@ -56,6 +58,7 @@ func (c *cache) flush() {
c.RLock()
defer c.RUnlock()
c.cacheMap = cacheMap{}
c.l.Info("flushed the cache", zap.Int("length", len(c.cacheMap)))
}
func (c *cache) callWebHooks() {
@ -78,8 +81,15 @@ func getCacheIDForRequest(r *http.Request) cacheID {
}
sort.Strings(keys)
for _, k := range keys {
id += k + strings.Join(r.Header[k], "-")
// each cf request is signed by an uuid in the X-Request-Id header
// we have to remove this from the ID-creation
if k != "X-Request-Id" {
id += k + strings.Join(r.Header[k], "-")
}
}
// hash it here maybe, to keep it shorter
hash := md5.New()
hash.Write([]byte(id))
id = hex.EncodeToString(hash.Sum(nil))
return cacheID(id)
}

17
proxy/proxy.go
View File

@ -13,11 +13,12 @@ type Proxy struct {
l *zap.Logger
cache *cache
backendURL string
pathPrefix string
chanRequestJob chan requestJob
chanFlushJob chan requestFlush
}
func NewProxy(ctx context.Context, l *zap.Logger, backendURL string, webHooks []WebHookURL) *Proxy {
func NewProxy(ctx context.Context, l *zap.Logger, backendURL string, pathPrefix string, webHooks []WebHookURL) (*Proxy, error) {
chanRequest := make(chan requestJob)
chanFlush := make(chan requestFlush)
c := &cache{
@ -30,9 +31,10 @@ func NewProxy(ctx context.Context, l *zap.Logger, backendURL string, webHooks []
l: l,
cache: c,
backendURL: backendURL,
pathPrefix: pathPrefix,
chanRequestJob: chanRequest,
chanFlushJob: chanFlush,
}
}, nil
}
func getLoop(
@ -49,8 +51,8 @@ func getLoop(
for {
select {
case command := <-chanFlush:
c.flush()
l.Info("cache flush command coming in", zap.String("flushCommand", string(command)))
c.flush()
c.callWebHooks()
case nextJob := <-chanRequestJob:
id := getCacheIDForRequest(nextJob.request)
@ -74,7 +76,7 @@ func getLoop(
func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
switch r.URL.Path {
case "/api/flush":
case p.pathPrefix + "/update":
command := requestFlush("doit")
p.chanFlushJob <- command
return
@ -93,13 +95,14 @@ func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
jobDone := <-chanDone
if jobDone.err != nil {
p.l.Error("cache / job error", zap.String("url", r.RequestURI))
http.Error(w, "cache / job error", http.StatusInternalServerError)
return
}
cachedResponse = jobDone.cachedResponse
p.l.Info("serve response after cache creation", zap.String("url", r.RequestURI))
p.l.Info("serve response after cache creation", zap.String("url", r.RequestURI), zap.String("cache id", string(cacheID)))
} else {
p.l.Info("serve response from cache", zap.String("url", r.RequestURI))
p.l.Info("serve response from cache", zap.String("url", r.RequestURI), zap.String("cache id", string(cacheID)))
}
for key, values := range cachedResponse.header {
for _, value := range values {
@ -108,7 +111,7 @@ func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
}
_, err := w.Write(cachedResponse.response)
if err != nil {
p.l.Info("writing cached response failed", zap.String("url", r.RequestURI))
p.l.Info("writing cached response failed", zap.String("url", r.RequestURI), zap.String("cache id", string(cacheID)))
}
default:
http.Error(w, "method not allowed", http.StatusMethodNotAllowed)

3
proxy/proxy_test.go
View File

@ -91,10 +91,11 @@ func getTestServer(t *testing.T) (gs func(path string) int, ws func(path string)
backendServer := httptest.NewServer(backendHandler)
webHookServer := httptest.NewServer(webHookHandler)
p := NewProxy(
p, _ := NewProxy(
context.Background(),
l,
backendServer.URL,
"",
[]WebHookURL{
WebHookURL(webHookServer.URL + "/update"),
WebHookURL(webHookServer.URL + "/flush"),