diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b61e171..a4e99cc 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -40,6 +40,14 @@ jobs: - name: Check Out Repo uses: actions/checkout@v2 + - name: Docker meta + id: meta + uses: docker/metadata-action@v3 + with: + images: foomo/contentfulproxy + tags: | + type=semver,pattern={{version}} + - name: Login to Docker Hub uses: docker/login-action@v1 with: @@ -57,7 +65,8 @@ jobs: context: ./ file: ./Dockerfile push: true - tags: ${{ secrets.DOCKER_HUB_USERNAME }}/contentfulproxy:latest + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} - name: Image digest run: echo ${{ steps.docker_build.outputs.digest }} diff --git a/Dockerfile b/Dockerfile index 56b0509..440812b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,6 +25,7 @@ RUN upx /src/bin/contentfulproxy ############################## ###### STAGE: PACKAGE ###### ############################## +# TODO: use non-root FROM alpine:latest ENV CONTENTFULPROXY_SERVER_ADDR=0.0.0.0:80 diff --git a/go.mod b/go.mod index 6f68fa5..bb7ab7f 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,8 @@ go 1.17 require ( github.com/foomo/keel v0.3.1 + github.com/prometheus/client_golang v1.10.0 + github.com/spf13/viper v1.7.1 github.com/stretchr/testify v1.7.0 go.uber.org/zap v1.19.1 ) @@ -27,7 +29,6 @@ require ( github.com/pelletier/go-toml v1.7.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_golang v1.10.0 // indirect github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.18.0 // indirect github.com/prometheus/procfs v0.6.0 // indirect @@ -36,7 +37,6 @@ require ( github.com/spf13/cast v1.3.0 // indirect github.com/spf13/jwalterweatherman v1.0.0 // indirect github.com/spf13/pflag v1.0.3 // indirect - github.com/spf13/viper v1.7.1 // indirect github.com/subosito/gotenv v1.2.0 // indirect go.opentelemetry.io/contrib v0.20.0 // indirect go.opentelemetry.io/contrib/instrumentation/host v0.20.0 // indirect diff --git a/packages/go/metrics/metrics.go b/packages/go/metrics/metrics.go new file mode 100644 index 0000000..33b4011 --- /dev/null +++ b/packages/go/metrics/metrics.go @@ -0,0 +1,15 @@ +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promauto" +) + +func NewCounter(name string, help string) prometheus.Counter { + return promauto.NewCounter(prometheus.CounterOpts{ + Namespace: "contentful", + Subsystem: "proxy", + Name: name, + Help: help, + }) +} diff --git a/proxy/cache.go b/proxy/cache.go index 7ed3512..19aee7a 100644 --- a/proxy/cache.go +++ b/proxy/cache.go @@ -85,8 +85,8 @@ func NewCache(l *zap.Logger, webHooks func() []string) *Cache { return c } -func getCacheIDForRequest(r *http.Request) cacheID { - id := r.URL.RequestURI() +func getCacheIDForRequest(r *http.Request, pathPrefix func() string) cacheID { + id := stripPrefixFromURL(r.URL.RequestURI(), pathPrefix) keys := make([]string, len(r.Header)) i := 0 for k := range r.Header { @@ -107,3 +107,7 @@ func getCacheIDForRequest(r *http.Request) cacheID { id = hex.EncodeToString(hash.Sum(nil)) return cacheID(id) } + +func stripPrefixFromURL(url string, pathPrefix func() string) string { + return strings.Replace(url, pathPrefix(), "", 1) +} diff --git a/proxy/jobs.go b/proxy/jobs.go index d659650..9c17c81 100644 --- a/proxy/jobs.go +++ b/proxy/jobs.go @@ -1,6 +1,11 @@ package proxy -import "net/http" +import ( + "net/http" + + "github.com/foomo/contentfulproxy/packages/go/log" + "go.uber.org/zap" +) type requestJobDone struct { cachedResponse *cachedResponse @@ -15,9 +20,12 @@ type requestJob struct { type jobRunner func(job requestJob, id cacheID) -func getJobRunner(c *Cache, backendURL func() string, chanJobDone chan requestJobDone) jobRunner { +func getJobRunner(l *zap.Logger, c *Cache, backendURL func() string, pathPrefix func() string, chanJobDone chan requestJobDone) jobRunner { return func(job requestJob, id cacheID) { - req, err := http.NewRequest("GET", backendURL()+job.request.URL.RequestURI(), nil) + // backend url is the contentful api domain like https://cdn.contenful.com + calledURL := backendURL() + stripPrefixFromURL(job.request.URL.RequestURI(), pathPrefix) + l.Info("URL called by job-runner", log.FURL(calledURL)) + req, err := http.NewRequest("GET", calledURL, nil) if err != nil { chanJobDone <- requestJobDone{ id: id, diff --git a/proxy/proxy.go b/proxy/proxy.go index e4749e3..e7698bd 100644 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -5,7 +5,12 @@ import ( "encoding/json" "net/http" + "github.com/foomo/contentfulproxy/packages/go/metrics" + + "github.com/prometheus/client_golang/prometheus" + "github.com/foomo/contentfulproxy/packages/go/log" + keellog "github.com/foomo/keel/log" "go.uber.org/zap" ) @@ -15,6 +20,12 @@ type Info struct { BackendURL string `json:"backendurl,omitempty"` } +type Metrics struct { + NumUpdate prometheus.Counter + NumProxyRequest prometheus.Counter + NumAPIRequest prometheus.Counter +} + type Proxy struct { l *zap.Logger cache *Cache @@ -22,11 +33,13 @@ type Proxy struct { pathPrefix func() string chanRequestJob chan requestJob chanFlushJob chan requestFlush + metrics *Metrics } func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case p.pathPrefix() + "/update": + p.metrics.NumUpdate.Inc() command := requestFlush("doit") p.chanFlushJob <- command return @@ -43,7 +56,8 @@ func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { switch r.Method { case http.MethodGet: p.l.Info("serve get request", zap.String("url", r.RequestURI)) - cacheID := getCacheIDForRequest(r) + p.metrics.NumProxyRequest.Inc() + cacheID := getCacheIDForRequest(r, p.pathPrefix) cachedResponse, ok := p.cache.get(cacheID) if !ok { chanDone := make(chan requestJobDone) @@ -53,14 +67,17 @@ func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { } jobDone := <-chanDone if jobDone.err != nil { - p.l.Error("Cache / job error", zap.String("url", r.RequestURI)) + keellog.WithError(p.l, jobDone.err).Error("Cache / job error") http.Error(w, "Cache / job error", http.StatusInternalServerError) return } cachedResponse = jobDone.cachedResponse p.l.Info("serve response after cache creation", log.FURL(r.RequestURI), log.FCacheID(string(cacheID))) + p.l.Info("length of response", keellog.FValue(len(cachedResponse.response))) + p.metrics.NumAPIRequest.Inc() } else { p.l.Info("serve response from cache", log.FURL(r.RequestURI), log.FCacheID(string(cacheID))) + p.l.Info("length of response", keellog.FValue(len(cachedResponse.response))) } for key, values := range cachedResponse.header { for _, value := range values { @@ -69,7 +86,7 @@ func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { } _, err := w.Write(cachedResponse.response) if err != nil { - p.l.Info("writing cached response failed", log.FURL(r.RequestURI), log.FCacheID(string(cacheID))) + keellog.WithError(p.l, err).Error("writing cached response failed", log.FCacheID(string(cacheID))) } default: http.Error(w, "method not allowed", http.StatusMethodNotAllowed) @@ -80,7 +97,7 @@ func NewProxy(ctx context.Context, l *zap.Logger, backendURL func() string, path chanRequest := make(chan requestJob) chanFlush := make(chan requestFlush) c := NewCache(l, webHooks) - go getLoop(ctx, l, backendURL, c, chanRequest, chanFlush) + go getLoop(ctx, l, backendURL, pathPrefix, c, chanRequest, chanFlush) return &Proxy{ l: l, cache: c, @@ -88,6 +105,7 @@ func NewProxy(ctx context.Context, l *zap.Logger, backendURL func() string, path pathPrefix: pathPrefix, chanRequestJob: chanRequest, chanFlushJob: chanFlush, + metrics: getMetrics(), }, nil } @@ -95,13 +113,14 @@ func getLoop( ctx context.Context, l *zap.Logger, backendURL func() string, + pathPrefix func() string, c *Cache, chanRequestJob chan requestJob, chanFlush chan requestFlush, ) { pendingRequests := map[cacheID][]chan requestJobDone{} chanJobDone := make(chan requestJobDone) - jobRunner := getJobRunner(c, backendURL, chanJobDone) + jobRunner := getJobRunner(l, c, backendURL, pathPrefix, chanJobDone) for { select { case <-chanFlush: @@ -109,7 +128,7 @@ func getLoop( c.update() c.callWebHooks() case nextJob := <-chanRequestJob: - cacheID := getCacheIDForRequest(nextJob.request) + cacheID := getCacheIDForRequest(nextJob.request, pathPrefix) pendingRequests[cacheID] = append(pendingRequests[cacheID], nextJob.chanDone) requests := pendingRequests[cacheID] if len(requests) == 1 { @@ -142,3 +161,11 @@ func jsonResponse(w http.ResponseWriter, v interface{}, statusCode int) { http.Error(w, "could not marshal info export", http.StatusInternalServerError) } } + +func getMetrics() *Metrics { + return &Metrics{ + NumUpdate: metrics.NewCounter("numupdates", "number of times the update webhook was called"), + NumAPIRequest: metrics.NewCounter("numapirequests", "number of times the proxy performed a contentful api-request"), + NumProxyRequest: metrics.NewCounter("numproxyrequests", "number of times the proxy received an api-request"), + } +}