diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 0000000..d058469 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,15 @@ +# EditorConfig is awesome: https://EditorConfig.org + +# top-most EditorConfig file +root = true + +[*] +charset = utf-8 +indent_size = 2 +indent_style = tab +insert_final_newline = true +trim_trailing_whitespace = true +end_of_line = lf + +[*.{yaml,yml}] +indent_style = space diff --git a/.gitignore b/.gitignore index 242cc22..6123827 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,172 @@ -.* -!.git* -bin/contentfulproxy +# File created using '.gitignore Generator' for Visual Studio Code: https://bit.ly/vscode-gig + +# Created by https://www.toptal.com/developers/gitignore/api/visualstudiocode,macos,go,jetbrains+all +# Edit at https://www.toptal.com/developers/gitignore?templates=visualstudiocode,macos,go,jetbrains+all + +### Go ### +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib + +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (remove the comment below to include it) +# vendor/ + +### Go Patch ### +/vendor/ +/Godeps/ + +### JetBrains+all ### +# Covers JetBrains IDEs: IntelliJ, RubyMine, PhpStorm, AppCode, PyCharm, CLion, Android Studio, WebStorm and Rider +# Reference: https://intellij-support.jetbrains.com/hc/en-us/articles/206544839 + +# User-specific stuff +.idea/**/workspace.xml +.idea/**/tasks.xml +.idea/**/usage.statistics.xml +.idea/**/dictionaries +.idea/**/shelf + +# AWS User-specific +.idea/**/aws.xml + +# Generated files +.idea/**/contentModel.xml + +# Sensitive or high-churn files +.idea/**/dataSources/ +.idea/**/dataSources.ids +.idea/**/dataSources.local.xml +.idea/**/sqlDataSources.xml +.idea/**/dynamic.xml +.idea/**/uiDesigner.xml +.idea/**/dbnavigator.xml + +# Gradle +.idea/**/gradle.xml +.idea/**/libraries + +# Gradle and Maven with auto-import +# When using Gradle or Maven with auto-import, you should exclude module files, +# since they will be recreated, and may cause churn. Uncomment if using +# auto-import. +# .idea/artifacts +# .idea/compiler.xml +# .idea/jarRepositories.xml +# .idea/modules.xml +# .idea/*.iml +# .idea/modules +# *.iml +# *.ipr + +# CMake +cmake-build-*/ + +# Mongo Explorer plugin +.idea/**/mongoSettings.xml + +# File-based project format +*.iws + +# IntelliJ +out/ + +# mpeltonen/sbt-idea plugin +.idea_modules/ + +# JIRA plugin +atlassian-ide-plugin.xml + +# Cursive Clojure plugin +.idea/replstate.xml + +# Crashlytics plugin (for Android Studio and IntelliJ) +com_crashlytics_export_strings.xml +crashlytics.properties +crashlytics-build.properties +fabric.properties + +# Editor-based Rest Client +.idea/httpRequests + +# Android studio 3.1+ serialized cache file +.idea/caches/build_file_checksums.ser + +### JetBrains+all Patch ### +# Ignores the whole .idea folder and all .iml files +# See https://github.com/joeblau/gitignore.io/issues/186 and https://github.com/joeblau/gitignore.io/issues/360 + +.idea/ + +# Reason: https://github.com/joeblau/gitignore.io/issues/186#issuecomment-249601023 + +*.iml +modules.xml +.idea/misc.xml +*.ipr + +# Sonarlint plugin +.idea/sonarlint + +### macOS ### +# General +.DS_Store +.AppleDouble +.LSOverride + +# Icon must end with two \r +Icon + + +# Thumbnails +._* + +# Files that might appear in the root of a volume +.DocumentRevisions-V100 +.fseventsd +.Spotlight-V100 +.TemporaryItems +.Trashes +.VolumeIcon.icns +.com.apple.timemachine.donotpresent + +# Directories potentially created on remote AFP share +.AppleDB +.AppleDesktop +Network Trash Folder +Temporary Items +.apdisk + +### VisualStudioCode ### +.vscode/* +!.vscode/settings.json +!.vscode/tasks.json +!.vscode/launch.json +!.vscode/extensions.json +*.code-workspace + +# Local History for Visual Studio Code +.history/ + +### VisualStudioCode Patch ### +# Ignore all local history of files +.history +.ionide + +# Support for Project snippet scope +!.vscode/*.code-snippets + +# End of https://www.toptal.com/developers/gitignore/api/visualstudiocode,macos,go,jetbrains+all + +# Custom rules (everything added below won't be overriden by 'Generate .gitignore File' if you use 'Update' option) + +bin/* +!bin/.gitkeep diff --git a/.golangci.yml b/.golangci.yml new file mode 100644 index 0000000..8a769bf --- /dev/null +++ b/.golangci.yml @@ -0,0 +1,89 @@ +run: + timeout: 5m + +linters-settings: + gci: + local-prefixes: github.com/foomo/squadron + golint: + min-confidence: 0 + goimports: + local-prefixes: github.com/foomo/squadron + revive: + rules: + - name: indent-error-flow + disabled: true + gocritic: + enabled-tags: + - diagnostic + - style + disabled-tags: + - performance + - experimental + - opinionated + lll: + line-length: 200 + +linters: + disable-all: true + enable: + - bodyclose + - deadcode + - dogsled + - exhaustive + - exportloopref + - goconst + - gofmt + - gofumpt + - goimports + - revive + - goprintffuncname + - govet + - ineffassign + - misspell + - nakedret + - nolintlint + - prealloc + - rowserrcheck + - sqlclosecheck + - staticcheck + - structcheck + - stylecheck + - typecheck + - unconvert + - unparam + - unused + - varcheck + - whitespace + - errcheck + - gocritic + - gosimple + + - gocyclo + - gosec + - lll + - exportloopref + + # unused + # - gci + # - noctx + # - dupl + # - godot + # - gocognit + # - nlreturn + # - gochecknoglobals + # - gochecknoinits + # - depguard + # - goheader + # - gomodguard + + # don't enable: + # - asciicheck + # - funlen + # - godox + # - goerr113 + # - gomnd + # - interfacer + # - maligned + # - nestif + # - testpackage + # - wsl diff --git a/.goreleaser.yml b/.goreleaser.yml new file mode 100644 index 0000000..d803596 --- /dev/null +++ b/.goreleaser.yml @@ -0,0 +1,31 @@ +# .goreleaser.yml +# Build customization +builds: + - binary: contentfulproxy + main: ./cmd/contentfulproxy/main.go + env: + - CGO_ENABLED=0 + ldflags: + - -s -w + goos: + - windows + - darwin + - linux + goarch: + - amd64 + +# .goreleaser.yml +archives: + - format: tar.gz + format_overrides: + - goos: windows + format: zip + +brews: + # Reporitory to push the tap to. + - tap: + owner: foomo + name: homebrew-contentfulproxy + caveats: "contentfulproxy -webserver-address=$CONTENTFULPROXY_SERVER_ADDR" + homepage: "https://github.com/foomo/contentfulproxy" + description: "An experimental proxy for read access to contentful to save your API quota" diff --git a/cmd/contentfulproxy/main.go b/cmd/contentfulproxy/main.go index 5f69090..d3cd72a 100644 --- a/cmd/contentfulproxy/main.go +++ b/cmd/contentfulproxy/main.go @@ -33,7 +33,6 @@ func main() { webserverPath := config.DefaultWebserverPath(c) backendURL := config.DefaultBackendURL(c) - // create proxy proxy, _ := proxy.NewProxy( context.Background(), diff --git a/packages/go/log/log.go b/packages/go/log/log.go index b9e9030..3ea1191 100644 --- a/packages/go/log/log.go +++ b/packages/go/log/log.go @@ -3,9 +3,9 @@ package log import "go.uber.org/zap" const ( - ServiceRoutineKey = "service_routine" - CacheIdKey = "cache_id" - URLKey = "url" + ServiceRoutineKey = "service_routine" + CacheIDKey = "cache_id" + URLKey = "url" NumberOfWaitingClientsKey = "num_waiting_clients" ) @@ -13,8 +13,8 @@ func FServiceRoutine(name string) zap.Field { return zap.String(ServiceRoutineKey, name) } -func FCacheId(name string) zap.Field { - return zap.String(CacheIdKey, name) +func FCacheID(name string) zap.Field { + return zap.String(CacheIDKey, name) } func FURL(name string) zap.Field { diff --git a/proxy/cache.go b/proxy/cache.go index 27489d7..7ed3512 100644 --- a/proxy/cache.go +++ b/proxy/cache.go @@ -1,15 +1,16 @@ package proxy import ( - "crypto/md5" + "crypto/md5" // nolint:gosec "encoding/hex" - "github.com/foomo/contentfulproxy/packages/go/log" - "go.uber.org/zap" "io/ioutil" "net/http" "sort" "strings" "sync" + + "github.com/foomo/contentfulproxy/packages/go/log" + "go.uber.org/zap" ) type cacheID string @@ -66,10 +67,11 @@ func (c *Cache) callWebHooks() { for _, url := range c.webHooks() { go func(url string, l *zap.Logger) { l.Info("call webhook") - _, err := http.Get(url) + resp, err := http.Get(url) // nolint:gosec if err != nil { l.Error("error while calling webhook", zap.Error(err)) } + defer resp.Body.Close() }(url, c.l.With(log.FURL(url))) } } @@ -100,7 +102,7 @@ func getCacheIDForRequest(r *http.Request) cacheID { } } // hash it here maybe, to keep it shorter - hash := md5.New() + hash := md5.New() // nolint:gosec hash.Write([]byte(id)) id = hex.EncodeToString(hash.Sum(nil)) return cacheID(id) diff --git a/proxy/jobs.go b/proxy/jobs.go index 8d01900..d659650 100644 --- a/proxy/jobs.go +++ b/proxy/jobs.go @@ -26,7 +26,7 @@ func getJobRunner(c *Cache, backendURL func() string, chanJobDone chan requestJo return } for k, v := range job.request.Header { - req.Header.Set(k, string(v[0])) + req.Header.Set(k, v[0]) } client := http.Client{} resp, err := client.Do(req) diff --git a/proxy/proxy.go b/proxy/proxy.go index 2de088a..055893d 100644 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -3,11 +3,12 @@ package proxy import ( "context" "encoding/json" - "github.com/foomo/contentfulproxy/packages/go/log" "github.com/foomo/contentfulproxy/packages/go/metrics" "net/http" "github.com/prometheus/client_golang/prometheus" + + "github.com/foomo/contentfulproxy/packages/go/log" "go.uber.org/zap" ) @@ -69,10 +70,10 @@ func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } cachedResponse = jobDone.cachedResponse - p.l.Info("serve response after cache creation", log.FURL(r.RequestURI), log.FCacheId(string(cacheID))) + p.l.Info("serve response after cache creation", log.FURL(r.RequestURI), log.FCacheID(string(cacheID))) p.metrics.NumApiRequest.Inc() } else { - p.l.Info("serve response from cache", log.FURL(r.RequestURI), log.FCacheId(string(cacheID))) + p.l.Info("serve response from cache", log.FURL(r.RequestURI), log.FCacheID(string(cacheID))) } for key, values := range cachedResponse.header { for _, value := range values { @@ -81,7 +82,7 @@ func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) { } _, err := w.Write(cachedResponse.response) if err != nil { - p.l.Info("writing cached response failed", log.FURL(r.RequestURI), log.FCacheId(string(cacheID))) + p.l.Info("writing cached response failed", log.FURL(r.RequestURI), log.FCacheID(string(cacheID))) } default: http.Error(w, "method not allowed", http.StatusMethodNotAllowed) @@ -126,11 +127,11 @@ func getLoop( pendingRequests[cacheID] = append(pendingRequests[cacheID], nextJob.chanDone) requests := pendingRequests[cacheID] if len(requests) == 1 { - l.Info("starting jobrunner for", log.FURL(nextJob.request.RequestURI), log.FCacheId(string(cacheID))) + l.Info("starting jobrunner for", log.FURL(nextJob.request.RequestURI), log.FCacheID(string(cacheID))) go jobRunner(nextJob, cacheID) } case jobDone := <-chanJobDone: - l.Info("request complete", log.FCacheId(string(jobDone.id)), log.FNumberOfWaitingClients(len(pendingRequests[jobDone.id]))) + l.Info("request complete", log.FCacheID(string(jobDone.id)), log.FNumberOfWaitingClients(len(pendingRequests[jobDone.id]))) for _, chanPending := range pendingRequests[jobDone.id] { chanPending <- jobDone } diff --git a/proxy/proxy_test.go b/proxy/proxy_test.go index f64b659..d05324b 100644 --- a/proxy/proxy_test.go +++ b/proxy/proxy_test.go @@ -2,25 +2,27 @@ package proxy import ( "context" - "github.com/stretchr/testify/assert" - "go.uber.org/zap" "io/ioutil" "net/http" "net/http/httptest" "sync" "testing" "time" + + "github.com/stretchr/testify/assert" + "go.uber.org/zap" ) const ( - responseFoo = `i am a foo response` - responseBar = `i am bar response` + responseFoo = `i am a foo response` + responseBar = `i am bar response` responseUpdate = `update` - responseFlush = `update` + responseFlush = `update` ) type getStats func(path string) int +// func GetBackend(t *testing.T) (getStats, http.HandlerFunc) { stats := map[string]int{} statLock := sync.RWMutex{} @@ -82,7 +84,6 @@ func GetWebHook(t *testing.T) (getStats, http.HandlerFunc) { } func getTestServer(t *testing.T) (gs func(path string) int, ws func(path string) int, s *httptest.Server) { - l, _ := zap.NewProduction() gs, backendHandler := GetBackend(t) @@ -93,8 +94,8 @@ func getTestServer(t *testing.T) (gs func(path string) int, ws func(path string) p, _ := NewProxy( context.Background(), l, - func() string {return backendServer.URL}, - func() string {return ""}, + func() string { return backendServer.URL }, + func() string { return "" }, func() []string { return []string{ webHookServer.URL + "/test1", @@ -105,7 +106,6 @@ func getTestServer(t *testing.T) (gs func(path string) int, ws func(path string) s = httptest.NewServer(p) t.Log("we have a proxy in front of it running on", s.URL) return gs, ws, s - } func TestProxy(t *testing.T) { @@ -132,13 +132,14 @@ func TestProxy(t *testing.T) { } assert.Equal(t, 1, gs("/foo")) - // check the current status - //response, err := http.Get(server.URL + "/info") - //assert.NoError(t, err) + // response, err := http.Get(server.URL + "/info") + // assert.NoError(t, err) // - _, _ = http.Get(server.URL + "/update") + resp, err := http.Get(server.URL + "/update") + assert.NoError(t, err) + defer resp.Body.Close() time.Sleep(time.Second * 1)