improve docker build + prevent concurrent map read and map write on fs cache

Dockerfile

@@ -1,17 +1,50 @@
-FROM scratch
+# -----------------------------------------------------------------------------
+# Builder Base
+# -----------------------------------------------------------------------------
+FROM golang:alpine as base
 
-COPY bin/neosproxy-linux-amd64 /usr/sbin/neosproxy
+RUN apk add --no-cache git glide upx \
+  && rm -rf /var/cache/apk/*
 
-# install ca root certificates for outgoing https calls
-# https://curl.haxx.se/docs/caextract.html
-# http://blog.codeship.com/building-minimal-docker-containers-for-go-applications/
-#ADD https://curl.haxx.se/ca/cacert.pem /etc/ssl/certs/ca-certificates.crt
-COPY files/cacert.pem /etc/ssl/certs/ca-certificates.crt
+WORKDIR /go/src/github.com/foomo/neosproxy
 
-COPY files/tmp /tmp
+COPY glide.yaml glide.lock ./
+RUN glide install
 
-VOLUME /htdocs
+
+
+# -----------------------------------------------------------------------------
+# Builder
+# -----------------------------------------------------------------------------
+FROM base as builder
+
+COPY . ./
+
+# Build the binary
+RUN glide install
+RUN CGO_ENABLED=0 go build -o /go/bin/neosproxy cmd/neosproxy/main.go
+
+# Compress the binary
+RUN upx /go/bin/neosproxy
+
+
+
+# -----------------------------------------------------------------------------
+# Container
+# -----------------------------------------------------------------------------
+FROM alpine:latest
+
+RUN apk add --no-cache \
+    tzdata ca-certificates \
+  && rm -rf /var/cache/apk/*
+
+# Required for alpine image and golang
+RUN echo "hosts: files dns" > /etc/nsswitch.conf
+
+COPY --from=builder /go/bin/neosproxy /usr/local/bin/neosproxy
+
+VOLUME ["/var/data/neosproxy"]
 
 EXPOSE 80
 
-ENTRYPOINT ["/usr/sbin/neosproxy"]
+ENTRYPOINT ["/usr/local/bin/neosproxy"]

Makefile

@@ -13,21 +13,14 @@ clean:
 	rm -fv bin/neosp*
 build: clean
 	go build -o bin/neosproxy cmd/neosproxy/main.go
-build-arch: clean build-linux
+build-arch: clean
 	GOOS=darwin GOARCH=amd64 go build -o bin/neosproxy-darwin-amd64 cmd/neosproxy/main.go
 build-linux: clean
 	GOOS=linux GOARCH=amd64 go build -o bin/neosproxy-linux-amd64 cmd/neosproxy/main.go
-build-docker: clean build-arch prepare-docker
-	docker build -t foomo/neosproxy:latest .
+build-docker: clean
+	docker build -t foomo/neosproxy:dev .
 prepare-docker:
 	curl -o files/cacert.pem https://curl.haxx.se/ca/cacert.pem
-#release: clean build-linux prepare-docker
-#	git add -f files/cacert.pem
-#	git add -f bin/neosproxy-linux-amd64
-#	git commit -m 'build release candidate - new binary added for docker autobuild'
-#	@echo "-------------------------"
-#	@echo "please make sure that version number has been bumped, then tag and push the git repo"
-#	@echo "-------------------------"
 test:
 	go test ./...
 

cache/content/store/fs/filesystem.go

@@ -12,12 +12,6 @@ import (
 	"github.com/foomo/neosproxy/logging"
 )
 
-//------------------------------------------------------------------
-// ~ CONSTANTS / VARS
-//------------------------------------------------------------------
-
-var _ store.CacheStore = fsCacheStore{}
-
 //------------------------------------------------------------------
 // ~ TYPES
 //------------------------------------------------------------------
@@ -29,8 +23,6 @@ type fsCacheStore struct {
 	lock sync.Mutex
 	rw   map[string]*sync.RWMutex
 	l    logging.Entry
-
-	store.CacheStore
 }
 
 //------------------------------------------------------------------
@@ -60,35 +52,43 @@ func NewCacheStore(cacheDir string) store.CacheStore {
 // ~ PUBLIC METHODS
 //------------------------------------------------------------------
 
-func (f fsCacheStore) Upsert(item store.CacheItem) (e error) {
+func (f *fsCacheStore) Upsert(item store.CacheItem) (e error) {
+	// key
 	key := f.getItemKey(item)
-	cacheFile := f.Lock(key)
-	defer f.Unlock(key)
 
+	// serialize
 	bytes, errMarshall := json.Marshal(item)
 	if errMarshall != nil {
 		return errMarshall
 	}
 
+	// lock
+	cacheFile := f.Lock(key)
+	defer f.Unlock(key)
+
+	// write to file
 	return ioutil.WriteFile(cacheFile, bytes, 0644)
 }
 
-func (f fsCacheStore) Get(hash string) (item store.CacheItem, e error) {
+func (f *fsCacheStore) Get(hash string) (item store.CacheItem, e error) {
 	key := f.getKey(hash)
 	cacheFile, _ := f.RLock(key)
-	defer f.RUnlock(key)
 
 	if _, err := os.Stat(cacheFile); os.IsNotExist(err) {
+		f.RUnlock(key)
 		e = content.ErrorNotFound
 		return
 	}
 
 	bytes, errReadFile := ioutil.ReadFile(cacheFile)
 	if errReadFile != nil {
+		f.RUnlock(key)
 		e = errReadFile
 		return
 	}
 
+	f.RUnlock(key)
+
 	item = store.CacheItem{}
 	errUnmarshall := json.Unmarshal(bytes, &item)
 	if errUnmarshall != nil {
@@ -100,7 +100,7 @@ func (f fsCacheStore) Get(hash string) (item store.CacheItem, e error) {
 	return
 }
 
-func (f fsCacheStore) GetAll() (items []store.CacheItem, e error) {
+func (f *fsCacheStore) GetAll() (items []store.CacheItem, e error) {
 	files, errReadDir := ioutil.ReadDir(f.CacheDir)
 	if errReadDir != nil {
 		e = errReadDir
@@ -127,7 +127,7 @@ func (f fsCacheStore) GetAll() (items []store.CacheItem, e error) {
 	return
 }
 
-func (f fsCacheStore) Count() (int, error) {
+func (f *fsCacheStore) Count() (int, error) {
 	i := 0
 	files, err := ioutil.ReadDir(f.CacheDir)
 	if err != nil {
@@ -141,7 +141,7 @@ func (f fsCacheStore) Count() (int, error) {
 	return i, nil
 }
 
-func (f fsCacheStore) Remove(hash string) (e error) {
+func (f *fsCacheStore) Remove(hash string) (e error) {
 	key := f.getKey(hash)
 	cacheFile := f.Lock(key)
 	defer f.Unlock(key)
@@ -149,11 +149,11 @@ func (f fsCacheStore) Remove(hash string) (e error) {
 	return os.Remove(cacheFile)
 }
 
-func (f fsCacheStore) createCacheDir() error {
+func (f *fsCacheStore) createCacheDir() error {
 	return os.MkdirAll(f.CacheDir, 0755)
 }
 
-func (f fsCacheStore) RemoveAll() (e error) {
+func (f *fsCacheStore) RemoveAll() (e error) {
 	f.lock.Lock()
 	defer f.lock.Unlock()
 
@@ -176,10 +176,10 @@ func (f fsCacheStore) RemoveAll() (e error) {
 // ~ PRIVATE METHODS
 //------------------------------------------------------------------
 
-func (f fsCacheStore) getItemKey(item store.CacheItem) string {
+func (f *fsCacheStore) getItemKey(item store.CacheItem) string {
 	return f.getKey(item.Hash)
 }
 
-func (f fsCacheStore) getKey(hash string) string {
+func (f *fsCacheStore) getKey(hash string) string {
 	return hash + ".json"
 }

cache/content/store/fs/lock.go

@@ -86,17 +86,18 @@ func (f *fsCacheStore) hashKey(key string) string {
 
 func (f *fsCacheStore) rwLock(hashKey string) *sync.RWMutex {
 	f.lock.Lock()
-	defer f.lock.Unlock()
 
 	if f.rw == nil {
 		f.rw = make(map[string]*sync.RWMutex)
 	}
 
 	if result, ok := f.rw[hashKey]; ok {
+		f.lock.Unlock()
 		return result
 	}
 
 	var result sync.RWMutex
 	f.rw[hashKey] = &result
+	f.lock.Unlock()
 	return &result
 }

cache/content/store/memory/cachestore.go

@@ -7,12 +7,6 @@ import (
 	"github.com/foomo/neosproxy/cache/content/store"
 )
 
-//------------------------------------------------------------------
-// ~ VARIABLES
-//------------------------------------------------------------------
-
-var _ store.CacheStore = &memoryCacheStore{}
-
 //------------------------------------------------------------------
 // ~ TYPES
 //------------------------------------------------------------------
@@ -20,8 +14,6 @@ var _ store.CacheStore = &memoryCacheStore{}
 type memoryCacheStore struct {
 	items map[string]store.CacheItem
 	lock  *sync.RWMutex
-
-	store.CacheStore
 }
 
 //------------------------------------------------------------------

cache/content/store/mongostore/cachestore.go

@@ -16,15 +16,12 @@ import (
 
 const cacheStoreCollection = "cache"
 
-var _ store.CacheStore = mongoCacheStore{}
-
 //------------------------------------------------------------------
 // ~ TYPES
 //------------------------------------------------------------------
 
 type mongoCacheStore struct {
 	persistor *persistence.Persistor
-	store.CacheStore
 }
 
 //------------------------------------------------------------------

cache/content/store/mongostore/mongostore.go

@@ -12,12 +12,6 @@ type MongoStore interface {
 	store.Store
 }
 
-//------------------------------------------------------------------
-// ~ CONSTANTS / VARS
-//------------------------------------------------------------------
-
-// var _ MongoStore = mongoStores{}
-
 //------------------------------------------------------------------
 // ~ STRUCTS
 //------------------------------------------------------------------
@@ -42,7 +36,7 @@ func NewMongoStore(url string) (s MongoStore, e error) {
 
 	// init stores
 	s = &mongoStores{
-		cache: newCacheStore(cachePersistor),
+		cache: NewCacheStore(cachePersistor),
 	}
 
 	return

notifier/contentserver.go

@@ -2,11 +2,12 @@ package notifier
 
 import (
 	"bytes"
-	"crypto/tls"
 	"encoding/json"
 	"errors"
 	"net/http"
 	"net/url"
+
+	"github.com/foomo/neosproxy/utils"
 )
 
 var _ Notifier = &ContentServer{}
@@ -28,11 +29,7 @@ func NewContentServerNotifier(name string, endpoint *url.URL, token string, veri
 		CheckRedirect: func(req *http.Request, via []*http.Request) error {
 			return redirectAttemptedError
 		},
-		Transport: &http.Transport{
-			TLSClientConfig: &tls.Config{
-				InsecureSkipVerify: verifyTLS,
-			},
-		},
+		Transport: utils.GetDefaultTransport(verifyTLS),
 	}
 
 	return &ContentServer{

utils/http.go

@@ -1,6 +1,7 @@
 package utils
 
 import (
+	"crypto/tls"
 	"errors"
 	"net"
 	"net/http"
@@ -9,18 +10,22 @@ import (
 	"time"
 )
 
-func GetDefaultTransport() *http.Transport {
+func GetDefaultTransport(verifyTLS bool) *http.Transport {
 	return &http.Transport{
 		Proxy: http.ProxyFromEnvironment,
 		DialContext: (&net.Dialer{
-			Timeout:   30 * time.Second,
-			KeepAlive: 30 * time.Second,
+			Timeout:   10 * time.Second,
+			KeepAlive: 10 * time.Second,
 			DualStack: true,
 		}).DialContext,
-		MaxIdleConns:          100,
-		IdleConnTimeout:       90 * time.Second,
-		TLSHandshakeTimeout:   10 * time.Second,
+		MaxIdleConns:          50,
+		IdleConnTimeout:       15 * time.Second,
+		TLSHandshakeTimeout:   5 * time.Second,
 		ExpectContinueTimeout: 1 * time.Second,
+
+		TLSClientConfig: &tls.Config{
+			InsecureSkipVerify: !verifyTLS,
+		},
 	}
 }
 
@@ -30,7 +35,7 @@ func GetDefaultTransportFor(name string) (t *http.Transport, err error) {
 		err = errProxyFunc
 		return
 	}
-	t = GetDefaultTransport()
+	t = GetDefaultTransport(true)
 	if proxyFunc != nil {
 		t.Proxy = proxyFunc
 	}