added history cleanup

repo/history.go

@@ -1,13 +1,18 @@
 package repo
 
 import (
+	"fmt"
 	"io/ioutil"
+	"os"
 	"path"
+	"sort"
+	"strings"
 	"time"
 )
 
 const historyRepoJSONPrefix = "contentserver-repo-"
 const historyRepoJSONSuffix = ".json"
+const maxHistoryVersions = 20
 
 type history struct {
 	varDir string
@@ -30,6 +35,41 @@ func (h *history) add(jsonBytes []byte) error {
 	return ioutil.WriteFile(h.getCurrentFilename(), jsonBytes, 0644)
 }
 
+func (h *history) getHistory() (files []string, err error) {
+	files = []string{}
+	fileInfos, err := ioutil.ReadDir(h.varDir)
+	if err != nil {
+		return
+	}
+	currentName := h.getCurrentFilename()
+	for _, f := range fileInfos {
+		if !f.IsDir() {
+			filename := f.Name()
+			if filename != currentName && (strings.HasPrefix(filename, historyRepoJSONPrefix) && strings.HasSuffix(filename, historyRepoJSONSuffix)) {
+				files = append(files, path.Join(h.varDir, filename))
+			}
+		}
+	}
+	sort.Strings(files)
+	return
+}
+
+func (h *history) cleanup() error {
+	files, err := h.getHistory()
+	if err != nil {
+		return err
+	}
+	if len(files) > maxHistoryVersions {
+		for i := maxHistoryVersions; i < len(files); i++ {
+			err := os.Remove(files[i])
+			if err != nil {
+				return fmt.Errorf("could not remove file %q got %q", files[i], err)
+			}
+		}
+	}
+	return nil
+}
+
 func (h *history) getCurrentFilename() string {
 	return path.Join(h.varDir, historyRepoJSONPrefix+"current"+historyRepoJSONSuffix)
 }

repo/history_test.go

@@ -0,0 +1,47 @@
+package repo
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"testing"
+	"time"
+)
+
+func testHistory() *history {
+	tempDir, err := ioutil.TempDir(os.TempDir(), "contentserver-history-test")
+	if err != nil {
+		panic(err)
+	}
+	return newHistory(tempDir)
+}
+
+func TestHistoryCurrent(t *testing.T) {
+	h := testHistory()
+	test := []byte("test")
+	h.add(test)
+	current, err := h.getCurrent()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if bytes.Compare(current, test) != 0 {
+		t.Fatal(fmt.Sprintf("expected %q, got %q", string(test), string(current)))
+	}
+}
+
+func TestHistoryCleanup(t *testing.T) {
+	h := testHistory()
+	for i := 0; i < 50; i++ {
+		h.add([]byte(fmt.Sprint(i)))
+		time.Sleep(time.Millisecond * 5)
+	}
+	h.cleanup()
+	files, err := h.getHistory()
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(files) != maxHistoryVersions {
+		t.Fatal("history too long", len(files), "instead of", maxHistoryVersions)
+	}
+}

repo/loader.go

@@ -10,7 +10,6 @@ import (
 
 	"github.com/foomo/contentserver/content"
 	"github.com/foomo/contentserver/log"
-	"github.com/foomo/contentserver/responses"
 )
 
 func (repo *Repo) updateRoutine() {
@@ -101,46 +100,6 @@ func loadNodesFromJSON(jsonBytes []byte) (nodes map[string]*content.RepoNode, er
 	return nodes, err
 }
 
-// Update - reload contents of repository with json from repo.server
-func (repo *Repo) Update() (updateResponse *responses.Update) {
-	floatSeconds := func(nanoSeconds int64) float64 {
-		return float64(float64(nanoSeconds) / float64(1000000000.0))
-	}
-	startTime := time.Now().UnixNano()
-	updateRepotime, jsonBytes, updateErr := repo.update()
-	updateResponse = &responses.Update{}
-	updateResponse.Stats.RepoRuntime = floatSeconds(updateRepotime)
-
-	if updateErr != nil {
-		updateResponse.Success = false
-		updateResponse.Stats.NumberOfNodes = -1
-		updateResponse.Stats.NumberOfURIs = -1
-		// let us try to restore the world from a file
-		log.Error("could not update repository:" + updateErr.Error())
-		updateResponse.ErrorMessage = updateErr.Error()
-		restoreErr := repo.tryToRestoreCurrent()
-		if restoreErr != nil {
-			log.Error("failed to restore preceding repo version: " + restoreErr.Error())
-		} else {
-			log.Record("restored current repo from local history")
-		}
-	} else {
-		updateResponse.Success = true
-		// persist the currently loaded one
-		historyErr := repo.history.add(jsonBytes)
-		if historyErr != nil {
-			log.Warning("could not persist current repo in history: " + historyErr.Error())
-		}
-		// add some stats
-		for dimension := range repo.Directory {
-			updateResponse.Stats.NumberOfNodes += len(repo.Directory[dimension].Directory)
-			updateResponse.Stats.NumberOfURIs += len(repo.Directory[dimension].URIDirectory)
-		}
-	}
-	updateResponse.Stats.OwnRuntime = floatSeconds(time.Now().UnixNano()-startTime) - updateResponse.Stats.RepoRuntime
-	return updateResponse
-}
-
 func (repo *Repo) tryToRestoreCurrent() error {
 	currentJSONBytes, err := repo.history.getCurrent()
 	if err != nil {
@@ -198,6 +157,12 @@ func (repo *Repo) loadJSONBytes(jsonBytes []byte) error {
 		} else {
 			log.Record("added valid json to history")
 		}
+		cleanUpErr := repo.history.cleanup()
+		if cleanUpErr != nil {
+			log.Warning("an error occured while cleaning up my history:", cleanUpErr)
+		} else {
+			log.Record("cleaned up history")
+		}
 	}
 	return err
 }

repo/repo.go

@@ -4,10 +4,12 @@ import (
 	"errors"
 	"fmt"
 	"strings"
+	"time"
 
 	"github.com/foomo/contentserver/content"
 	"github.com/foomo/contentserver/log"
 	"github.com/foomo/contentserver/requests"
+	"github.com/foomo/contentserver/responses"
 )
 
 // Dimension dimension in a repo
@@ -52,39 +54,6 @@ func NewRepo(server string, varDir string) *Repo {
 	return repo
 }
 
-// ResolveContent find content in a repository
-func (repo *Repo) ResolveContent(dimensions []string, URI string) (resolved bool, resolvedURI string, resolvedDimension string, repoNode *content.RepoNode) {
-	parts := strings.Split(URI, content.PathSeparator)
-	resolved = false
-	resolvedURI = ""
-	resolvedDimension = ""
-	repoNode = nil
-	log.Debug("repo.ResolveContent: " + URI)
-	for _, dimension := range dimensions {
-		if d, ok := repo.Directory[dimension]; ok {
-			for i := len(parts); i > 0; i-- {
-				testURI := strings.Join(parts[0:i], content.PathSeparator)
-				if testURI == "" {
-					testURI = content.PathSeparator
-				}
-				log.Debug("  testing[" + dimension + "]: " + testURI)
-				if repoNode, ok := d.URIDirectory[testURI]; ok {
-					resolved = true
-					log.Debug("  found  => " + testURI)
-					log.Debug("    destination " + fmt.Sprint(repoNode.DestinationID))
-					if len(repoNode.DestinationID) > 0 {
-						if destionationNode, destinationNodeOk := d.Directory[repoNode.DestinationID]; destinationNodeOk {
-							repoNode = destionationNode
-						}
-					}
-					return true, testURI, dimension, repoNode
-				}
-			}
-		}
-	}
-	return
-}
-
 // GetURIs get many uris at once
 func (repo *Repo) GetURIs(dimension string, ids []string) map[string]string {
 	uris := make(map[string]string)
@@ -94,40 +63,6 @@ func (repo *Repo) GetURIs(dimension string, ids []string) map[string]string {
 	return uris
 }
 
-func (repo *Repo) getURIForNode(dimension string, repoNode *content.RepoNode) string {
-
-	if len(repoNode.LinkID) == 0 {
-		return repoNode.URI
-	}
-	linkedNode, ok := repo.Directory[dimension].Directory[repoNode.LinkID]
-	if ok {
-		return repo.getURIForNode(dimension, linkedNode)
-	}
-	return ""
-}
-
-func (repo *Repo) getURI(dimension string, id string) string {
-	repoNode, ok := repo.Directory[dimension].Directory[id]
-	if ok {
-		return repo.getURIForNode(dimension, repoNode)
-	}
-	return ""
-}
-
-func (repo *Repo) getNode(repoNode *content.RepoNode, expanded bool, mimeTypes []string, path []*content.Item, level int, groups []string, dataFields []string) *content.Node {
-	node := content.NewNode()
-	node.Item = repoNode.ToItem(dataFields)
-	log.Debug("repo.GetNode: " + repoNode.ID)
-	for _, childID := range repoNode.Index {
-		childNode := repoNode.Nodes[childID]
-		if (level == 0 || expanded || !expanded && childNode.InPath(path)) && !childNode.Hidden && childNode.CanBeAccessedByGroups(groups) && childNode.IsOneOfTheseMimeTypes(mimeTypes) {
-			node.Nodes[childID] = repo.getNode(childNode, expanded, mimeTypes, path, level+1, groups, dataFields)
-			node.Index = append(node.Index, childID)
-		}
-	}
-	return node
-}
-
 // GetNodes get nodes
 func (repo *Repo) GetNodes(r *requests.Nodes) map[string]*content.Node {
 	nodes := make(map[string]*content.Node)
@@ -149,38 +84,6 @@ func (repo *Repo) GetNodes(r *requests.Nodes) map[string]*content.Node {
 	return nodes
 }
 
-func (repo *Repo) validateContentRequest(req *requests.Content) (err error) {
-	if req == nil {
-		return errors.New("request must not be nil")
-	}
-	if len(req.URI) == 0 {
-		return errors.New("request URI must not be empty")
-	}
-
-	if req.Env == nil {
-		return errors.New("request.Env must not be nil")
-	}
-	if len(req.Env.Dimensions) == 0 {
-		return errors.New("request.Env.Dimensions must not be empty")
-	}
-
-	for _, envDimension := range req.Env.Dimensions {
-		if !repo.hasDimension(envDimension) {
-			availableDimensions := []string{}
-			for availableDimension := range repo.Directory {
-				availableDimensions = append(availableDimensions, availableDimension)
-			}
-			return fmt.Errorf("unknown dimension %q in r.Env must be one of %q", envDimension, availableDimensions)
-		}
-	}
-	return nil
-}
-
-func (repo *Repo) hasDimension(d string) bool {
-	_, hasDimension := repo.Directory[d]
-	return hasDimension
-}
-
 // GetContent resolves content and fetches nodes in one call. It combines those
 // two tasks for performance reasons.
 //
@@ -199,7 +102,7 @@ func (repo *Repo) GetContent(r *requests.Content) (c *content.SiteContent, err e
 	}
 	log.Debug("repo.GetContent: ", r.URI)
 	c = content.NewSiteContent()
-	resolved, resolvedURI, resolvedDimension, node := repo.ResolveContent(r.Env.Dimensions, r.URI)
+	resolved, resolvedURI, resolvedDimension, node := repo.resolveContent(r.Env.Dimensions, r.URI)
 	if resolved {
 		log.Notice("200 for " + r.URI)
 		// forbidden ?!
@@ -254,6 +157,145 @@ func (repo *Repo) GetRepo() map[string]*content.RepoNode {
 	return response
 }
 
+// Update - reload contents of repository with json from repo.server
+func (repo *Repo) Update() (updateResponse *responses.Update) {
+	floatSeconds := func(nanoSeconds int64) float64 {
+		return float64(float64(nanoSeconds) / float64(1000000000.0))
+	}
+	startTime := time.Now().UnixNano()
+	updateRepotime, jsonBytes, updateErr := repo.update()
+	updateResponse = &responses.Update{}
+	updateResponse.Stats.RepoRuntime = floatSeconds(updateRepotime)
+
+	if updateErr != nil {
+		updateResponse.Success = false
+		updateResponse.Stats.NumberOfNodes = -1
+		updateResponse.Stats.NumberOfURIs = -1
+		// let us try to restore the world from a file
+		log.Error("could not update repository:" + updateErr.Error())
+		updateResponse.ErrorMessage = updateErr.Error()
+		restoreErr := repo.tryToRestoreCurrent()
+		if restoreErr != nil {
+			log.Error("failed to restore preceding repo version: " + restoreErr.Error())
+		} else {
+			log.Record("restored current repo from local history")
+		}
+	} else {
+		updateResponse.Success = true
+		// persist the currently loaded one
+		historyErr := repo.history.add(jsonBytes)
+		if historyErr != nil {
+			log.Warning("could not persist current repo in history: " + historyErr.Error())
+		}
+		// add some stats
+		for dimension := range repo.Directory {
+			updateResponse.Stats.NumberOfNodes += len(repo.Directory[dimension].Directory)
+			updateResponse.Stats.NumberOfURIs += len(repo.Directory[dimension].URIDirectory)
+		}
+	}
+	updateResponse.Stats.OwnRuntime = floatSeconds(time.Now().UnixNano()-startTime) - updateResponse.Stats.RepoRuntime
+	return updateResponse
+}
+
+// resolveContent find content in a repository
+func (repo *Repo) resolveContent(dimensions []string, URI string) (resolved bool, resolvedURI string, resolvedDimension string, repoNode *content.RepoNode) {
+	parts := strings.Split(URI, content.PathSeparator)
+	resolved = false
+	resolvedURI = ""
+	resolvedDimension = ""
+	repoNode = nil
+	log.Debug("repo.ResolveContent: " + URI)
+	for _, dimension := range dimensions {
+		if d, ok := repo.Directory[dimension]; ok {
+			for i := len(parts); i > 0; i-- {
+				testURI := strings.Join(parts[0:i], content.PathSeparator)
+				if testURI == "" {
+					testURI = content.PathSeparator
+				}
+				log.Debug("  testing[" + dimension + "]: " + testURI)
+				if repoNode, ok := d.URIDirectory[testURI]; ok {
+					resolved = true
+					log.Debug("  found  => " + testURI)
+					log.Debug("    destination " + fmt.Sprint(repoNode.DestinationID))
+					if len(repoNode.DestinationID) > 0 {
+						if destionationNode, destinationNodeOk := d.Directory[repoNode.DestinationID]; destinationNodeOk {
+							repoNode = destionationNode
+						}
+					}
+					return true, testURI, dimension, repoNode
+				}
+			}
+		}
+	}
+	return
+}
+
+func (repo *Repo) getURIForNode(dimension string, repoNode *content.RepoNode) string {
+
+	if len(repoNode.LinkID) == 0 {
+		return repoNode.URI
+	}
+	linkedNode, ok := repo.Directory[dimension].Directory[repoNode.LinkID]
+	if ok {
+		return repo.getURIForNode(dimension, linkedNode)
+	}
+	return ""
+}
+
+func (repo *Repo) getURI(dimension string, id string) string {
+	repoNode, ok := repo.Directory[dimension].Directory[id]
+	if ok {
+		return repo.getURIForNode(dimension, repoNode)
+	}
+	return ""
+}
+
+func (repo *Repo) getNode(repoNode *content.RepoNode, expanded bool, mimeTypes []string, path []*content.Item, level int, groups []string, dataFields []string) *content.Node {
+	node := content.NewNode()
+	node.Item = repoNode.ToItem(dataFields)
+	log.Debug("repo.GetNode: " + repoNode.ID)
+	for _, childID := range repoNode.Index {
+		childNode := repoNode.Nodes[childID]
+		if (level == 0 || expanded || !expanded && childNode.InPath(path)) && !childNode.Hidden && childNode.CanBeAccessedByGroups(groups) && childNode.IsOneOfTheseMimeTypes(mimeTypes) {
+			node.Nodes[childID] = repo.getNode(childNode, expanded, mimeTypes, path, level+1, groups, dataFields)
+			node.Index = append(node.Index, childID)
+		}
+	}
+	return node
+}
+
+func (repo *Repo) validateContentRequest(req *requests.Content) (err error) {
+	if req == nil {
+		return errors.New("request must not be nil")
+	}
+	if len(req.URI) == 0 {
+		return errors.New("request URI must not be empty")
+	}
+
+	if req.Env == nil {
+		return errors.New("request.Env must not be nil")
+	}
+	if len(req.Env.Dimensions) == 0 {
+		return errors.New("request.Env.Dimensions must not be empty")
+	}
+
+	for _, envDimension := range req.Env.Dimensions {
+		if !repo.hasDimension(envDimension) {
+			availableDimensions := []string{}
+			for availableDimension := range repo.Directory {
+				availableDimensions = append(availableDimensions, availableDimension)
+			}
+			return fmt.Errorf("unknown dimension %q in r.Env must be one of %q", envDimension, availableDimensions)
+		}
+	}
+	return nil
+}
+
+func (repo *Repo) hasDimension(d string) bool {
+	_, hasDimension := repo.Directory[d]
+	return hasDimension
+}
+
 func uriKeyForState(state string, uri string) string {
 	return state + "-" + uri
 }