diff --git a/repo/history.go b/repo/history.go index 525b12a..21b7abc 100644 --- a/repo/history.go +++ b/repo/history.go @@ -1,13 +1,18 @@ package repo import ( + "fmt" "io/ioutil" + "os" "path" + "sort" + "strings" "time" ) const historyRepoJSONPrefix = "contentserver-repo-" const historyRepoJSONSuffix = ".json" +const maxHistoryVersions = 20 type history struct { varDir string @@ -30,6 +35,41 @@ func (h *history) add(jsonBytes []byte) error { return ioutil.WriteFile(h.getCurrentFilename(), jsonBytes, 0644) } +func (h *history) getHistory() (files []string, err error) { + files = []string{} + fileInfos, err := ioutil.ReadDir(h.varDir) + if err != nil { + return + } + currentName := h.getCurrentFilename() + for _, f := range fileInfos { + if !f.IsDir() { + filename := f.Name() + if filename != currentName && (strings.HasPrefix(filename, historyRepoJSONPrefix) && strings.HasSuffix(filename, historyRepoJSONSuffix)) { + files = append(files, path.Join(h.varDir, filename)) + } + } + } + sort.Strings(files) + return +} + +func (h *history) cleanup() error { + files, err := h.getHistory() + if err != nil { + return err + } + if len(files) > maxHistoryVersions { + for i := maxHistoryVersions; i < len(files); i++ { + err := os.Remove(files[i]) + if err != nil { + return fmt.Errorf("could not remove file %q got %q", files[i], err) + } + } + } + return nil +} + func (h *history) getCurrentFilename() string { return path.Join(h.varDir, historyRepoJSONPrefix+"current"+historyRepoJSONSuffix) } diff --git a/repo/history_test.go b/repo/history_test.go new file mode 100644 index 0000000..77dbe05 --- /dev/null +++ b/repo/history_test.go @@ -0,0 +1,47 @@ +package repo + +import ( + "bytes" + "fmt" + "io/ioutil" + "os" + "testing" + "time" +) + +func testHistory() *history { + tempDir, err := ioutil.TempDir(os.TempDir(), "contentserver-history-test") + if err != nil { + panic(err) + } + return newHistory(tempDir) +} + +func TestHistoryCurrent(t *testing.T) { + h := testHistory() + test := []byte("test") + h.add(test) + current, err := h.getCurrent() + if err != nil { + t.Fatal(err) + } + if bytes.Compare(current, test) != 0 { + t.Fatal(fmt.Sprintf("expected %q, got %q", string(test), string(current))) + } +} + +func TestHistoryCleanup(t *testing.T) { + h := testHistory() + for i := 0; i < 50; i++ { + h.add([]byte(fmt.Sprint(i))) + time.Sleep(time.Millisecond * 5) + } + h.cleanup() + files, err := h.getHistory() + if err != nil { + t.Fatal(err) + } + if len(files) != maxHistoryVersions { + t.Fatal("history too long", len(files), "instead of", maxHistoryVersions) + } +} diff --git a/repo/loader.go b/repo/loader.go index 86bd6d0..71ccffa 100644 --- a/repo/loader.go +++ b/repo/loader.go @@ -10,7 +10,6 @@ import ( "github.com/foomo/contentserver/content" "github.com/foomo/contentserver/log" - "github.com/foomo/contentserver/responses" ) func (repo *Repo) updateRoutine() { @@ -101,46 +100,6 @@ func loadNodesFromJSON(jsonBytes []byte) (nodes map[string]*content.RepoNode, er return nodes, err } -// Update - reload contents of repository with json from repo.server -func (repo *Repo) Update() (updateResponse *responses.Update) { - floatSeconds := func(nanoSeconds int64) float64 { - return float64(float64(nanoSeconds) / float64(1000000000.0)) - } - startTime := time.Now().UnixNano() - updateRepotime, jsonBytes, updateErr := repo.update() - updateResponse = &responses.Update{} - updateResponse.Stats.RepoRuntime = floatSeconds(updateRepotime) - - if updateErr != nil { - updateResponse.Success = false - updateResponse.Stats.NumberOfNodes = -1 - updateResponse.Stats.NumberOfURIs = -1 - // let us try to restore the world from a file - log.Error("could not update repository:" + updateErr.Error()) - updateResponse.ErrorMessage = updateErr.Error() - restoreErr := repo.tryToRestoreCurrent() - if restoreErr != nil { - log.Error("failed to restore preceding repo version: " + restoreErr.Error()) - } else { - log.Record("restored current repo from local history") - } - } else { - updateResponse.Success = true - // persist the currently loaded one - historyErr := repo.history.add(jsonBytes) - if historyErr != nil { - log.Warning("could not persist current repo in history: " + historyErr.Error()) - } - // add some stats - for dimension := range repo.Directory { - updateResponse.Stats.NumberOfNodes += len(repo.Directory[dimension].Directory) - updateResponse.Stats.NumberOfURIs += len(repo.Directory[dimension].URIDirectory) - } - } - updateResponse.Stats.OwnRuntime = floatSeconds(time.Now().UnixNano()-startTime) - updateResponse.Stats.RepoRuntime - return updateResponse -} - func (repo *Repo) tryToRestoreCurrent() error { currentJSONBytes, err := repo.history.getCurrent() if err != nil { @@ -198,6 +157,12 @@ func (repo *Repo) loadJSONBytes(jsonBytes []byte) error { } else { log.Record("added valid json to history") } + cleanUpErr := repo.history.cleanup() + if cleanUpErr != nil { + log.Warning("an error occured while cleaning up my history:", cleanUpErr) + } else { + log.Record("cleaned up history") + } } return err } diff --git a/repo/repo.go b/repo/repo.go index 00308d6..dc50bf7 100644 --- a/repo/repo.go +++ b/repo/repo.go @@ -4,10 +4,12 @@ import ( "errors" "fmt" "strings" + "time" "github.com/foomo/contentserver/content" "github.com/foomo/contentserver/log" "github.com/foomo/contentserver/requests" + "github.com/foomo/contentserver/responses" ) // Dimension dimension in a repo @@ -52,39 +54,6 @@ func NewRepo(server string, varDir string) *Repo { return repo } -// ResolveContent find content in a repository -func (repo *Repo) ResolveContent(dimensions []string, URI string) (resolved bool, resolvedURI string, resolvedDimension string, repoNode *content.RepoNode) { - parts := strings.Split(URI, content.PathSeparator) - resolved = false - resolvedURI = "" - resolvedDimension = "" - repoNode = nil - log.Debug("repo.ResolveContent: " + URI) - for _, dimension := range dimensions { - if d, ok := repo.Directory[dimension]; ok { - for i := len(parts); i > 0; i-- { - testURI := strings.Join(parts[0:i], content.PathSeparator) - if testURI == "" { - testURI = content.PathSeparator - } - log.Debug(" testing[" + dimension + "]: " + testURI) - if repoNode, ok := d.URIDirectory[testURI]; ok { - resolved = true - log.Debug(" found => " + testURI) - log.Debug(" destination " + fmt.Sprint(repoNode.DestinationID)) - if len(repoNode.DestinationID) > 0 { - if destionationNode, destinationNodeOk := d.Directory[repoNode.DestinationID]; destinationNodeOk { - repoNode = destionationNode - } - } - return true, testURI, dimension, repoNode - } - } - } - } - return -} - // GetURIs get many uris at once func (repo *Repo) GetURIs(dimension string, ids []string) map[string]string { uris := make(map[string]string) @@ -94,40 +63,6 @@ func (repo *Repo) GetURIs(dimension string, ids []string) map[string]string { return uris } -func (repo *Repo) getURIForNode(dimension string, repoNode *content.RepoNode) string { - - if len(repoNode.LinkID) == 0 { - return repoNode.URI - } - linkedNode, ok := repo.Directory[dimension].Directory[repoNode.LinkID] - if ok { - return repo.getURIForNode(dimension, linkedNode) - } - return "" -} - -func (repo *Repo) getURI(dimension string, id string) string { - repoNode, ok := repo.Directory[dimension].Directory[id] - if ok { - return repo.getURIForNode(dimension, repoNode) - } - return "" -} - -func (repo *Repo) getNode(repoNode *content.RepoNode, expanded bool, mimeTypes []string, path []*content.Item, level int, groups []string, dataFields []string) *content.Node { - node := content.NewNode() - node.Item = repoNode.ToItem(dataFields) - log.Debug("repo.GetNode: " + repoNode.ID) - for _, childID := range repoNode.Index { - childNode := repoNode.Nodes[childID] - if (level == 0 || expanded || !expanded && childNode.InPath(path)) && !childNode.Hidden && childNode.CanBeAccessedByGroups(groups) && childNode.IsOneOfTheseMimeTypes(mimeTypes) { - node.Nodes[childID] = repo.getNode(childNode, expanded, mimeTypes, path, level+1, groups, dataFields) - node.Index = append(node.Index, childID) - } - } - return node -} - // GetNodes get nodes func (repo *Repo) GetNodes(r *requests.Nodes) map[string]*content.Node { nodes := make(map[string]*content.Node) @@ -149,38 +84,6 @@ func (repo *Repo) GetNodes(r *requests.Nodes) map[string]*content.Node { return nodes } -func (repo *Repo) validateContentRequest(req *requests.Content) (err error) { - if req == nil { - return errors.New("request must not be nil") - } - if len(req.URI) == 0 { - return errors.New("request URI must not be empty") - } - - if req.Env == nil { - return errors.New("request.Env must not be nil") - } - if len(req.Env.Dimensions) == 0 { - return errors.New("request.Env.Dimensions must not be empty") - } - - for _, envDimension := range req.Env.Dimensions { - if !repo.hasDimension(envDimension) { - availableDimensions := []string{} - for availableDimension := range repo.Directory { - availableDimensions = append(availableDimensions, availableDimension) - } - return fmt.Errorf("unknown dimension %q in r.Env must be one of %q", envDimension, availableDimensions) - } - } - return nil -} - -func (repo *Repo) hasDimension(d string) bool { - _, hasDimension := repo.Directory[d] - return hasDimension -} - // GetContent resolves content and fetches nodes in one call. It combines those // two tasks for performance reasons. // @@ -199,7 +102,7 @@ func (repo *Repo) GetContent(r *requests.Content) (c *content.SiteContent, err e } log.Debug("repo.GetContent: ", r.URI) c = content.NewSiteContent() - resolved, resolvedURI, resolvedDimension, node := repo.ResolveContent(r.Env.Dimensions, r.URI) + resolved, resolvedURI, resolvedDimension, node := repo.resolveContent(r.Env.Dimensions, r.URI) if resolved { log.Notice("200 for " + r.URI) // forbidden ?! @@ -254,6 +157,145 @@ func (repo *Repo) GetRepo() map[string]*content.RepoNode { return response } +// Update - reload contents of repository with json from repo.server +func (repo *Repo) Update() (updateResponse *responses.Update) { + floatSeconds := func(nanoSeconds int64) float64 { + return float64(float64(nanoSeconds) / float64(1000000000.0)) + } + startTime := time.Now().UnixNano() + updateRepotime, jsonBytes, updateErr := repo.update() + updateResponse = &responses.Update{} + updateResponse.Stats.RepoRuntime = floatSeconds(updateRepotime) + + if updateErr != nil { + updateResponse.Success = false + updateResponse.Stats.NumberOfNodes = -1 + updateResponse.Stats.NumberOfURIs = -1 + // let us try to restore the world from a file + log.Error("could not update repository:" + updateErr.Error()) + updateResponse.ErrorMessage = updateErr.Error() + restoreErr := repo.tryToRestoreCurrent() + if restoreErr != nil { + log.Error("failed to restore preceding repo version: " + restoreErr.Error()) + } else { + log.Record("restored current repo from local history") + } + } else { + updateResponse.Success = true + // persist the currently loaded one + historyErr := repo.history.add(jsonBytes) + if historyErr != nil { + log.Warning("could not persist current repo in history: " + historyErr.Error()) + } + // add some stats + for dimension := range repo.Directory { + updateResponse.Stats.NumberOfNodes += len(repo.Directory[dimension].Directory) + updateResponse.Stats.NumberOfURIs += len(repo.Directory[dimension].URIDirectory) + } + } + updateResponse.Stats.OwnRuntime = floatSeconds(time.Now().UnixNano()-startTime) - updateResponse.Stats.RepoRuntime + return updateResponse +} + +// resolveContent find content in a repository +func (repo *Repo) resolveContent(dimensions []string, URI string) (resolved bool, resolvedURI string, resolvedDimension string, repoNode *content.RepoNode) { + parts := strings.Split(URI, content.PathSeparator) + resolved = false + resolvedURI = "" + resolvedDimension = "" + repoNode = nil + log.Debug("repo.ResolveContent: " + URI) + for _, dimension := range dimensions { + if d, ok := repo.Directory[dimension]; ok { + for i := len(parts); i > 0; i-- { + testURI := strings.Join(parts[0:i], content.PathSeparator) + if testURI == "" { + testURI = content.PathSeparator + } + log.Debug(" testing[" + dimension + "]: " + testURI) + if repoNode, ok := d.URIDirectory[testURI]; ok { + resolved = true + log.Debug(" found => " + testURI) + log.Debug(" destination " + fmt.Sprint(repoNode.DestinationID)) + if len(repoNode.DestinationID) > 0 { + if destionationNode, destinationNodeOk := d.Directory[repoNode.DestinationID]; destinationNodeOk { + repoNode = destionationNode + } + } + return true, testURI, dimension, repoNode + } + } + } + } + return +} + +func (repo *Repo) getURIForNode(dimension string, repoNode *content.RepoNode) string { + + if len(repoNode.LinkID) == 0 { + return repoNode.URI + } + linkedNode, ok := repo.Directory[dimension].Directory[repoNode.LinkID] + if ok { + return repo.getURIForNode(dimension, linkedNode) + } + return "" +} + +func (repo *Repo) getURI(dimension string, id string) string { + repoNode, ok := repo.Directory[dimension].Directory[id] + if ok { + return repo.getURIForNode(dimension, repoNode) + } + return "" +} + +func (repo *Repo) getNode(repoNode *content.RepoNode, expanded bool, mimeTypes []string, path []*content.Item, level int, groups []string, dataFields []string) *content.Node { + node := content.NewNode() + node.Item = repoNode.ToItem(dataFields) + log.Debug("repo.GetNode: " + repoNode.ID) + for _, childID := range repoNode.Index { + childNode := repoNode.Nodes[childID] + if (level == 0 || expanded || !expanded && childNode.InPath(path)) && !childNode.Hidden && childNode.CanBeAccessedByGroups(groups) && childNode.IsOneOfTheseMimeTypes(mimeTypes) { + node.Nodes[childID] = repo.getNode(childNode, expanded, mimeTypes, path, level+1, groups, dataFields) + node.Index = append(node.Index, childID) + } + } + return node +} + +func (repo *Repo) validateContentRequest(req *requests.Content) (err error) { + if req == nil { + return errors.New("request must not be nil") + } + if len(req.URI) == 0 { + return errors.New("request URI must not be empty") + } + + if req.Env == nil { + return errors.New("request.Env must not be nil") + } + if len(req.Env.Dimensions) == 0 { + return errors.New("request.Env.Dimensions must not be empty") + } + + for _, envDimension := range req.Env.Dimensions { + if !repo.hasDimension(envDimension) { + availableDimensions := []string{} + for availableDimension := range repo.Directory { + availableDimensions = append(availableDimensions, availableDimension) + } + return fmt.Errorf("unknown dimension %q in r.Env must be one of %q", envDimension, availableDimensions) + } + } + return nil +} + +func (repo *Repo) hasDimension(d string) bool { + _, hasDimension := repo.Directory[d] + return hasDimension +} + func uriKeyForState(state string, uri string) string { return state + "-" + uri }