package repo import ( "bytes" "crypto/tls" "errors" "fmt" "io" "net/http" "os" "strings" "time" "github.com/foomo/contentserver/status" "go.uber.org/zap" "github.com/foomo/contentserver/content" "github.com/foomo/contentserver/logger" "github.com/foomo/contentserver/requests" "github.com/foomo/contentserver/responses" ) const maxGetURIForNodeRecursionLevel = 1000 // Dimension dimension in a repo type Dimension struct { Directory map[string]*content.RepoNode URIDirectory map[string]*content.RepoNode Node *content.RepoNode } // Repo content repositiory type Repo struct { pollForUpdates bool pollVersion string server string recovered bool Directory map[string]*Dimension // updateLock sync.Mutex dimensionUpdateChannel chan *repoDimension dimensionUpdateDoneChannel chan error history *history updateInProgressChannel chan chan updateResponse // jsonBytes []byte jsonBuf bytes.Buffer httpClient *http.Client } type repoDimension struct { Dimension string Node *content.RepoNode } // NewRepo constructor func NewRepo(server string, varDir string, repositoryTimeout time.Duration, pollForUpdates bool) *Repo { logger.Log.Info("creating new repo", zap.String("server", server), zap.String("varDir", varDir), ) repo := &Repo{ pollForUpdates: pollForUpdates, recovered: false, server: server, Directory: map[string]*Dimension{}, history: newHistory(varDir), dimensionUpdateChannel: make(chan *repoDimension), dimensionUpdateDoneChannel: make(chan error), httpClient: getDefaultHTTPClient(repositoryTimeout), updateInProgressChannel: make(chan chan updateResponse), } go repo.updateRoutine() go repo.dimensionUpdateRoutine() if pollForUpdates { go func() { ticker := time.NewTicker(10 * time.Second) for range ticker.C { chanReponse := make(chan updateResponse) repo.updateInProgressChannel <- chanReponse response := <-chanReponse if response.err == nil { logger.Log.Info("poll update success", zap.String("revision", repo.pollVersion)) } else { logger.Log.Info("poll error", zap.Error(response.err)) } } }() } logger.Log.Info("trying to restore previous state") restoreErr := repo.tryToRestoreCurrent() if restoreErr != nil { logger.Log.Error(" could not restore previous repo content", zap.Error(restoreErr)) } else { repo.recovered = true logger.Log.Info("restored previous repo content") } return repo } func getDefaultHTTPClient(timeout time.Duration) *http.Client { client := &http.Client{ Transport: &http.Transport{ DisableKeepAlives: true, TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, TLSHandshakeTimeout: 5 * time.Second, }, Timeout: timeout, } return client } func (repo *Repo) Recovered() bool { return repo.recovered } // GetURIs get many uris at once func (repo *Repo) GetURIs(dimension string, ids []string) map[string]string { uris := map[string]string{} for _, id := range ids { uris[id] = repo.getURI(dimension, id) } return uris } // GetNodes get nodes func (repo *Repo) GetNodes(r *requests.Nodes) map[string]*content.Node { return repo.getNodes(r.Nodes, r.Env) } func (repo *Repo) getNodes(nodeRequests map[string]*requests.Node, env *requests.Env) map[string]*content.Node { var ( nodes = map[string]*content.Node{} path = []*content.Item{} ) for nodeName, nodeRequest := range nodeRequests { if nodeName == "" || nodeRequest.ID == "" { logger.Log.Info("invalid node request", zap.Error(errors.New("nodeName or nodeRequest.ID empty"))) continue } logger.Log.Debug("adding node", zap.String("name", nodeName), zap.String("requestID", nodeRequest.ID)) groups := env.Groups if len(nodeRequest.Groups) > 0 { groups = nodeRequest.Groups } dimensionNode, ok := repo.Directory[nodeRequest.Dimension] nodes[nodeName] = nil if !ok && nodeRequest.Dimension == "" { logger.Log.Debug("Could not get dimension root node", zap.String("dimension", nodeRequest.Dimension)) for _, dimension := range env.Dimensions { dimensionNode, ok = repo.Directory[dimension] if ok { logger.Log.Debug("Found root node in env.Dimensions", zap.String("dimension", dimension)) break } logger.Log.Debug("Could NOT find root node in env.Dimensions", zap.String("dimension", dimension)) } } if !ok { logger.Log.Error("could not get dimension root node", zap.String("nodeRequest.Dimension", nodeRequest.Dimension)) continue } treeNode, ok := dimensionNode.Directory[nodeRequest.ID] if !ok { logger.Log.Error("Invalid tree node requested", zap.String("nodeName", nodeName), zap.String("nodeID", nodeRequest.ID), ) status.M.InvalidNodeTreeRequests.WithLabelValues().Inc() continue } nodes[nodeName] = repo.getNode(treeNode, nodeRequest.Expand, nodeRequest.MimeTypes, path, 0, groups, nodeRequest.DataFields, nodeRequest.ExposeHiddenNodes) } return nodes } // GetContent resolves content and fetches nodes in one call. It combines those // two tasks for performance reasons. // // In the first step it uses r.URI to look up content in all given // r.Env.Dimensions of repo.Directory. // // In the second step it collects the requested nodes. // // those two steps are independent. func (repo *Repo) GetContent(r *requests.Content) (c *content.SiteContent, err error) { // add more input validation err = repo.validateContentRequest(r) if err != nil { logger.Log.Error("repo.GetContent invalid request", zap.Error(err)) return } logger.Log.Debug("repo.GetContent", zap.String("URI", r.URI)) c = content.NewSiteContent() resolved, resolvedURI, resolvedDimension, node := repo.resolveContent(r.Env.Dimensions, r.URI) if resolved { if !node.CanBeAccessedByGroups(r.Env.Groups) { logger.Log.Warn("Resolved content cannot be accessed by specified group", zap.String("URI", r.URI)) c.Status = content.StatusForbidden } else { logger.Log.Info("Content resolved", zap.String("URI", r.URI)) c.Status = content.StatusOk c.Data = node.Data } c.MimeType = node.MimeType c.Dimension = resolvedDimension c.URI = resolvedURI c.Item = node.ToItem(r.DataFields) c.Path = node.GetPath(r.PathDataFields) // fetch URIs for all dimensions uris := make(map[string]string) for dimensionName := range repo.Directory { uris[dimensionName] = repo.getURI(dimensionName, node.ID) } c.URIs = uris } else { logger.Log.Info("Content not found", zap.String("URI", r.URI)) c.Status = content.StatusNotFound c.Dimension = r.Env.Dimensions[0] logger.Log.Debug("Failed to resolve, falling back to default dimension", zap.String("URI", r.URI), zap.String("defaultDimension", r.Env.Dimensions[0]), ) // r.Env.Dimensions is validated => we can access it resolvedDimension = r.Env.Dimensions[0] } // add navigation trees for _, node := range r.Nodes { if node.Dimension == "" { node.Dimension = resolvedDimension } } c.Nodes = repo.getNodes(r.Nodes, r.Env) return c, nil } // GetRepo get the whole repo in all dimensions func (repo *Repo) GetRepo() map[string]*content.RepoNode { response := make(map[string]*content.RepoNode) for dimensionName, dimension := range repo.Directory { response[dimensionName] = dimension.Node } return response } // WriteRepoBytes get the whole repo in all dimensions // reads the JSON history file from the Filesystem and copies it directly in to the supplied buffer // the result is wrapped as service response, e.g: {"reply": } func (repo *Repo) WriteRepoBytes(w io.Writer) { f, err := os.Open(repo.history.getCurrentFilename()) if err != nil { logger.Log.Error("Failed to serve Repo JSON", zap.Error(err)) } _, _ = w.Write([]byte("{\"reply\":")) _, err = io.Copy(w, f) if err != nil { logger.Log.Error("Failed to serve Repo JSON", zap.Error(err)) } _, _ = w.Write([]byte("}")) } // Update - reload contents of repository with json from repo.server func (repo *Repo) Update() (updateResponse *responses.Update) { floatSeconds := func(nanoSeconds int64) float64 { return float64(float64(nanoSeconds) / float64(1000000000.0)) } logger.Log.Info("Update triggered") // Log.Info(ansi.Yellow + "BUFFER LENGTH BEFORE tryUpdate(): " + strconv.Itoa(len(repo.jsonBuf.Bytes())) + ansi.Reset) startTime := time.Now().UnixNano() updateRepotime, updateErr := repo.tryUpdate() updateResponse = &responses.Update{} updateResponse.Stats.RepoRuntime = floatSeconds(updateRepotime) if updateErr != nil { updateResponse.Success = false updateResponse.Stats.NumberOfNodes = -1 updateResponse.Stats.NumberOfURIs = -1 // let us try to restore the world from a file // Log.Info(ansi.Yellow + "BUFFER LENGTH AFTER ERROR: " + strconv.Itoa(len(repo.jsonBuf.Bytes())) + ansi.Reset) // only try to restore if the update failed during processing if updateErr != errUpdateRejected { updateResponse.ErrorMessage = updateErr.Error() logger.Log.Error("Failed to update repository", zap.Error(updateErr)) restoreErr := repo.tryToRestoreCurrent() if restoreErr != nil { logger.Log.Error("Failed to restore preceding repository version", zap.Error(restoreErr)) } else { logger.Log.Info("Successfully restored current repository from local history") } } } else { updateResponse.Success = true // persist the currently loaded one historyErr := repo.history.add(repo.jsonBuf.Bytes()) if historyErr != nil { logger.Log.Error("Could not persist current repo in history", zap.Error(historyErr)) status.M.HistoryPersistFailedCounter.WithLabelValues().Inc() } // add some stats for dimension := range repo.Directory { updateResponse.Stats.NumberOfNodes += len(repo.Directory[dimension].Directory) updateResponse.Stats.NumberOfURIs += len(repo.Directory[dimension].URIDirectory) } } updateResponse.Stats.OwnRuntime = floatSeconds(time.Now().UnixNano()-startTime) - updateResponse.Stats.RepoRuntime return updateResponse } // resolveContent find content in a repository func (repo *Repo) resolveContent(dimensions []string, URI string) (resolved bool, resolvedURI string, resolvedDimension string, repoNode *content.RepoNode) { parts := strings.Split(URI, content.PathSeparator) logger.Log.Debug("repo.ResolveContent", zap.String("URI", URI)) for i := len(parts); i > 0; i-- { testURI := strings.Join(parts[0:i], content.PathSeparator) if testURI == "" { testURI = content.PathSeparator } for _, dimension := range dimensions { if d, ok := repo.Directory[dimension]; ok { logger.Log.Debug("Checking node", zap.String("dimension", dimension), zap.String("URI", testURI), ) if repoNode, ok := d.URIDirectory[testURI]; ok { resolved = true logger.Log.Debug("Node found", zap.String("URI", testURI), zap.String("destination", repoNode.DestinationID)) if len(repoNode.DestinationID) > 0 { if destionationNode, destinationNodeOk := d.Directory[repoNode.DestinationID]; destinationNodeOk { repoNode = destionationNode } } return true, testURI, dimension, repoNode } } } } return } func (repo *Repo) getURIForNode(dimension string, repoNode *content.RepoNode, recursionLevel int64) (uri string) { if len(repoNode.LinkID) == 0 { uri = repoNode.URI return } linkedNode, ok := repo.Directory[dimension].Directory[repoNode.LinkID] if ok { if recursionLevel > maxGetURIForNodeRecursionLevel { logger.Log.Error("maxGetURIForNodeRecursionLevel reached", zap.String("repoNode.ID", repoNode.ID), zap.String("linkID", repoNode.LinkID), zap.String("dimension", dimension)) return "" } return repo.getURIForNode(dimension, linkedNode, recursionLevel+1) } return } func (repo *Repo) getURI(dimension string, id string) string { repoNode, ok := repo.Directory[dimension].Directory[id] if ok { return repo.getURIForNode(dimension, repoNode, 0) } return "" } func (repo *Repo) getNode( repoNode *content.RepoNode, expanded bool, mimeTypes []string, path []*content.Item, level int, groups []string, dataFields []string, exposeHiddenNodes bool, ) *content.Node { node := content.NewNode() node.Item = repoNode.ToItem(dataFields) logger.Log.Debug("getNode", zap.String("ID", repoNode.ID)) for _, childID := range repoNode.Index { childNode := repoNode.Nodes[childID] if (level == 0 || expanded || !expanded && childNode.InPath(path)) && (!childNode.Hidden || exposeHiddenNodes) && childNode.CanBeAccessedByGroups(groups) && childNode.IsOneOfTheseMimeTypes(mimeTypes) { node.Nodes[childID] = repo.getNode(childNode, expanded, mimeTypes, path, level+1, groups, dataFields, exposeHiddenNodes) node.Index = append(node.Index, childID) } } return node } func (repo *Repo) validateContentRequest(req *requests.Content) (err error) { if req == nil { return errors.New("request must not be nil") } if len(req.URI) == 0 { return errors.New("request URI must not be empty") } if req.Env == nil { return errors.New("request.Env must not be nil") } if len(req.Env.Dimensions) == 0 { return errors.New("request.Env.Dimensions must not be empty") } for _, envDimension := range req.Env.Dimensions { if !repo.hasDimension(envDimension) { availableDimensions := []string{} for availableDimension := range repo.Directory { availableDimensions = append(availableDimensions, availableDimension) } return errors.New(fmt.Sprint( "unknown dimension ", envDimension, " in r.Env must be one of ", availableDimensions, " repo has ", len(repo.Directory), " dimensions", )) } } return nil } func (repo *Repo) hasDimension(d string) bool { _, hasDimension := repo.Directory[d] return hasDimension }