diff --git a/Dockerfile b/Dockerfile index a34a820..51a1a98 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,4 +17,4 @@ EXPOSE 80 ENTRYPOINT ["/usr/sbin/contentserver"] -CMD ["-address=$CONTENT_SERVER_ADDR", "-logLevel=CONTENT_SERVER_LOG_LEVEL", "-protocol=$CONTENT_SERVER_PROTOCOL", "-vardir=$CONTENT_SERVER_VAR_DIR"] +CMD ["-address=$CONTENT_SERVER_ADDR", "-logLevel=$CONTENT_SERVER_LOG_LEVEL", "-protocol=$CONTENT_SERVER_PROTOCOL", "-vardir=$CONTENT_SERVER_VAR_DIR"] diff --git a/README.md b/README.md index ecc2887..6bb4b44 100644 --- a/README.md +++ b/README.md @@ -55,51 +55,6 @@ Usage of bin/contentserver: -vardir="127.0.0.1:8081": where to put my data ``` -Packaging & Deployment ----------------------- - -In order to build packages and upload to Package Cloud, please install the following requirements and run the make task. - -[Package Cloud Command Line Client](https://packagecloud.io/docs#cli_install) - -``` -$ gem install package_cloud -``` - -[FPM](https://github.com/jordansissel/fpm) - -``` -$ gem install fpm -``` - -Building package - -``` -$ make package -``` - -*NOTE: you will be prompted for Package Cloud credentials.* - -Testing -------- - -``` -$ git clone https://github.com/foomo/contentserver.git -$ cd contentserver -$ make test -``` - -Contributing ------------- - -In lieu of a formal styleguide, take care to maintain the existing coding style. Add unit tests and examples for any new or changed functionality. - -1. Fork it -2. Create your feature branch (`git checkout -b my-new-feature`\) -3. Commit your changes (`git commit -am 'Add some feature'`\) -4. Push to the branch (`git push origin my-new-feature`\) -5. Create new Pull Request - License ------- diff --git a/content/constants.go b/content/constants.go new file mode 100644 index 0000000..9d4307c --- /dev/null +++ b/content/constants.go @@ -0,0 +1,8 @@ +package content + +const ( + // Indent for json indentation + Indent string = "\t" + // PathSeparator seprator for paths in URIs + PathSeparator = "/" +) diff --git a/server/repo/content/item.go b/content/item.go similarity index 71% rename from server/repo/content/item.go rename to content/item.go index 1d7e002..ef561e1 100644 --- a/server/repo/content/item.go +++ b/content/item.go @@ -1,15 +1,15 @@ package content -import () - +// Item on a node in a content tree - "payload" of an item type Item struct { - Id string `json:"id"` + ID string `json:"id"` Name string `json:"name"` URI string `json:"URI"` MimeType string `json:"mimeType"` Data map[string]interface{} `json:"data"` } +// NewItem item contructor func NewItem() *Item { item := new(Item) item.Data = make(map[string]interface{}) diff --git a/server/repo/content/node.go b/content/node.go similarity index 82% rename from server/repo/content/node.go rename to content/node.go index 991e4bb..b8c2b38 100644 --- a/server/repo/content/node.go +++ b/content/node.go @@ -1,17 +1,16 @@ package content -import () - +// Node a node in a content tree type Node struct { Item *Item `json:"item"` Nodes map[string]*Node `json:"nodes"` Index []string `json:"index"` } +// NewNode constructor func NewNode() *Node { node := new(Node) node.Item = NewItem() - //node.Index = [] node.Nodes = make(map[string]*Node) return node } diff --git a/server/repo/content/repo_node.go b/content/reponode.go similarity index 70% rename from server/repo/content/repo_node.go rename to content/reponode.go index 217d387..eaea820 100644 --- a/server/repo/content/repo_node.go +++ b/content/reponode.go @@ -5,15 +5,16 @@ import ( "strings" ) +// RepoNode node in a content tree type RepoNode struct { - Id string `json:"id"` // unique identifier - it is your responsibility, that they are unique + ID string `json:"id"` // unique identifier - it is your responsibility, that they are unique MimeType string `json:"mimeType"` // well a mime type http://www.ietf.org/rfc/rfc2046.txt - LinkId string `json:"linkId"` // (symbolic) link/alias to another node + LinkID string `json:"linkId"` // (symbolic) link/alias to another node Groups []string `json:"groups"` // which groups have access to the node, if empty everybody has access to it URI string `json:"URI"` Name string `json:"name"` Hidden bool `json:"hidden"` // hidden in content.nodes, but can still be resolved when being directly addressed - DestinationId string `json:"destinationId"` // if a node does not have any content like a folder the destinationIds can point to nodes that do aka. the first displayable child node + DestinationID string `json:"destinationId"` // if a node does not have any content like a folder the destinationIds can point to nodes that do aka. the first displayable child node Data map[string]interface{} `json:"data"` // what ever you want to stuff into it - the payload you want to attach to a node Nodes map[string]*RepoNode `json:"nodes"` // child nodes Index []string `json:"index"` // defines the order of the child nodes @@ -21,6 +22,16 @@ type RepoNode struct { // published from - to is going to be an array of fromTos } +// NewRepoNode constructor +func NewRepoNode() *RepoNode { + return &RepoNode{ + Data: make(map[string]interface{}), + Nodes: make(map[string]*RepoNode), + } +} + +// WireParents helper method to reference from child to parent in a tree +// recursively func (node *RepoNode) WireParents() { for _, childNode := range node.Nodes { childNode.parent = node @@ -28,16 +39,18 @@ func (node *RepoNode) WireParents() { } } +// InPath is the given node in a path func (node *RepoNode) InPath(path []*Item) bool { - myParentId := node.parent.Id + myParentID := node.parent.ID for _, pathItem := range path { - if pathItem.Id == myParentId { + if pathItem.ID == myParentID { return true } } return false } +// GetPath get a path for a repo node func (node *RepoNode) GetPath() []*Item { parentNode := node.parent pathLength := 0 @@ -56,9 +69,10 @@ func (node *RepoNode) GetPath() []*Item { return path } +// ToItem convert a re po node to a simple repo item func (node *RepoNode) ToItem(dataFields []string) *Item { item := NewItem() - item.Id = node.Id + item.ID = node.ID item.Name = node.Name item.MimeType = node.MimeType item.URI = node.URI @@ -70,53 +84,51 @@ func (node *RepoNode) ToItem(dataFields []string) *Item { return item } +// GetParent get the parent node of a node func (node *RepoNode) GetParent() *RepoNode { return node.parent } +// AddNode adds a named child node func (node *RepoNode) AddNode(name string, childNode *RepoNode) *RepoNode { node.Nodes[name] = childNode return node } +// IsOneOfTheseMimeTypes is the node one of the given mime types func (node *RepoNode) IsOneOfTheseMimeTypes(mimeTypes []string) bool { if len(mimeTypes) == 0 { return true - } else { - for _, mimeType := range mimeTypes { - if mimeType == node.MimeType { - return true - } - } - return false } + for _, mimeType := range mimeTypes { + if mimeType == node.MimeType { + return true + } + } + return false } +// CanBeAccessedByGroups can this node be accessed by at least one the given +// groups func (node *RepoNode) CanBeAccessedByGroups(groups []string) bool { if len(groups) == 0 || len(node.Groups) == 0 { return true - } else { - // @todo is there sth like in_array ... or some array intersection - for _, group := range groups { - for _, myGroup := range node.Groups { - if group == myGroup { - return true - } + } + for _, group := range groups { + for _, myGroup := range node.Groups { + if group == myGroup { + return true } } - return false } + return false } +// PrintNode essentially a recursive dump func (node *RepoNode) PrintNode(id string, level int) { - prefix := strings.Repeat(INDENT, level) + prefix := strings.Repeat(Indent, level) fmt.Printf("%s %s %s:\n", prefix, id, node.Name) for key, childNode := range node.Nodes { childNode.PrintNode(key, level+1) } } - -func NewRepoNode() *RepoNode { - node := new(RepoNode) - return node -} diff --git a/server/repo/content/site_content_example.json b/content/site_content_example.json similarity index 99% rename from server/repo/content/site_content_example.json rename to content/site_content_example.json index f3b1342..49495f0 100644 --- a/server/repo/content/site_content_example.json +++ b/content/site_content_example.json @@ -12,7 +12,7 @@ }, "meta" : { - + } }, "content" : { @@ -41,4 +41,4 @@ } // no other regions for this one } -} \ No newline at end of file +} diff --git a/content/sitecontent.go b/content/sitecontent.go new file mode 100644 index 0000000..68a78e2 --- /dev/null +++ b/content/sitecontent.go @@ -0,0 +1,34 @@ +package content + +// Status status type SiteContent respnses +type Status int + +const ( + // StatusOk we found content + StatusOk Status = 200 + // StatusForbidden we found content but you mst not access it + StatusForbidden = 403 + // StatusNotFound we did not find content + StatusNotFound = 404 +) + +// SiteContent resolved content for a site +type SiteContent struct { + Status Status `json:"status"` + URI string `json:"URI"` + Dimension string `json:"dimension"` + MimeType string `json:"mimeType"` + Item *Item `json:"item"` + Data interface{} `json:"data"` + Path []*Item `json:"path"` + URIs map[string]string `json:"URIs"` + Nodes map[string]*Node `json:"nodes"` +} + +// NewSiteContent constructor +func NewSiteContent() *SiteContent { + return &SiteContent{ + Nodes: make(map[string]*Node), + URIs: make(map[string]string), + } +} diff --git a/contentserver.go b/contentserver.go index ef4aa15..303a8d4 100644 --- a/contentserver.go +++ b/contentserver.go @@ -6,19 +6,16 @@ import ( "os" "strings" + "github.com/foomo/contentserver/log" "github.com/foomo/contentserver/server" - "github.com/foomo/contentserver/server/log" ) const ( - PROTOCOL_TCP = "tcp" -) - -type ExitCode int - -const ( - EXIT_CODE_OK = 0 - EXIT_CODE_INSUFFICIENT_ARGS = 1 + logLevelDebug = "debug" + logLevelNotice = "notice" + logLevelWarning = "warning" + logLevelRecord = "record" + logLevelError = "error" ) var contentServer string @@ -26,19 +23,19 @@ var contentServer string var uniqushPushVersion = "content-server 1.2.0" var showVersionFlag = flag.Bool("version", false, "Version info") -var protocol = flag.String("protocol", PROTOCOL_TCP, "what protocol to server for") var address = flag.String("address", "127.0.0.1:8081", "address to bind host:port") -var varDir = flag.String("vardir", "127.0.0.1:8081", "where to put my data") +var varDir = flag.String("var-dir", "/var/lib/contentserver", "where to put my data") var logLevelOptions = []string{ - log.LOG_LEVEL_NAME_ERROR, - log.LOG_LEVEL_NAME_RECORD, - log.LOG_LEVEL_NAME_WARNING, - log.LOG_LEVEL_NAME_NOTICE, - log.LOG_LEVEL_NAME_DEBUG} + logLevelError, + logLevelRecord, + logLevelWarning, + logLevelNotice, + logLevelDebug, +} var logLevel = flag.String( - "logLevel", - log.LOG_LEVEL_NAME_RECORD, + "log-level", + logLevelRecord, fmt.Sprintf( "one of %s", strings.Join(logLevelOptions, ", "))) @@ -56,16 +53,24 @@ func main() { return } if len(flag.Args()) == 1 { - fmt.Println(*protocol, *address, flag.Arg(0)) - log.SetLogLevel(log.GetLogLevelByName(*logLevel)) - switch *protocol { - case PROTOCOL_TCP: - server.RunSocketServer(flag.Arg(0), *address, *varDir) - break - default: - exitUsage(EXIT_CODE_INSUFFICIENT_ARGS) + fmt.Println(*address, flag.Arg(0)) + level := log.LevelRecord + switch *logLevel { + case logLevelError: + level = log.LevelError + case logLevelRecord: + level = log.LevelRecord + case logLevelWarning: + level = log.LevelWarning + case logLevelNotice: + level = log.LevelNotice + case logLevelDebug: + level = log.LevelDebug + } + log.SelectedLevel = level + server.RunSocketServer(flag.Arg(0), *address, *varDir) } else { - exitUsage(EXIT_CODE_INSUFFICIENT_ARGS) + exitUsage(1) } } diff --git a/log/log.go b/log/log.go index 8f41644..e2db1ef 100644 --- a/log/log.go +++ b/log/log.go @@ -6,24 +6,24 @@ import ( "time" ) -// LogLevel logging level enum +// Level logging level enum type Level int const ( // LevelError an error - as bad as it gets LevelError Level = 0 // LevelRecord put this to the logs in any case - LevelRecord = 1 + LevelRecord Level = 1 // LevelWarning not that bad - LevelWarning = 2 + LevelWarning Level = 2 // LevelNotice almost on debug level - LevelNotice = 3 + LevelNotice Level = 3 // LevelDebug we are debugging - LevelDebug = 4 + LevelDebug Level = 4 ) // SelectedLevel selected log level -var SelectedLevel Level = LevelDebug +var SelectedLevel = LevelDebug var prefices = map[Level]string{ LevelRecord: "record : ", diff --git a/pkg/README.md b/pkg/README.md new file mode 100644 index 0000000..cec37b8 --- /dev/null +++ b/pkg/README.md @@ -0,0 +1,44 @@ +Packaging & Deployment +---------------------- + +In order to build packages and upload to Package Cloud, please install the following requirements and run the make task. + +[Package Cloud Command Line Client](https://packagecloud.io/docs#cli_install) + +``` +$ gem install package_cloud +``` + +[FPM](https://github.com/jordansissel/fpm) + +``` +$ gem install fpm +``` + +Building package + +``` +$ make package +``` + +*NOTE: you will be prompted for Package Cloud credentials.* + +Testing +------- + +``` +$ git clone https://github.com/foomo/contentserver.git +$ cd contentserver +$ make test +``` + +Contributing +------------ + +In lieu of a formal styleguide, take care to maintain the existing coding style. Add unit tests and examples for any new or changed functionality. + +1. Fork it +2. Create your feature branch (`git checkout -b my-new-feature`\) +3. Commit your changes (`git commit -am 'Add some feature'`\) +4. Push to the branch (`git push origin my-new-feature`\) +5. Create new Pull Request diff --git a/server/repo/history.go b/repo/history.go similarity index 100% rename from server/repo/history.go rename to repo/history.go diff --git a/server/repo/loader.go b/repo/loader.go similarity index 84% rename from server/repo/loader.go rename to repo/loader.go index 01f832e..86bd6d0 100644 --- a/server/repo/loader.go +++ b/repo/loader.go @@ -3,13 +3,14 @@ package repo import ( "encoding/json" "errors" + "fmt" + "io/ioutil" + "net/http" "time" - "github.com/foomo/contentserver/server/log" - "github.com/foomo/contentserver/server/repo/content" - "github.com/foomo/contentserver/server/responses" - "github.com/foomo/contentserver/server/utils" - //golog "log" + "github.com/foomo/contentserver/content" + "github.com/foomo/contentserver/log" + "github.com/foomo/contentserver/responses" ) func (repo *Repo) updateRoutine() { @@ -31,7 +32,7 @@ func (repo *Repo) updateRoutine() { } func (repo *Repo) updateDimension(dimension string, node *content.RepoNode) error { - repo.updateChannel <- &RepoDimension{ + repo.updateChannel <- &repoDimension{ Dimension: dimension, Node: node, } @@ -61,15 +62,15 @@ func (repo *Repo) _updateDimension(dimension string, newNode *content.RepoNode) } func builDirectory(dirNode *content.RepoNode, directory map[string]*content.RepoNode, uRIDirectory map[string]*content.RepoNode) error { - log.Debug("repo.buildDirectory: " + dirNode.Id) - existingNode, ok := directory[dirNode.Id] + log.Debug("repo.buildDirectory: " + dirNode.ID) + existingNode, ok := directory[dirNode.ID] if ok { - return errors.New("duplicate node with id:" + existingNode.Id) + return errors.New("duplicate node with id:" + existingNode.ID) } - directory[dirNode.Id] = dirNode + directory[dirNode.ID] = dirNode //todo handle duplicate uris if _, thereIsAnExistingURINode := uRIDirectory[dirNode.URI]; thereIsAnExistingURINode { - return errors.New("duplicate uri: " + dirNode.URI + " (bad node id: " + dirNode.Id + ")") + return errors.New("duplicate uri: " + dirNode.URI + " (bad node id: " + dirNode.ID + ")") } uRIDirectory[dirNode.URI] = dirNode for _, childNode := range dirNode.Nodes { @@ -83,11 +84,11 @@ func builDirectory(dirNode *content.RepoNode, directory map[string]*content.Repo func wireAliases(directory map[string]*content.RepoNode) error { for _, repoNode := range directory { - if len(repoNode.LinkId) > 0 { - if destinationNode, ok := directory[repoNode.LinkId]; ok { + if len(repoNode.LinkID) > 0 { + if destinationNode, ok := directory[repoNode.LinkID]; ok { repoNode.URI = destinationNode.URI } else { - return errors.New("that link id points nowhere " + repoNode.LinkId + " from " + repoNode.Id) + return errors.New("that link id points nowhere " + repoNode.LinkID + " from " + repoNode.ID) } } } @@ -148,14 +149,28 @@ func (repo *Repo) tryToRestoreCurrent() error { return repo.loadJSONBytes(currentJSONBytes) } +func get(URL string) (data []byte, err error) { + response, err := http.Get(URL) + if err != nil { + return data, err + } + defer response.Body.Close() + if response.StatusCode != http.StatusOK { + return data, fmt.Errorf("Bad HTTP Response: %q", response.Status) + } + return ioutil.ReadAll(response.Body) +} + func (repo *Repo) update() (repoRuntime int64, jsonBytes []byte, err error) { startTimeRepo := time.Now().UnixNano() - jsonBytes, err = utils.Get(repo.server) + jsonBytes, err = get(repo.server) repoRuntime = time.Now().UnixNano() - startTimeRepo if err != nil { // we have no json to load - the repo server did not reply + log.Debug("we have no json to load - the repo server did not reply", err) return repoRuntime, jsonBytes, err } + log.Debug("loading json from: "+repo.server, string(jsonBytes)) nodes, err := loadNodesFromJSON(jsonBytes) if err != nil { // could not load nodes from json @@ -172,6 +187,7 @@ func (repo *Repo) update() (repoRuntime int64, jsonBytes []byte, err error) { func (repo *Repo) loadJSONBytes(jsonBytes []byte) error { nodes, err := loadNodesFromJSON(jsonBytes) if err != nil { + log.Debug("could not parse json", string(jsonBytes)) return err } err = repo.loadNodes(nodes) diff --git a/server/repo/mock/repo-broken-json.json b/repo/mock/repo-broken-json.json similarity index 100% rename from server/repo/mock/repo-broken-json.json rename to repo/mock/repo-broken-json.json diff --git a/server/repo/mock/repo-duplicate-uris.json b/repo/mock/repo-duplicate-uris.json similarity index 100% rename from server/repo/mock/repo-duplicate-uris.json rename to repo/mock/repo-duplicate-uris.json diff --git a/server/repo/mock/repo-link-broken.json b/repo/mock/repo-link-broken.json similarity index 100% rename from server/repo/mock/repo-link-broken.json rename to repo/mock/repo-link-broken.json diff --git a/server/repo/mock/repo-link-ok.json b/repo/mock/repo-link-ok.json similarity index 100% rename from server/repo/mock/repo-link-ok.json rename to repo/mock/repo-link-ok.json diff --git a/server/repo/mock/repo-ok.json b/repo/mock/repo-ok.json similarity index 100% rename from server/repo/mock/repo-ok.json rename to repo/mock/repo-ok.json diff --git a/server/repo/mock/repo-two-dimensions.json b/repo/mock/repo-two-dimensions.json similarity index 100% rename from server/repo/mock/repo-two-dimensions.json rename to repo/mock/repo-two-dimensions.json diff --git a/repo/repo.go b/repo/repo.go new file mode 100644 index 0000000..00308d6 --- /dev/null +++ b/repo/repo.go @@ -0,0 +1,259 @@ +package repo + +import ( + "errors" + "fmt" + "strings" + + "github.com/foomo/contentserver/content" + "github.com/foomo/contentserver/log" + "github.com/foomo/contentserver/requests" +) + +// Dimension dimension in a repo +type Dimension struct { + Directory map[string]*content.RepoNode + URIDirectory map[string]*content.RepoNode + Node *content.RepoNode +} + +// Repo content repositiory +type Repo struct { + server string + Directory map[string]*Dimension + updateChannel chan *repoDimension + updateDoneChannel chan error + history *history +} + +type repoDimension struct { + Dimension string + Node *content.RepoNode +} + +// NewRepo constructor +func NewRepo(server string, varDir string) *Repo { + log.Notice("creating new repo for " + server) + log.Notice(" using var dir:" + varDir) + repo := new(Repo) + repo.Directory = make(map[string]*Dimension) + repo.server = server + repo.history = newHistory(varDir) + repo.updateChannel = make(chan *repoDimension) + repo.updateDoneChannel = make(chan error) + go repo.updateRoutine() + log.Record("trying to restore pervious state") + restoreErr := repo.tryToRestoreCurrent() + if restoreErr != nil { + log.Record(" could not restore previous repo content:" + restoreErr.Error()) + } else { + log.Record(" restored previous repo content") + } + return repo +} + +// ResolveContent find content in a repository +func (repo *Repo) ResolveContent(dimensions []string, URI string) (resolved bool, resolvedURI string, resolvedDimension string, repoNode *content.RepoNode) { + parts := strings.Split(URI, content.PathSeparator) + resolved = false + resolvedURI = "" + resolvedDimension = "" + repoNode = nil + log.Debug("repo.ResolveContent: " + URI) + for _, dimension := range dimensions { + if d, ok := repo.Directory[dimension]; ok { + for i := len(parts); i > 0; i-- { + testURI := strings.Join(parts[0:i], content.PathSeparator) + if testURI == "" { + testURI = content.PathSeparator + } + log.Debug(" testing[" + dimension + "]: " + testURI) + if repoNode, ok := d.URIDirectory[testURI]; ok { + resolved = true + log.Debug(" found => " + testURI) + log.Debug(" destination " + fmt.Sprint(repoNode.DestinationID)) + if len(repoNode.DestinationID) > 0 { + if destionationNode, destinationNodeOk := d.Directory[repoNode.DestinationID]; destinationNodeOk { + repoNode = destionationNode + } + } + return true, testURI, dimension, repoNode + } + } + } + } + return +} + +// GetURIs get many uris at once +func (repo *Repo) GetURIs(dimension string, ids []string) map[string]string { + uris := make(map[string]string) + for _, id := range ids { + uris[id] = repo.getURI(dimension, id) + } + return uris +} + +func (repo *Repo) getURIForNode(dimension string, repoNode *content.RepoNode) string { + + if len(repoNode.LinkID) == 0 { + return repoNode.URI + } + linkedNode, ok := repo.Directory[dimension].Directory[repoNode.LinkID] + if ok { + return repo.getURIForNode(dimension, linkedNode) + } + return "" +} + +func (repo *Repo) getURI(dimension string, id string) string { + repoNode, ok := repo.Directory[dimension].Directory[id] + if ok { + return repo.getURIForNode(dimension, repoNode) + } + return "" +} + +func (repo *Repo) getNode(repoNode *content.RepoNode, expanded bool, mimeTypes []string, path []*content.Item, level int, groups []string, dataFields []string) *content.Node { + node := content.NewNode() + node.Item = repoNode.ToItem(dataFields) + log.Debug("repo.GetNode: " + repoNode.ID) + for _, childID := range repoNode.Index { + childNode := repoNode.Nodes[childID] + if (level == 0 || expanded || !expanded && childNode.InPath(path)) && !childNode.Hidden && childNode.CanBeAccessedByGroups(groups) && childNode.IsOneOfTheseMimeTypes(mimeTypes) { + node.Nodes[childID] = repo.getNode(childNode, expanded, mimeTypes, path, level+1, groups, dataFields) + node.Index = append(node.Index, childID) + } + } + return node +} + +// GetNodes get nodes +func (repo *Repo) GetNodes(r *requests.Nodes) map[string]*content.Node { + nodes := make(map[string]*content.Node) + path := []*content.Item{} + for nodeName, nodeRequest := range r.Nodes { + log.Debug(" adding node " + nodeName + " " + nodeRequest.ID) + dimensionNode, ok := repo.Directory[nodeRequest.Dimension] + nodes[nodeName] = nil + if !ok { + log.Warning("could not get dimension root node for nodeRequest.Dimension: " + nodeRequest.Dimension) + continue + } + if treeNode, ok := dimensionNode.Directory[nodeRequest.ID]; ok { + nodes[nodeName] = repo.getNode(treeNode, nodeRequest.Expand, nodeRequest.MimeTypes, path, 0, r.Env.Groups, nodeRequest.DataFields) + } else { + log.Warning("you are requesting an invalid tree node for " + nodeName + " : " + nodeRequest.ID) + } + } + return nodes +} + +func (repo *Repo) validateContentRequest(req *requests.Content) (err error) { + if req == nil { + return errors.New("request must not be nil") + } + if len(req.URI) == 0 { + return errors.New("request URI must not be empty") + } + + if req.Env == nil { + return errors.New("request.Env must not be nil") + } + if len(req.Env.Dimensions) == 0 { + return errors.New("request.Env.Dimensions must not be empty") + } + + for _, envDimension := range req.Env.Dimensions { + if !repo.hasDimension(envDimension) { + availableDimensions := []string{} + for availableDimension := range repo.Directory { + availableDimensions = append(availableDimensions, availableDimension) + } + return fmt.Errorf("unknown dimension %q in r.Env must be one of %q", envDimension, availableDimensions) + } + } + return nil +} + +func (repo *Repo) hasDimension(d string) bool { + _, hasDimension := repo.Directory[d] + return hasDimension +} + +// GetContent resolves content and fetches nodes in one call. It combines those +// two tasks for performance reasons. +// +// In the first step it uses r.URI to look up content in all given +// r.Env.Dimensions of repo.Directory. +// +// In the second step it collects the requested nodes. +// +// those two steps are independent. +func (repo *Repo) GetContent(r *requests.Content) (c *content.SiteContent, err error) { + // add more input validation + err = repo.validateContentRequest(r) + if err != nil { + log.Debug("repo.GetContent invalid request", err) + return + } + log.Debug("repo.GetContent: ", r.URI) + c = content.NewSiteContent() + resolved, resolvedURI, resolvedDimension, node := repo.ResolveContent(r.Env.Dimensions, r.URI) + if resolved { + log.Notice("200 for " + r.URI) + // forbidden ?! + c.Status = content.StatusOk + c.MimeType = node.MimeType + c.Dimension = resolvedDimension + c.URI = resolvedURI + c.Item = node.ToItem([]string{}) + c.Path = node.GetPath() + c.Data = node.Data + // fetch URIs for all dimensions + uris := make(map[string]string) + for dimensionName := range repo.Directory { + uris[dimensionName] = repo.getURI(dimensionName, node.ID) + } + c.URIs = uris + } else { + log.Notice("404 for " + r.URI) + c.Status = content.StatusNotFound + c.Dimension = r.Env.Dimensions[0] + } + log.Debug(fmt.Sprintf("resolved: %v, uri: %v, dim: %v, n: %v", resolved, resolvedURI, resolvedDimension, node)) + if resolved == false { + log.Debug("repo.GetContent", r.URI, "could not be resolved falling back to default dimension", r.Env.Dimensions[0]) + // r.Env.Dimensions is validated => we can access it + resolvedDimension = r.Env.Dimensions[0] + } + // add navigation trees + for treeName, treeRequest := range r.Nodes { + log.Debug(" adding tree " + treeName + " " + treeRequest.ID) + rootNode, ok := repo.Directory[resolvedDimension] + if !ok { + log.Warning("could not resolve rootNode for resolved dimension " + resolvedDimension) + c.Nodes[treeName] = nil + continue + } + if treeNode, ok := rootNode.Directory[treeRequest.ID]; ok { + c.Nodes[treeName] = repo.getNode(treeNode, treeRequest.Expand, treeRequest.MimeTypes, c.Path, 0, r.Env.Groups, treeRequest.DataFields) + } else { + log.Warning("you are requesting an invalid tree node for " + treeName + " : " + treeRequest.ID) + } + } + return c, nil +} + +// GetRepo get the whole repo in all dimensions +func (repo *Repo) GetRepo() map[string]*content.RepoNode { + response := make(map[string]*content.RepoNode) + for dimensionName, dimension := range repo.Directory { + response[dimensionName] = dimension.Node + } + return response +} + +func uriKeyForState(state string, uri string) string { + return state + "-" + uri +} diff --git a/server/repo/repo_test.go b/repo/repo_test.go similarity index 76% rename from server/repo/repo_test.go rename to repo/repo_test.go index 4353d69..34f7cf6 100644 --- a/server/repo/repo_test.go +++ b/repo/repo_test.go @@ -10,7 +10,7 @@ import ( "testing" "time" - "github.com/foomo/contentserver/server/requests" + "github.com/foomo/contentserver/requests" ) func getMockData(t *testing.T) (server *httptest.Server, varDir string) { @@ -116,35 +116,29 @@ func TestDimensionHygiene(t *testing.T) { } } -func TestResolveContent(t *testing.T) { +func getTestRepo(path string, t *testing.T) *Repo { mockServer, varDir := getMockData(t) - server := mockServer.URL + "/repo-two-dimensions.json" + server := mockServer.URL + path r := NewRepo(server, varDir) response := r.Update() if !response.Success { t.Fatal("well those two dimension should be fine") } - dimensions := []string{"dimension_foo"} - contentRequest := &requests.Content{ - URI: "/a", - Env: &requests.Env{ - Dimensions: dimensions, - Groups: []string{}, - }, - Nodes: map[string]*requests.Node{ - "id-root": &requests.Node{ - Id: "id-root", - Dimension: dimensions[0], - MimeTypes: []string{"application/x-node"}, - Expand: true, - DataFields: []string{}, - }, - }, - } - siteContent := r.GetContent(contentRequest) + return r +} + +func TestResolveContent(t *testing.T) { + r := getTestRepo("/repo-two-dimensions.json", t) + + contentRequest := makeValidRequest() + + siteContent, err := r.GetContent(contentRequest) if siteContent.URI != contentRequest.URI { t.Fatal("failed to resolve uri") } + if err != nil { + t.Fatal(err) + } } func TestLinkIds(t *testing.T) { @@ -164,3 +158,57 @@ func TestLinkIds(t *testing.T) { } } + +func makeValidRequest() *requests.Content { + dimensions := []string{"dimension_foo"} + return &requests.Content{ + URI: "/a", + Env: &requests.Env{ + Dimensions: dimensions, + Groups: []string{}, + }, + Nodes: map[string]*requests.Node{ + "id-root": &requests.Node{ + ID: "id-root", + Dimension: dimensions[0], + MimeTypes: []string{"application/x-node"}, + Expand: true, + DataFields: []string{}, + }, + }, + } + +} + +func TestInvalidRequest(t *testing.T) { + + r := getTestRepo("/repo-two-dimensions.json", t) + + if r.validateContentRequest(makeValidRequest()) != nil { + t.Fatal("failed validation a valid request") + } + + tests := map[string]*requests.Content{} + + rEmptyURI := makeValidRequest() + rEmptyURI.URI = "" + tests["empty uri"] = rEmptyURI + + rEmptyEnv := makeValidRequest() + rEmptyEnv.Env = nil + tests["empty env"] = rEmptyEnv + + rEmptyEnvDimensions := makeValidRequest() + rEmptyEnvDimensions.Env.Dimensions = []string{} + tests["empty env dimensions"] = rEmptyEnvDimensions + + //rNodesValidID := makeValidRequest() + //rNodesValidID.Nodes["id-root"].Id = "" + //tests["nodes must have a valid id"] = rNodesValidID + + for comment, req := range tests { + if r.validateContentRequest(req) == nil { + t.Fatal(comment, "should have failed") + } + } +} diff --git a/server/requests/requests.go b/requests/requests.go similarity index 96% rename from server/requests/requests.go rename to requests/requests.go index 908cf21..ccb29dd 100644 --- a/server/requests/requests.go +++ b/requests/requests.go @@ -13,7 +13,7 @@ type Env struct { // Node - an abdtract node request, use this one to request navigations type Node struct { // this one should be obvious - Id string `json:"id"` + ID string `json:"id"` // from which dimension Dimension string `json:"dimension"` // what do you want to see in your navigations, folders, images or unicorns @@ -49,7 +49,7 @@ type Repo struct { // ItemMap - map of items type ItemMap struct { - Id string `json:"id"` + ID string `json:"id"` DataFields []string `json:"dataFields"` } diff --git a/server/responses/responses.go b/responses/responses.go similarity index 100% rename from server/responses/responses.go rename to responses/responses.go diff --git a/server/repo/content/constants.go b/server/repo/content/constants.go deleted file mode 100644 index 6fc2024..0000000 --- a/server/repo/content/constants.go +++ /dev/null @@ -1,6 +0,0 @@ -package content - -const ( - INDENT string = "\t" - PATH_SEPARATOR = "/" -) diff --git a/server/repo/content/site_content.go b/server/repo/content/site_content.go deleted file mode 100644 index ea07ba8..0000000 --- a/server/repo/content/site_content.go +++ /dev/null @@ -1,26 +0,0 @@ -package content - -const ( - STATUS_OK = 200 - STATUS_FORBIDDEN = 403 - STATUS_NOT_FOUND = 404 -) - -type SiteContent struct { - Status int `json:"status"` - URI string `json:"URI"` - Dimension string `json:"dimension"` - MimeType string `json:"mimeType"` - Item *Item `json:"item"` - Data interface{} `json:"data"` - Path []*Item `json:"path"` - URIs map[string]string `json:"URIs"` - Nodes map[string]*Node `json:"nodes"` -} - -func NewSiteContent() *SiteContent { - c := new(SiteContent) - c.Nodes = make(map[string]*Node) - c.URIs = make(map[string]string) - return c -} diff --git a/server/repo/repo.go b/server/repo/repo.go deleted file mode 100644 index afa0a86..0000000 --- a/server/repo/repo.go +++ /dev/null @@ -1,193 +0,0 @@ -package repo - -import ( - "fmt" - "strings" - - "github.com/foomo/contentserver/server/log" - "github.com/foomo/contentserver/server/repo/content" - "github.com/foomo/contentserver/server/requests" -) - -type Dimension struct { - Directory map[string]*content.RepoNode - URIDirectory map[string]*content.RepoNode - Node *content.RepoNode -} - -type RepoDimension struct { - Dimension string - Node *content.RepoNode -} - -type Repo struct { - server string - Directory map[string]*Dimension - updateChannel chan *RepoDimension - updateDoneChannel chan error - history *history -} - -func NewRepo(server string, varDir string) *Repo { - log.Notice("creating new repo for " + server) - log.Notice(" using var dir:" + varDir) - repo := new(Repo) - repo.Directory = make(map[string]*Dimension) - repo.server = server - repo.history = newHistory(varDir) - repo.updateChannel = make(chan *RepoDimension) - repo.updateDoneChannel = make(chan error) - go repo.updateRoutine() - log.Record("trying to restore pervious state") - restoreErr := repo.tryToRestoreCurrent() - if restoreErr != nil { - log.Record(" could not restore previous repo content:" + restoreErr.Error()) - } else { - log.Record(" restored previous repo content") - } - return repo -} - -func (repo *Repo) ResolveContent(dimensions []string, URI string) (resolved bool, resolvedURI string, resolvedDimension string, repoNode *content.RepoNode) { - parts := strings.Split(URI, content.PATH_SEPARATOR) - resolved = false - resolvedURI = "" - resolvedDimension = "" - repoNode = nil - log.Debug("repo.ResolveContent: " + URI) - for _, dimension := range dimensions { - if d, ok := repo.Directory[dimension]; ok { - for i := len(parts); i > 0; i-- { - testURI := strings.Join(parts[0:i], content.PATH_SEPARATOR) - if testURI == "" { - testURI = content.PATH_SEPARATOR - } - log.Debug(" testing[" + dimension + "]: " + testURI) - if repoNode, ok := d.URIDirectory[testURI]; ok { - resolved = true - log.Debug(" found => " + testURI) - log.Debug(" destination " + fmt.Sprint(repoNode.DestinationId)) - if len(repoNode.DestinationId) > 0 { - if destionationNode, destinationNodeOk := d.Directory[repoNode.DestinationId]; destinationNodeOk { - repoNode = destionationNode - } - } - return true, testURI, dimension, repoNode - } - } - } - } - return -} - -func (repo *Repo) GetURIs(dimension string, ids []string) map[string]string { - uris := make(map[string]string) - for _, id := range ids { - uris[id] = repo.GetURI(dimension, id) - } - return uris -} - -func (repo *Repo) GetURIForNode(dimension string, repoNode *content.RepoNode) string { - - if len(repoNode.LinkId) == 0 { - return repoNode.URI - } else { - linkedNode, ok := repo.Directory[dimension].Directory[repoNode.LinkId] - if ok { - return repo.GetURIForNode(dimension, linkedNode) - } else { - return "" - } - } -} - -func (repo *Repo) GetURI(dimension string, id string) string { - repoNode, ok := repo.Directory[dimension].Directory[id] - if ok { - return repo.GetURIForNode(dimension, repoNode) - } - return "" -} - -func (repo *Repo) GetNode(repoNode *content.RepoNode, expanded bool, mimeTypes []string, path []*content.Item, level int, groups []string, dataFields []string) *content.Node { - node := content.NewNode() - node.Item = repoNode.ToItem(dataFields) - log.Debug("repo.GetNode: " + repoNode.Id) - for _, childId := range repoNode.Index { - childNode := repoNode.Nodes[childId] - if (level == 0 || expanded || !expanded && childNode.InPath(path)) && !childNode.Hidden && childNode.CanBeAccessedByGroups(groups) && childNode.IsOneOfTheseMimeTypes(mimeTypes) { - node.Nodes[childId] = repo.GetNode(childNode, expanded, mimeTypes, path, level+1, groups, dataFields) - node.Index = append(node.Index, childId) - } - } - return node -} - -func (repo *Repo) GetNodes(r *requests.Nodes) map[string]*content.Node { - nodes := make(map[string]*content.Node) - path := make([]*content.Item, 0) - for nodeName, nodeRequest := range r.Nodes { - log.Debug(" adding node " + nodeName + " " + nodeRequest.Id) - if treeNode, ok := repo.Directory[nodeRequest.Dimension].Directory[nodeRequest.Id]; ok { - nodes[nodeName] = repo.GetNode(treeNode, nodeRequest.Expand, nodeRequest.MimeTypes, path, 0, r.Env.Groups, nodeRequest.DataFields) - } else { - log.Warning("you are requesting an invalid tree node for " + nodeName + " : " + nodeRequest.Id) - } - } - return nodes -} - -func (repo *Repo) GetContent(r *requests.Content) *content.SiteContent { - // add more input validation - log.Debug("repo.GetContent: " + r.URI) - c := content.NewSiteContent() - resolved, resolvedURI, resolvedDimension, node := repo.ResolveContent(r.Env.Dimensions, r.URI) - if resolved { - log.Notice("200 for " + r.URI) - // forbidden ?! - c.Status = content.STATUS_OK - c.MimeType = node.MimeType - c.Dimension = resolvedDimension - c.URI = resolvedURI - c.Item = node.ToItem([]string{}) - c.Path = node.GetPath() - c.Data = node.Data - // fetch URIs for all dimensions - uris := make(map[string]string) - for dimensionName, _ := range repo.Directory { - uris[dimensionName] = repo.GetURI(dimensionName, node.Id) - } - c.URIs = uris - } else { - log.Notice("404 for " + r.URI) - c.Status = content.STATUS_NOT_FOUND - c.Dimension = r.Env.Dimensions[0] - } - log.Debug(fmt.Sprintf("resolved: %v, uri: %v, dim: %v, n: %v", resolved, resolvedURI, resolvedDimension, node)) - if resolved == false { - resolvedDimension = r.Env.Dimensions[0] - } - // add navigation trees - for treeName, treeRequest := range r.Nodes { - log.Debug(" adding tree " + treeName + " " + treeRequest.Id) - if treeNode, ok := repo.Directory[resolvedDimension].Directory[treeRequest.Id]; ok { - c.Nodes[treeName] = repo.GetNode(treeNode, treeRequest.Expand, treeRequest.MimeTypes, c.Path, 0, r.Env.Groups, treeRequest.DataFields) - } else { - log.Warning("you are requesting an invalid tree node for " + treeName + " : " + treeRequest.Id) - } - } - return c -} - -func (repo *Repo) GetRepo() map[string]*content.RepoNode { - response := make(map[string]*content.RepoNode) - for dimensionName, dimension := range repo.Directory { - response[dimensionName] = dimension.Node - } - return response -} - -func uriKeyForState(state string, uri string) string { - return state + "-" + uri -} diff --git a/server/server.go b/server/server.go index ab85d89..31aecaf 100644 --- a/server/server.go +++ b/server/server.go @@ -1,28 +1,28 @@ package server -import ( - "github.com/foomo/contentserver/server/repo" -) - -// our data - -type Stats struct { - requests int64 +type stats struct { + requests int64 + chanCount chan int } -func NewStats() *Stats { - stats := new(Stats) - stats.requests = 0 - return stats +func newStats() *stats { + s := &stats{ + requests: 0, + chanCount: make(chan int), + } + go func() { + for { + select { + case <-s.chanCount: + s.requests++ + s.chanCount <- 1 + } + } + }() + return s } -var stats *Stats = NewStats() -var contentRepo *repo.Repo - -func countRequest() { - stats.requests++ -} - -func numRequests() int64 { - return stats.requests +func (s *stats) countRequest() { + s.chanCount <- 1 + <-s.chanCount } diff --git a/server/socket.go b/server/socket.go index 4ca5170..4f05f78 100644 --- a/server/socket.go +++ b/server/socket.go @@ -8,12 +8,17 @@ import ( "strconv" "strings" - "github.com/foomo/contentserver/server/log" - "github.com/foomo/contentserver/server/repo" - "github.com/foomo/contentserver/server/requests" - "github.com/foomo/contentserver/server/responses" + "github.com/foomo/contentserver/log" + "github.com/foomo/contentserver/repo" + "github.com/foomo/contentserver/requests" + "github.com/foomo/contentserver/responses" ) +type socketServer struct { + stats *stats + repo *repo.Repo +} + // there should be sth. built in ?! // anyway this ony concatenates two "ByteArrays" func concat(a []byte, b []byte) []byte { @@ -23,46 +28,48 @@ func concat(a []byte, b []byte) []byte { return newslice } -func handleSocketRequest(handler string, jsonBuffer []byte) (replyBytes []byte, err error) { - countRequest() +func (s *socketServer) handleSocketRequest(handler string, jsonBuffer []byte) (replyBytes []byte, err error) { + s.stats.countRequest() var reply interface{} + var apiErr error var jsonErr error - log.Record(fmt.Sprintf("socket.handleSocketRequest(%d): %s %s", numRequests(), handler, string(jsonBuffer))) + log.Record(fmt.Sprintf("socket.handleSocketRequest(%d): %s %s", s.stats.requests, handler, string(jsonBuffer))) switch handler { case "getURIs": getURIRequest := &requests.URIs{} jsonErr = json.Unmarshal(jsonBuffer, &getURIRequest) log.Debug(" getURIRequest: " + fmt.Sprint(getURIRequest)) - uris := contentRepo.GetURIs(getURIRequest.Dimension, getURIRequest.Ids) + uris := s.repo.GetURIs(getURIRequest.Dimension, getURIRequest.Ids) log.Debug(" resolved: " + fmt.Sprint(uris)) reply = uris break case "content": contentRequest := &requests.Content{} jsonErr = json.Unmarshal(jsonBuffer, &contentRequest) - log.Debug(" contentRequest: " + fmt.Sprint(contentRequest)) - content := contentRepo.GetContent(contentRequest) + log.Debug("contentRequest:", contentRequest) + content, apiErr := s.repo.GetContent(contentRequest) + log.Debug(apiErr) reply = content break case "getNodes": nodesRequest := &requests.Nodes{} jsonErr = json.Unmarshal(jsonBuffer, &nodesRequest) log.Debug(" nodesRequest: " + fmt.Sprint(nodesRequest)) - nodesMap := contentRepo.GetNodes(nodesRequest) + nodesMap := s.repo.GetNodes(nodesRequest) reply = nodesMap break case "update": updateRequest := &requests.Update{} jsonErr = json.Unmarshal(jsonBuffer, &updateRequest) log.Debug(" updateRequest: " + fmt.Sprint(updateRequest)) - updateResponse := contentRepo.Update() + updateResponse := s.repo.Update() reply = updateResponse break case "getRepo": repoRequest := &requests.Repo{} jsonErr = json.Unmarshal(jsonBuffer, &repoRequest) log.Debug(" getRepoRequest: " + fmt.Sprint(repoRequest)) - repoResponse := contentRepo.GetRepo() + repoResponse := s.repo.GetRepo() reply = repoResponse break default: @@ -74,6 +81,8 @@ func handleSocketRequest(handler string, jsonBuffer []byte) (replyBytes []byte, log.Error(" could not read incoming json: " + fmt.Sprint(jsonErr)) errorResponse := responses.NewError(2, "could not read incoming json "+jsonErr.Error()) reply = errorResponse + } else if apiErr != nil { + reply = responses.NewError(3, "internal error "+apiErr.Error()) } encodedBytes, jsonErr := json.MarshalIndent(map[string]interface{}{"reply": reply}, "", " ") if jsonErr != nil { @@ -85,7 +94,7 @@ func handleSocketRequest(handler string, jsonBuffer []byte) (replyBytes []byte, return replyBytes, err } -func handleConnection(conn net.Conn) { +func (s *socketServer) handleConnection(conn net.Conn) { log.Debug("socket.handleConnection") var headerBuffer [1]byte header := "" @@ -113,7 +122,7 @@ func handleConnection(conn net.Conn) { return } log.Debug(" read json: " + string(jsonBuffer)) - reply, handlingError := handleSocketRequest(requestHandler, jsonBuffer) + reply, handlingError := s.handleSocketRequest(requestHandler, jsonBuffer) if handlingError != nil { log.Error("socket.handleConnection: handlingError " + fmt.Sprint(handlingError)) if reply == nil { @@ -146,7 +155,10 @@ func handleConnection(conn net.Conn) { // RunSocketServer - let it run and enjoy on a socket near you func RunSocketServer(server string, address string, varDir string) { log.Record("building repo with content from " + server) - contentRepo = repo.NewRepo(server, varDir) + s := &socketServer{ + stats: newStats(), + repo: repo.NewRepo(server, varDir), + } ln, err := net.Listen("tcp", address) if err != nil { // failed to create socket @@ -154,17 +166,15 @@ func RunSocketServer(server string, address string, varDir string) { } else { // there we go log.Record("RunSocketServer: started to listen on " + address) - if len(contentRepo.Directory) == 0 { - contentRepo.Update() - } + s.repo.Update() for { conn, err := ln.Accept() // this blocks until connection or error if err != nil { log.Error("RunSocketServer: could not accept connection" + fmt.Sprint(err)) continue - } else { - go handleConnection(conn) // a goroutine handles conn so that the loop can accept other connections } + // a goroutine handles conn so that the loop can accept other connections + go s.handleConnection(conn) } } } diff --git a/server/utils/utils.go b/server/utils/utils.go deleted file mode 100644 index 5afaaff..0000000 --- a/server/utils/utils.go +++ /dev/null @@ -1,61 +0,0 @@ -package utils - -import ( - "encoding/json" - "errors" - "fmt" - "io/ioutil" - "net/http" -) - -func JsonResponse(w http.ResponseWriter, obj interface{}) { - fmt.Fprint(w, toJson(obj)) -} - -func ToJSON(obj interface{}) string { - return toJson(obj) -} - -func toJson(obj interface{}) string { - //b, err := json.MarshalIndent(obj, "", "\t") - b, err := json.Marshal(obj) - if err != nil { - return "" - } else { - return string(b) - } -} - -func extractJsonFromRequestFileUpload(r *http.Request) []byte { - file, _, err := r.FormFile("request") - if err != nil { - fmt.Println(err, r) - } - data, err := ioutil.ReadAll(file) - if err != nil { - fmt.Println(err) - } - return data -} -func extractJsonFromRequest(r *http.Request) []byte { - bytes := []byte(r.PostFormValue("request")) - return bytes -} - -func PopulateRequest(r *http.Request, obj interface{}) { - json.Unmarshal(extractJsonFromRequest(r), obj) -} - -func Get(URL string) (data []byte, err error) { - response, err := http.Get(URL) - if err != nil { - return data, err - } else { - defer response.Body.Close() - if response.StatusCode != http.StatusOK { - return data, errors.New(fmt.Sprintf("Bad HTTP Response: %v", response.Status)) - } else { - return ioutil.ReadAll(response.Body) - } - } -}