foomo/contentserver
1
0
Fork 0
mirror of https://github.com/foomo/contentserver.git synced 2025-10-16 12:25:44 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

flattened project structure and listened to golint

Browse Source
This commit is contained in:
Jan Halfar 2016-03-07 15:36:34 +01:00
parent ca2521e9ea
commit e18e19f6ac
29 changed files with 584 additions and 480 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Dockerfile
View File

@ -17,4 +17,4 @@ EXPOSE 80
ENTRYPOINT ["/usr/sbin/contentserver"]
CMD ["-address=$CONTENT_SERVER_ADDR", "-logLevel=CONTENT_SERVER_LOG_LEVEL", "-protocol=$CONTENT_SERVER_PROTOCOL", "-vardir=$CONTENT_SERVER_VAR_DIR"]
CMD ["-address=$CONTENT_SERVER_ADDR", "-logLevel=$CONTENT_SERVER_LOG_LEVEL", "-protocol=$CONTENT_SERVER_PROTOCOL", "-vardir=$CONTENT_SERVER_VAR_DIR"]

45
README.md
View File

@ -55,51 +55,6 @@ Usage of bin/contentserver:
-vardir="127.0.0.1:8081": where to put my data
```
Packaging & Deployment
----------------------
In order to build packages and upload to Package Cloud, please install the following requirements and run the make task.
[Package Cloud Command Line Client](https://packagecloud.io/docs#cli_install)
```
$ gem install package_cloud
```
[FPM](https://github.com/jordansissel/fpm)
```
$ gem install fpm
```
Building package
```
$ make package
```
*NOTE: you will be prompted for Package Cloud credentials.*
Testing
-------
```
$ git clone https://github.com/foomo/contentserver.git
$ cd contentserver
$ make test
```
Contributing
------------
In lieu of a formal styleguide, take care to maintain the existing coding style. Add unit tests and examples for any new or changed functionality.
1. Fork it
2. Create your feature branch (`git checkout -b my-new-feature`\)
3. Commit your changes (`git commit -am 'Add some feature'`\)
4. Push to the branch (`git push origin my-new-feature`\)
5. Create new Pull Request
License
-------

8
content/constants.go Normal file
View File

@ -0,0 +1,8 @@
package content
const (
// Indent for json indentation
Indent string = "\t"
// PathSeparator seprator for paths in URIs
PathSeparator = "/"
)

6
server/repo/content/item.go → content/item.go
View File

@ -1,15 +1,15 @@
package content
import ()
// Item on a node in a content tree - "payload" of an item
type Item struct {
Id string `json:"id"`
ID string `json:"id"`
Name string `json:"name"`
URI string `json:"URI"`
MimeType string `json:"mimeType"`
Data map[string]interface{} `json:"data"`
}
// NewItem item contructor
func NewItem() *Item {
item := new(Item)
item.Data = make(map[string]interface{})

5
server/repo/content/node.go → content/node.go
View File

@ -1,17 +1,16 @@
package content
import ()
// Node a node in a content tree
type Node struct {
Item *Item `json:"item"`
Nodes map[string]*Node `json:"nodes"`
Index []string `json:"index"`
}
// NewNode constructor
func NewNode() *Node {
node := new(Node)
node.Item = NewItem()
//node.Index = []
node.Nodes = make(map[string]*Node)
return node
}

66
server/repo/content/repo_node.go → content/reponode.go
View File

@ -5,15 +5,16 @@ import (
"strings"
)
// RepoNode node in a content tree
type RepoNode struct {
Id string `json:"id"` // unique identifier - it is your responsibility, that they are unique
ID string `json:"id"` // unique identifier - it is your responsibility, that they are unique
MimeType string `json:"mimeType"` // well a mime type http://www.ietf.org/rfc/rfc2046.txt
LinkId string `json:"linkId"` // (symbolic) link/alias to another node
LinkID string `json:"linkId"` // (symbolic) link/alias to another node
Groups []string `json:"groups"` // which groups have access to the node, if empty everybody has access to it
URI string `json:"URI"`
Name string `json:"name"`
Hidden bool `json:"hidden"` // hidden in content.nodes, but can still be resolved when being directly addressed
DestinationId string `json:"destinationId"` // if a node does not have any content like a folder the destinationIds can point to nodes that do aka. the first displayable child node
DestinationID string `json:"destinationId"` // if a node does not have any content like a folder the destinationIds can point to nodes that do aka. the first displayable child node
Data map[string]interface{} `json:"data"` // what ever you want to stuff into it - the payload you want to attach to a node
Nodes map[string]*RepoNode `json:"nodes"` // child nodes
Index []string `json:"index"` // defines the order of the child nodes
@ -21,6 +22,16 @@ type RepoNode struct {
// published from - to is going to be an array of fromTos
}
// NewRepoNode constructor
func NewRepoNode() *RepoNode {
return &RepoNode{
Data: make(map[string]interface{}),
Nodes: make(map[string]*RepoNode),
}
}
// WireParents helper method to reference from child to parent in a tree
// recursively
func (node *RepoNode) WireParents() {
for _, childNode := range node.Nodes {
childNode.parent = node
@ -28,16 +39,18 @@ func (node *RepoNode) WireParents() {
}
}
// InPath is the given node in a path
func (node *RepoNode) InPath(path []*Item) bool {
myParentId := node.parent.Id
myParentID := node.parent.ID
for _, pathItem := range path {
if pathItem.Id == myParentId {
if pathItem.ID == myParentID {
return true
}
}
return false
}
// GetPath get a path for a repo node
func (node *RepoNode) GetPath() []*Item {
parentNode := node.parent
pathLength := 0
@ -56,9 +69,10 @@ func (node *RepoNode) GetPath() []*Item {
return path
}
// ToItem convert a re po node to a simple repo item
func (node *RepoNode) ToItem(dataFields []string) *Item {
item := NewItem()
item.Id = node.Id
item.ID = node.ID
item.Name = node.Name
item.MimeType = node.MimeType
item.URI = node.URI
@ -70,53 +84,51 @@ func (node *RepoNode) ToItem(dataFields []string) *Item {
return item
}
// GetParent get the parent node of a node
func (node *RepoNode) GetParent() *RepoNode {
return node.parent
}
// AddNode adds a named child node
func (node *RepoNode) AddNode(name string, childNode *RepoNode) *RepoNode {
node.Nodes[name] = childNode
return node
}
// IsOneOfTheseMimeTypes is the node one of the given mime types
func (node *RepoNode) IsOneOfTheseMimeTypes(mimeTypes []string) bool {
if len(mimeTypes) == 0 {
return true
} else {
for _, mimeType := range mimeTypes {
if mimeType == node.MimeType {
return true
}
}
return false
}
for _, mimeType := range mimeTypes {
if mimeType == node.MimeType {
return true
}
}
return false
}
// CanBeAccessedByGroups can this node be accessed by at least one the given
// groups
func (node *RepoNode) CanBeAccessedByGroups(groups []string) bool {
if len(groups) == 0 || len(node.Groups) == 0 {
return true
} else {
// @todo is there sth like in_array ... or some array intersection
for _, group := range groups {
for _, myGroup := range node.Groups {
if group == myGroup {
return true
}
}
for _, group := range groups {
for _, myGroup := range node.Groups {
if group == myGroup {
return true
}
}
return false
}
return false
}
// PrintNode essentially a recursive dump
func (node *RepoNode) PrintNode(id string, level int) {
prefix := strings.Repeat(INDENT, level)
prefix := strings.Repeat(Indent, level)
fmt.Printf("%s %s %s:\n", prefix, id, node.Name)
for key, childNode := range node.Nodes {
childNode.PrintNode(key, level+1)
}
}
func NewRepoNode() *RepoNode {
node := new(RepoNode)
return node
}

0
server/repo/content/site_content_example.json → content/site_content_example.json
View File

34
content/sitecontent.go Normal file
View File

@ -0,0 +1,34 @@
package content
// Status status type SiteContent respnses
type Status int
const (
// StatusOk we found content
StatusOk Status = 200
// StatusForbidden we found content but you mst not access it
StatusForbidden = 403
// StatusNotFound we did not find content
StatusNotFound = 404
)
// SiteContent resolved content for a site
type SiteContent struct {
Status Status `json:"status"`
URI string `json:"URI"`
Dimension string `json:"dimension"`
MimeType string `json:"mimeType"`
Item *Item `json:"item"`
Data interface{} `json:"data"`
Path []*Item `json:"path"`
URIs map[string]string `json:"URIs"`
Nodes map[string]*Node `json:"nodes"`
}
// NewSiteContent constructor
func NewSiteContent() *SiteContent {
return &SiteContent{
Nodes: make(map[string]*Node),
URIs: make(map[string]string),
}
}

59
contentserver.go
View File

@ -6,19 +6,16 @@ import (
"os"
"strings"
"github.com/foomo/contentserver/log"
"github.com/foomo/contentserver/server"
"github.com/foomo/contentserver/server/log"
)
const (
PROTOCOL_TCP = "tcp"
)
type ExitCode int
const (
EXIT_CODE_OK = 0
EXIT_CODE_INSUFFICIENT_ARGS = 1
logLevelDebug = "debug"
logLevelNotice = "notice"
logLevelWarning = "warning"
logLevelRecord = "record"
logLevelError = "error"
)
var contentServer string
@ -26,19 +23,19 @@ var contentServer string
var uniqushPushVersion = "content-server 1.2.0"
var showVersionFlag = flag.Bool("version", false, "Version info")
var protocol = flag.String("protocol", PROTOCOL_TCP, "what protocol to server for")
var address = flag.String("address", "127.0.0.1:8081", "address to bind host:port")
var varDir = flag.String("vardir", "127.0.0.1:8081", "where to put my data")
var varDir = flag.String("var-dir", "/var/lib/contentserver", "where to put my data")
var logLevelOptions = []string{
log.LOG_LEVEL_NAME_ERROR,
log.LOG_LEVEL_NAME_RECORD,
log.LOG_LEVEL_NAME_WARNING,
log.LOG_LEVEL_NAME_NOTICE,
log.LOG_LEVEL_NAME_DEBUG}
logLevelError,
logLevelRecord,
logLevelWarning,
logLevelNotice,
logLevelDebug,
}
var logLevel = flag.String(
"logLevel",
log.LOG_LEVEL_NAME_RECORD,
"log-level",
logLevelRecord,
fmt.Sprintf(
"one of %s",
strings.Join(logLevelOptions, ", ")))
@ -56,16 +53,24 @@ func main() {
return
}
if len(flag.Args()) == 1 {
fmt.Println(*protocol, *address, flag.Arg(0))
log.SetLogLevel(log.GetLogLevelByName(*logLevel))
switch *protocol {
case PROTOCOL_TCP:
server.RunSocketServer(flag.Arg(0), *address, *varDir)
break
default:
exitUsage(EXIT_CODE_INSUFFICIENT_ARGS)
fmt.Println(*address, flag.Arg(0))
level := log.LevelRecord
switch *logLevel {
case logLevelError:
level = log.LevelError
case logLevelRecord:
level = log.LevelRecord
case logLevelWarning:
level = log.LevelWarning
case logLevelNotice:
level = log.LevelNotice
case logLevelDebug:
level = log.LevelDebug
}
log.SelectedLevel = level
server.RunSocketServer(flag.Arg(0), *address, *varDir)
} else {
exitUsage(EXIT_CODE_INSUFFICIENT_ARGS)
exitUsage(1)
}
}

12
log/log.go
View File

@ -6,24 +6,24 @@ import (
"time"
)
// LogLevel logging level enum
// Level logging level enum
type Level int
const (
// LevelError an error - as bad as it gets
LevelError Level = 0
// LevelRecord put this to the logs in any case
LevelRecord = 1
LevelRecord Level = 1
// LevelWarning not that bad
LevelWarning = 2
LevelWarning Level = 2
// LevelNotice almost on debug level
LevelNotice = 3
LevelNotice Level = 3
// LevelDebug we are debugging
LevelDebug = 4
LevelDebug Level = 4
)
// SelectedLevel selected log level
var SelectedLevel Level = LevelDebug
var SelectedLevel = LevelDebug
var prefices = map[Level]string{
LevelRecord: "record : ",

44
pkg/README.md Normal file
View File

@ -0,0 +1,44 @@
Packaging & Deployment
----------------------
In order to build packages and upload to Package Cloud, please install the following requirements and run the make task.
[Package Cloud Command Line Client](https://packagecloud.io/docs#cli_install)
```
$ gem install package_cloud
```
[FPM](https://github.com/jordansissel/fpm)
```
$ gem install fpm
```
Building package
```
$ make package
```
*NOTE: you will be prompted for Package Cloud credentials.*
Testing
-------
```
$ git clone https://github.com/foomo/contentserver.git
$ cd contentserver
$ make test
```
Contributing
------------
In lieu of a formal styleguide, take care to maintain the existing coding style. Add unit tests and examples for any new or changed functionality.
1. Fork it
2. Create your feature branch (`git checkout -b my-new-feature`\)
3. Commit your changes (`git commit -am 'Add some feature'`\)
4. Push to the branch (`git push origin my-new-feature`\)
5. Create new Pull Request

0
server/repo/history.go → repo/history.go
View File

46
server/repo/loader.go → repo/loader.go
View File

@ -3,13 +3,14 @@ package repo
import (
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"net/http"
"time"
"github.com/foomo/contentserver/server/log"
"github.com/foomo/contentserver/server/repo/content"
"github.com/foomo/contentserver/server/responses"
"github.com/foomo/contentserver/server/utils"
//golog "log"
"github.com/foomo/contentserver/content"
"github.com/foomo/contentserver/log"
"github.com/foomo/contentserver/responses"
)
func (repo *Repo) updateRoutine() {
@ -31,7 +32,7 @@ func (repo *Repo) updateRoutine() {
}
func (repo *Repo) updateDimension(dimension string, node *content.RepoNode) error {
repo.updateChannel <- &RepoDimension{
repo.updateChannel <- &repoDimension{
Dimension: dimension,
Node: node,
}
@ -61,15 +62,15 @@ func (repo *Repo) _updateDimension(dimension string, newNode *content.RepoNode)
}
func builDirectory(dirNode *content.RepoNode, directory map[string]*content.RepoNode, uRIDirectory map[string]*content.RepoNode) error {
log.Debug("repo.buildDirectory: " + dirNode.Id)
existingNode, ok := directory[dirNode.Id]
log.Debug("repo.buildDirectory: " + dirNode.ID)
existingNode, ok := directory[dirNode.ID]
if ok {
return errors.New("duplicate node with id:" + existingNode.Id)
return errors.New("duplicate node with id:" + existingNode.ID)
}
directory[dirNode.Id] = dirNode
directory[dirNode.ID] = dirNode
//todo handle duplicate uris
if _, thereIsAnExistingURINode := uRIDirectory[dirNode.URI]; thereIsAnExistingURINode {
return errors.New("duplicate uri: " + dirNode.URI + " (bad node id: " + dirNode.Id + ")")
return errors.New("duplicate uri: " + dirNode.URI + " (bad node id: " + dirNode.ID + ")")
}
uRIDirectory[dirNode.URI] = dirNode
for _, childNode := range dirNode.Nodes {
@ -83,11 +84,11 @@ func builDirectory(dirNode *content.RepoNode, directory map[string]*content.Repo
func wireAliases(directory map[string]*content.RepoNode) error {
for _, repoNode := range directory {
if len(repoNode.LinkId) > 0 {
if destinationNode, ok := directory[repoNode.LinkId]; ok {
if len(repoNode.LinkID) > 0 {
if destinationNode, ok := directory[repoNode.LinkID]; ok {
repoNode.URI = destinationNode.URI
} else {
return errors.New("that link id points nowhere " + repoNode.LinkId + " from " + repoNode.Id)
return errors.New("that link id points nowhere " + repoNode.LinkID + " from " + repoNode.ID)
}
}
}
@ -148,14 +149,28 @@ func (repo *Repo) tryToRestoreCurrent() error {
return repo.loadJSONBytes(currentJSONBytes)
}
func get(URL string) (data []byte, err error) {
response, err := http.Get(URL)
if err != nil {
return data, err
}
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
return data, fmt.Errorf("Bad HTTP Response: %q", response.Status)
}
return ioutil.ReadAll(response.Body)
}
func (repo *Repo) update() (repoRuntime int64, jsonBytes []byte, err error) {
startTimeRepo := time.Now().UnixNano()
jsonBytes, err = utils.Get(repo.server)
jsonBytes, err = get(repo.server)
repoRuntime = time.Now().UnixNano() - startTimeRepo
if err != nil {
// we have no json to load - the repo server did not reply
log.Debug("we have no json to load - the repo server did not reply", err)
return repoRuntime, jsonBytes, err
}
log.Debug("loading json from: "+repo.server, string(jsonBytes))
nodes, err := loadNodesFromJSON(jsonBytes)
if err != nil {
// could not load nodes from json
@ -172,6 +187,7 @@ func (repo *Repo) update() (repoRuntime int64, jsonBytes []byte, err error) {
func (repo *Repo) loadJSONBytes(jsonBytes []byte) error {
nodes, err := loadNodesFromJSON(jsonBytes)
if err != nil {
log.Debug("could not parse json", string(jsonBytes))
return err
}
err = repo.loadNodes(nodes)

0
server/repo/mock/repo-broken-json.json → repo/mock/repo-broken-json.json
View File

0
server/repo/mock/repo-duplicate-uris.json → repo/mock/repo-duplicate-uris.json
View File

0
server/repo/mock/repo-link-broken.json → repo/mock/repo-link-broken.json
View File

0
server/repo/mock/repo-link-ok.json → repo/mock/repo-link-ok.json
View File

0
server/repo/mock/repo-ok.json → repo/mock/repo-ok.json
View File

0
server/repo/mock/repo-two-dimensions.json → repo/mock/repo-two-dimensions.json
View File

259
repo/repo.go Normal file
View File

@ -0,0 +1,259 @@
package repo
import (
"errors"
"fmt"
"strings"
"github.com/foomo/contentserver/content"
"github.com/foomo/contentserver/log"
"github.com/foomo/contentserver/requests"
)
// Dimension dimension in a repo
type Dimension struct {
Directory map[string]*content.RepoNode
URIDirectory map[string]*content.RepoNode
Node *content.RepoNode
}
// Repo content repositiory
type Repo struct {
server string
Directory map[string]*Dimension
updateChannel chan *repoDimension
updateDoneChannel chan error
history *history
}
type repoDimension struct {
Dimension string
Node *content.RepoNode
}
// NewRepo constructor
func NewRepo(server string, varDir string) *Repo {
log.Notice("creating new repo for " + server)
log.Notice(" using var dir:" + varDir)
repo := new(Repo)
repo.Directory = make(map[string]*Dimension)
repo.server = server
repo.history = newHistory(varDir)
repo.updateChannel = make(chan *repoDimension)
repo.updateDoneChannel = make(chan error)
go repo.updateRoutine()
log.Record("trying to restore pervious state")
restoreErr := repo.tryToRestoreCurrent()
if restoreErr != nil {
log.Record(" could not restore previous repo content:" + restoreErr.Error())
} else {
log.Record(" restored previous repo content")
}
return repo
}
// ResolveContent find content in a repository
func (repo *Repo) ResolveContent(dimensions []string, URI string) (resolved bool, resolvedURI string, resolvedDimension string, repoNode *content.RepoNode) {
parts := strings.Split(URI, content.PathSeparator)
resolved = false
resolvedURI = ""
resolvedDimension = ""
repoNode = nil
log.Debug("repo.ResolveContent: " + URI)
for _, dimension := range dimensions {
if d, ok := repo.Directory[dimension]; ok {
for i := len(parts); i > 0; i-- {
testURI := strings.Join(parts[0:i], content.PathSeparator)
if testURI == "" {
testURI = content.PathSeparator
}
log.Debug(" testing[" + dimension + "]: " + testURI)
if repoNode, ok := d.URIDirectory[testURI]; ok {
resolved = true
log.Debug(" found => " + testURI)
log.Debug(" destination " + fmt.Sprint(repoNode.DestinationID))
if len(repoNode.DestinationID) > 0 {
if destionationNode, destinationNodeOk := d.Directory[repoNode.DestinationID]; destinationNodeOk {
repoNode = destionationNode
}
}
return true, testURI, dimension, repoNode
}
}
}
}
return
}
// GetURIs get many uris at once
func (repo *Repo) GetURIs(dimension string, ids []string) map[string]string {
uris := make(map[string]string)
for _, id := range ids {
uris[id] = repo.getURI(dimension, id)
}
return uris
}
func (repo *Repo) getURIForNode(dimension string, repoNode *content.RepoNode) string {
if len(repoNode.LinkID) == 0 {
return repoNode.URI
}
linkedNode, ok := repo.Directory[dimension].Directory[repoNode.LinkID]
if ok {
return repo.getURIForNode(dimension, linkedNode)
}
return ""
}
func (repo *Repo) getURI(dimension string, id string) string {
repoNode, ok := repo.Directory[dimension].Directory[id]
if ok {
return repo.getURIForNode(dimension, repoNode)
}
return ""
}
func (repo *Repo) getNode(repoNode *content.RepoNode, expanded bool, mimeTypes []string, path []*content.Item, level int, groups []string, dataFields []string) *content.Node {
node := content.NewNode()
node.Item = repoNode.ToItem(dataFields)
log.Debug("repo.GetNode: " + repoNode.ID)
for _, childID := range repoNode.Index {
childNode := repoNode.Nodes[childID]
if (level == 0 || expanded || !expanded && childNode.InPath(path)) && !childNode.Hidden && childNode.CanBeAccessedByGroups(groups) && childNode.IsOneOfTheseMimeTypes(mimeTypes) {
node.Nodes[childID] = repo.getNode(childNode, expanded, mimeTypes, path, level+1, groups, dataFields)
node.Index = append(node.Index, childID)
}
}
return node
}
// GetNodes get nodes
func (repo *Repo) GetNodes(r *requests.Nodes) map[string]*content.Node {
nodes := make(map[string]*content.Node)
path := []*content.Item{}
for nodeName, nodeRequest := range r.Nodes {
log.Debug(" adding node " + nodeName + " " + nodeRequest.ID)
dimensionNode, ok := repo.Directory[nodeRequest.Dimension]
nodes[nodeName] = nil
if !ok {
log.Warning("could not get dimension root node for nodeRequest.Dimension: " + nodeRequest.Dimension)
continue
}
if treeNode, ok := dimensionNode.Directory[nodeRequest.ID]; ok {
nodes[nodeName] = repo.getNode(treeNode, nodeRequest.Expand, nodeRequest.MimeTypes, path, 0, r.Env.Groups, nodeRequest.DataFields)
} else {
log.Warning("you are requesting an invalid tree node for " + nodeName + " : " + nodeRequest.ID)
}
}
return nodes
}
func (repo *Repo) validateContentRequest(req *requests.Content) (err error) {
if req == nil {
return errors.New("request must not be nil")
}
if len(req.URI) == 0 {
return errors.New("request URI must not be empty")
}
if req.Env == nil {
return errors.New("request.Env must not be nil")
}
if len(req.Env.Dimensions) == 0 {
return errors.New("request.Env.Dimensions must not be empty")
}
for _, envDimension := range req.Env.Dimensions {
if !repo.hasDimension(envDimension) {
availableDimensions := []string{}
for availableDimension := range repo.Directory {
availableDimensions = append(availableDimensions, availableDimension)
}
return fmt.Errorf("unknown dimension %q in r.Env must be one of %q", envDimension, availableDimensions)
}
}
return nil
}
func (repo *Repo) hasDimension(d string) bool {
_, hasDimension := repo.Directory[d]
return hasDimension
}
// GetContent resolves content and fetches nodes in one call. It combines those
// two tasks for performance reasons.
//
// In the first step it uses r.URI to look up content in all given
// r.Env.Dimensions of repo.Directory.
//
// In the second step it collects the requested nodes.
//
// those two steps are independent.
func (repo *Repo) GetContent(r *requests.Content) (c *content.SiteContent, err error) {
// add more input validation
err = repo.validateContentRequest(r)
if err != nil {
log.Debug("repo.GetContent invalid request", err)
return
}
log.Debug("repo.GetContent: ", r.URI)
c = content.NewSiteContent()
resolved, resolvedURI, resolvedDimension, node := repo.ResolveContent(r.Env.Dimensions, r.URI)
if resolved {
log.Notice("200 for " + r.URI)
// forbidden ?!
c.Status = content.StatusOk
c.MimeType = node.MimeType
c.Dimension = resolvedDimension
c.URI = resolvedURI
c.Item = node.ToItem([]string{})
c.Path = node.GetPath()
c.Data = node.Data
// fetch URIs for all dimensions
uris := make(map[string]string)
for dimensionName := range repo.Directory {
uris[dimensionName] = repo.getURI(dimensionName, node.ID)
}
c.URIs = uris
} else {
log.Notice("404 for " + r.URI)
c.Status = content.StatusNotFound
c.Dimension = r.Env.Dimensions[0]
}
log.Debug(fmt.Sprintf("resolved: %v, uri: %v, dim: %v, n: %v", resolved, resolvedURI, resolvedDimension, node))
if resolved == false {
log.Debug("repo.GetContent", r.URI, "could not be resolved falling back to default dimension", r.Env.Dimensions[0])
// r.Env.Dimensions is validated => we can access it
resolvedDimension = r.Env.Dimensions[0]
}
// add navigation trees
for treeName, treeRequest := range r.Nodes {
log.Debug(" adding tree " + treeName + " " + treeRequest.ID)
rootNode, ok := repo.Directory[resolvedDimension]
if !ok {
log.Warning("could not resolve rootNode for resolved dimension " + resolvedDimension)
c.Nodes[treeName] = nil
continue
}
if treeNode, ok := rootNode.Directory[treeRequest.ID]; ok {
c.Nodes[treeName] = repo.getNode(treeNode, treeRequest.Expand, treeRequest.MimeTypes, c.Path, 0, r.Env.Groups, treeRequest.DataFields)
} else {
log.Warning("you are requesting an invalid tree node for " + treeName + " : " + treeRequest.ID)
}
}
return c, nil
}
// GetRepo get the whole repo in all dimensions
func (repo *Repo) GetRepo() map[string]*content.RepoNode {
response := make(map[string]*content.RepoNode)
for dimensionName, dimension := range repo.Directory {
response[dimensionName] = dimension.Node
}
return response
}
func uriKeyForState(state string, uri string) string {
return state + "-" + uri
}

90
server/repo/repo_test.go → repo/repo_test.go
View File

@ -10,7 +10,7 @@ import (
"testing"
"time"
"github.com/foomo/contentserver/server/requests"
"github.com/foomo/contentserver/requests"
)
func getMockData(t *testing.T) (server *httptest.Server, varDir string) {
@ -116,35 +116,29 @@ func TestDimensionHygiene(t *testing.T) {
}
}
func TestResolveContent(t *testing.T) {
func getTestRepo(path string, t *testing.T) *Repo {
mockServer, varDir := getMockData(t)
server := mockServer.URL + "/repo-two-dimensions.json"
server := mockServer.URL + path
r := NewRepo(server, varDir)
response := r.Update()
if !response.Success {
t.Fatal("well those two dimension should be fine")
}
dimensions := []string{"dimension_foo"}
contentRequest := &requests.Content{
URI: "/a",
Env: &requests.Env{
Dimensions: dimensions,
Groups: []string{},
},
Nodes: map[string]*requests.Node{
"id-root": &requests.Node{
Id: "id-root",
Dimension: dimensions[0],
MimeTypes: []string{"application/x-node"},
Expand: true,
DataFields: []string{},
},
},
}
siteContent := r.GetContent(contentRequest)
return r
}
func TestResolveContent(t *testing.T) {
r := getTestRepo("/repo-two-dimensions.json", t)
contentRequest := makeValidRequest()
siteContent, err := r.GetContent(contentRequest)
if siteContent.URI != contentRequest.URI {
t.Fatal("failed to resolve uri")
}
if err != nil {
t.Fatal(err)
}
}
func TestLinkIds(t *testing.T) {
@ -164,3 +158,57 @@ func TestLinkIds(t *testing.T) {
}
}
func makeValidRequest() *requests.Content {
dimensions := []string{"dimension_foo"}
return &requests.Content{
URI: "/a",
Env: &requests.Env{
Dimensions: dimensions,
Groups: []string{},
},
Nodes: map[string]*requests.Node{
"id-root": &requests.Node{
ID: "id-root",
Dimension: dimensions[0],
MimeTypes: []string{"application/x-node"},
Expand: true,
DataFields: []string{},
},
},
}
}
func TestInvalidRequest(t *testing.T) {
r := getTestRepo("/repo-two-dimensions.json", t)
if r.validateContentRequest(makeValidRequest()) != nil {
t.Fatal("failed validation a valid request")
}
tests := map[string]*requests.Content{}
rEmptyURI := makeValidRequest()
rEmptyURI.URI = ""
tests["empty uri"] = rEmptyURI
rEmptyEnv := makeValidRequest()
rEmptyEnv.Env = nil
tests["empty env"] = rEmptyEnv
rEmptyEnvDimensions := makeValidRequest()
rEmptyEnvDimensions.Env.Dimensions = []string{}
tests["empty env dimensions"] = rEmptyEnvDimensions
//rNodesValidID := makeValidRequest()
//rNodesValidID.Nodes["id-root"].Id = ""
//tests["nodes must have a valid id"] = rNodesValidID
for comment, req := range tests {
if r.validateContentRequest(req) == nil {
t.Fatal(comment, "should have failed")
}
}
}

4
server/requests/requests.go → requests/requests.go
View File

@ -13,7 +13,7 @@ type Env struct {
// Node - an abdtract node request, use this one to request navigations
type Node struct {
// this one should be obvious
Id string `json:"id"`
ID string `json:"id"`
// from which dimension
Dimension string `json:"dimension"`
// what do you want to see in your navigations, folders, images or unicorns
@ -49,7 +49,7 @@ type Repo struct {
// ItemMap - map of items
type ItemMap struct {
Id string `json:"id"`
ID string `json:"id"`
DataFields []string `json:"dataFields"`
}

0
server/responses/responses.go → responses/responses.go
View File

6
server/repo/content/constants.go
View File

@ -1,6 +0,0 @@
package content
const (
INDENT string = "\t"
PATH_SEPARATOR = "/"
)

26
server/repo/content/site_content.go
View File

@ -1,26 +0,0 @@
package content
const (
STATUS_OK = 200
STATUS_FORBIDDEN = 403
STATUS_NOT_FOUND = 404
)
type SiteContent struct {
Status int `json:"status"`
URI string `json:"URI"`
Dimension string `json:"dimension"`
MimeType string `json:"mimeType"`
Item *Item `json:"item"`
Data interface{} `json:"data"`
Path []*Item `json:"path"`
URIs map[string]string `json:"URIs"`
Nodes map[string]*Node `json:"nodes"`
}
func NewSiteContent() *SiteContent {
c := new(SiteContent)
c.Nodes = make(map[string]*Node)
c.URIs = make(map[string]string)
return c
}

193
server/repo/repo.go
View File

@ -1,193 +0,0 @@
package repo
import (
"fmt"
"strings"
"github.com/foomo/contentserver/server/log"
"github.com/foomo/contentserver/server/repo/content"
"github.com/foomo/contentserver/server/requests"
)
type Dimension struct {
Directory map[string]*content.RepoNode
URIDirectory map[string]*content.RepoNode
Node *content.RepoNode
}
type RepoDimension struct {
Dimension string
Node *content.RepoNode
}
type Repo struct {
server string
Directory map[string]*Dimension
updateChannel chan *RepoDimension
updateDoneChannel chan error
history *history
}
func NewRepo(server string, varDir string) *Repo {
log.Notice("creating new repo for " + server)
log.Notice(" using var dir:" + varDir)
repo := new(Repo)
repo.Directory = make(map[string]*Dimension)
repo.server = server
repo.history = newHistory(varDir)
repo.updateChannel = make(chan *RepoDimension)
repo.updateDoneChannel = make(chan error)
go repo.updateRoutine()
log.Record("trying to restore pervious state")
restoreErr := repo.tryToRestoreCurrent()
if restoreErr != nil {
log.Record(" could not restore previous repo content:" + restoreErr.Error())
} else {
log.Record(" restored previous repo content")
}
return repo
}
func (repo *Repo) ResolveContent(dimensions []string, URI string) (resolved bool, resolvedURI string, resolvedDimension string, repoNode *content.RepoNode) {
parts := strings.Split(URI, content.PATH_SEPARATOR)
resolved = false
resolvedURI = ""
resolvedDimension = ""
repoNode = nil
log.Debug("repo.ResolveContent: " + URI)
for _, dimension := range dimensions {
if d, ok := repo.Directory[dimension]; ok {
for i := len(parts); i > 0; i-- {
testURI := strings.Join(parts[0:i], content.PATH_SEPARATOR)
if testURI == "" {
testURI = content.PATH_SEPARATOR
}
log.Debug(" testing[" + dimension + "]: " + testURI)
if repoNode, ok := d.URIDirectory[testURI]; ok {
resolved = true
log.Debug(" found => " + testURI)
log.Debug(" destination " + fmt.Sprint(repoNode.DestinationId))
if len(repoNode.DestinationId) > 0 {
if destionationNode, destinationNodeOk := d.Directory[repoNode.DestinationId]; destinationNodeOk {
repoNode = destionationNode
}
}
return true, testURI, dimension, repoNode
}
}
}
}
return
}
func (repo *Repo) GetURIs(dimension string, ids []string) map[string]string {
uris := make(map[string]string)
for _, id := range ids {
uris[id] = repo.GetURI(dimension, id)
}
return uris
}
func (repo *Repo) GetURIForNode(dimension string, repoNode *content.RepoNode) string {
if len(repoNode.LinkId) == 0 {
return repoNode.URI
} else {
linkedNode, ok := repo.Directory[dimension].Directory[repoNode.LinkId]
if ok {
return repo.GetURIForNode(dimension, linkedNode)
} else {
return ""
}
}
}
func (repo *Repo) GetURI(dimension string, id string) string {
repoNode, ok := repo.Directory[dimension].Directory[id]
if ok {
return repo.GetURIForNode(dimension, repoNode)
}
return ""
}
func (repo *Repo) GetNode(repoNode *content.RepoNode, expanded bool, mimeTypes []string, path []*content.Item, level int, groups []string, dataFields []string) *content.Node {
node := content.NewNode()
node.Item = repoNode.ToItem(dataFields)
log.Debug("repo.GetNode: " + repoNode.Id)
for _, childId := range repoNode.Index {
childNode := repoNode.Nodes[childId]
if (level == 0 || expanded || !expanded && childNode.InPath(path)) && !childNode.Hidden && childNode.CanBeAccessedByGroups(groups) && childNode.IsOneOfTheseMimeTypes(mimeTypes) {
node.Nodes[childId] = repo.GetNode(childNode, expanded, mimeTypes, path, level+1, groups, dataFields)
node.Index = append(node.Index, childId)
}
}
return node
}
func (repo *Repo) GetNodes(r *requests.Nodes) map[string]*content.Node {
nodes := make(map[string]*content.Node)
path := make([]*content.Item, 0)
for nodeName, nodeRequest := range r.Nodes {
log.Debug(" adding node " + nodeName + " " + nodeRequest.Id)
if treeNode, ok := repo.Directory[nodeRequest.Dimension].Directory[nodeRequest.Id]; ok {
nodes[nodeName] = repo.GetNode(treeNode, nodeRequest.Expand, nodeRequest.MimeTypes, path, 0, r.Env.Groups, nodeRequest.DataFields)
} else {
log.Warning("you are requesting an invalid tree node for " + nodeName + " : " + nodeRequest.Id)
}
}
return nodes
}
func (repo *Repo) GetContent(r *requests.Content) *content.SiteContent {
// add more input validation
log.Debug("repo.GetContent: " + r.URI)
c := content.NewSiteContent()
resolved, resolvedURI, resolvedDimension, node := repo.ResolveContent(r.Env.Dimensions, r.URI)
if resolved {
log.Notice("200 for " + r.URI)
// forbidden ?!
c.Status = content.STATUS_OK
c.MimeType = node.MimeType
c.Dimension = resolvedDimension
c.URI = resolvedURI
c.Item = node.ToItem([]string{})
c.Path = node.GetPath()
c.Data = node.Data
// fetch URIs for all dimensions
uris := make(map[string]string)
for dimensionName, _ := range repo.Directory {
uris[dimensionName] = repo.GetURI(dimensionName, node.Id)
}
c.URIs = uris
} else {
log.Notice("404 for " + r.URI)
c.Status = content.STATUS_NOT_FOUND
c.Dimension = r.Env.Dimensions[0]
}
log.Debug(fmt.Sprintf("resolved: %v, uri: %v, dim: %v, n: %v", resolved, resolvedURI, resolvedDimension, node))
if resolved == false {
resolvedDimension = r.Env.Dimensions[0]
}
// add navigation trees
for treeName, treeRequest := range r.Nodes {
log.Debug(" adding tree " + treeName + " " + treeRequest.Id)
if treeNode, ok := repo.Directory[resolvedDimension].Directory[treeRequest.Id]; ok {
c.Nodes[treeName] = repo.GetNode(treeNode, treeRequest.Expand, treeRequest.MimeTypes, c.Path, 0, r.Env.Groups, treeRequest.DataFields)
} else {
log.Warning("you are requesting an invalid tree node for " + treeName + " : " + treeRequest.Id)
}
}
return c
}
func (repo *Repo) GetRepo() map[string]*content.RepoNode {
response := make(map[string]*content.RepoNode)
for dimensionName, dimension := range repo.Directory {
response[dimensionName] = dimension.Node
}
return response
}
func uriKeyForState(state string, uri string) string {
return state + "-" + uri
}

42
server/server.go
View File

@ -1,28 +1,28 @@
package server
import (
"github.com/foomo/contentserver/server/repo"
)
// our data
type Stats struct {
requests int64
type stats struct {
requests int64
chanCount chan int
}
func NewStats() *Stats {
stats := new(Stats)
stats.requests = 0
return stats
func newStats() *stats {
s := &stats{
requests: 0,
chanCount: make(chan int),
}
go func() {
for {
select {
case <-s.chanCount:
s.requests++
s.chanCount <- 1
}
}
}()
return s
}
var stats *Stats = NewStats()
var contentRepo *repo.Repo
func countRequest() {
stats.requests++
}
func numRequests() int64 {
return stats.requests
func (s *stats) countRequest() {
s.chanCount <- 1
<-s.chanCount
}

52
server/socket.go
View File

@ -8,12 +8,17 @@ import (
"strconv"
"strings"
"github.com/foomo/contentserver/server/log"
"github.com/foomo/contentserver/server/repo"
"github.com/foomo/contentserver/server/requests"
"github.com/foomo/contentserver/server/responses"
"github.com/foomo/contentserver/log"
"github.com/foomo/contentserver/repo"
"github.com/foomo/contentserver/requests"
"github.com/foomo/contentserver/responses"
)
type socketServer struct {
stats *stats
repo *repo.Repo
}
// there should be sth. built in ?!
// anyway this ony concatenates two "ByteArrays"
func concat(a []byte, b []byte) []byte {
@ -23,46 +28,48 @@ func concat(a []byte, b []byte) []byte {
return newslice
}
func handleSocketRequest(handler string, jsonBuffer []byte) (replyBytes []byte, err error) {
countRequest()
func (s *socketServer) handleSocketRequest(handler string, jsonBuffer []byte) (replyBytes []byte, err error) {
s.stats.countRequest()
var reply interface{}
var apiErr error
var jsonErr error
log.Record(fmt.Sprintf("socket.handleSocketRequest(%d): %s %s", numRequests(), handler, string(jsonBuffer)))
log.Record(fmt.Sprintf("socket.handleSocketRequest(%d): %s %s", s.stats.requests, handler, string(jsonBuffer)))
switch handler {
case "getURIs":
getURIRequest := &requests.URIs{}
jsonErr = json.Unmarshal(jsonBuffer, &getURIRequest)
log.Debug(" getURIRequest: " + fmt.Sprint(getURIRequest))
uris := contentRepo.GetURIs(getURIRequest.Dimension, getURIRequest.Ids)
uris := s.repo.GetURIs(getURIRequest.Dimension, getURIRequest.Ids)
log.Debug(" resolved: " + fmt.Sprint(uris))
reply = uris
break
case "content":
contentRequest := &requests.Content{}
jsonErr = json.Unmarshal(jsonBuffer, &contentRequest)
log.Debug(" contentRequest: " + fmt.Sprint(contentRequest))
content := contentRepo.GetContent(contentRequest)
log.Debug("contentRequest:", contentRequest)
content, apiErr := s.repo.GetContent(contentRequest)
log.Debug(apiErr)
reply = content
break
case "getNodes":
nodesRequest := &requests.Nodes{}
jsonErr = json.Unmarshal(jsonBuffer, &nodesRequest)
log.Debug(" nodesRequest: " + fmt.Sprint(nodesRequest))
nodesMap := contentRepo.GetNodes(nodesRequest)
nodesMap := s.repo.GetNodes(nodesRequest)
reply = nodesMap
break
case "update":
updateRequest := &requests.Update{}
jsonErr = json.Unmarshal(jsonBuffer, &updateRequest)
log.Debug(" updateRequest: " + fmt.Sprint(updateRequest))
updateResponse := contentRepo.Update()
updateResponse := s.repo.Update()
reply = updateResponse
break
case "getRepo":
repoRequest := &requests.Repo{}
jsonErr = json.Unmarshal(jsonBuffer, &repoRequest)
log.Debug(" getRepoRequest: " + fmt.Sprint(repoRequest))
repoResponse := contentRepo.GetRepo()
repoResponse := s.repo.GetRepo()
reply = repoResponse
break
default:
@ -74,6 +81,8 @@ func handleSocketRequest(handler string, jsonBuffer []byte) (replyBytes []byte,
log.Error(" could not read incoming json: " + fmt.Sprint(jsonErr))
errorResponse := responses.NewError(2, "could not read incoming json "+jsonErr.Error())
reply = errorResponse
} else if apiErr != nil {
reply = responses.NewError(3, "internal error "+apiErr.Error())
}
encodedBytes, jsonErr := json.MarshalIndent(map[string]interface{}{"reply": reply}, "", " ")
if jsonErr != nil {
@ -85,7 +94,7 @@ func handleSocketRequest(handler string, jsonBuffer []byte) (replyBytes []byte,
return replyBytes, err
}
func handleConnection(conn net.Conn) {
func (s *socketServer) handleConnection(conn net.Conn) {
log.Debug("socket.handleConnection")
var headerBuffer [1]byte
header := ""
@ -113,7 +122,7 @@ func handleConnection(conn net.Conn) {
return
}
log.Debug(" read json: " + string(jsonBuffer))
reply, handlingError := handleSocketRequest(requestHandler, jsonBuffer)
reply, handlingError := s.handleSocketRequest(requestHandler, jsonBuffer)
if handlingError != nil {
log.Error("socket.handleConnection: handlingError " + fmt.Sprint(handlingError))
if reply == nil {
@ -146,7 +155,10 @@ func handleConnection(conn net.Conn) {
// RunSocketServer - let it run and enjoy on a socket near you
func RunSocketServer(server string, address string, varDir string) {
log.Record("building repo with content from " + server)
contentRepo = repo.NewRepo(server, varDir)
s := &socketServer{
stats: newStats(),
repo: repo.NewRepo(server, varDir),
}
ln, err := net.Listen("tcp", address)
if err != nil {
// failed to create socket
@ -154,17 +166,15 @@ func RunSocketServer(server string, address string, varDir string) {
} else {
// there we go
log.Record("RunSocketServer: started to listen on " + address)
if len(contentRepo.Directory) == 0 {
contentRepo.Update()
}
s.repo.Update()
for {
conn, err := ln.Accept() // this blocks until connection or error
if err != nil {
log.Error("RunSocketServer: could not accept connection" + fmt.Sprint(err))
continue
} else {
go handleConnection(conn) // a goroutine handles conn so that the loop can accept other connections
}
// a goroutine handles conn so that the loop can accept other connections
go s.handleConnection(conn)
}
}
}

61
server/utils/utils.go
View File

@ -1,61 +0,0 @@
package utils
import (
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"net/http"
)
func JsonResponse(w http.ResponseWriter, obj interface{}) {
fmt.Fprint(w, toJson(obj))
}
func ToJSON(obj interface{}) string {
return toJson(obj)
}
func toJson(obj interface{}) string {
//b, err := json.MarshalIndent(obj, "", "\t")
b, err := json.Marshal(obj)
if err != nil {
return ""
} else {
return string(b)
}
}
func extractJsonFromRequestFileUpload(r *http.Request) []byte {
file, _, err := r.FormFile("request")
if err != nil {
fmt.Println(err, r)
}
data, err := ioutil.ReadAll(file)
if err != nil {
fmt.Println(err)
}
return data
}
func extractJsonFromRequest(r *http.Request) []byte {
bytes := []byte(r.PostFormValue("request"))
return bytes
}
func PopulateRequest(r *http.Request, obj interface{}) {
json.Unmarshal(extractJsonFromRequest(r), obj)
}
func Get(URL string) (data []byte, err error) {
response, err := http.Get(URL)
if err != nil {
return data, err
} else {
defer response.Body.Close()
if response.StatusCode != http.StatusOK {
return data, errors.New(fmt.Sprintf("Bad HTTP Response: %v", response.Status))
} else {
return ioutil.ReadAll(response.Body)
}
}
}