initial commit

.gitignore

@@ -0,0 +1,5 @@
+.*
+/.vscode
+!.git*
+!.*ignore
+/vendor

exporter/export.go

@@ -0,0 +1,30 @@
+package exporter
+
+import (
+	"errors"
+
+	"github.com/foomo/contentserver/content"
+	"github.com/foomo/neoscontentrepository/model"
+)
+
+type NodeType string
+
+type NodeExporter interface {
+	GetRepoNode(node *model.NodeData) (repoNode *content.RepoNode, export bool, err error)
+}
+
+var nodeExporterMap = map[string]NodeExporter{}
+var ErrorUnsupportedNodeType = errors.New("unsupported node type: no exporter registered for given node type")
+
+func RegisterExporter(nodeType NodeType, exporter NodeExporter) {
+	nodeExporterMap[string(nodeType)] = exporter
+}
+
+func Export(node *model.NodeData) (repoNode *content.RepoNode, ok bool, err error) {
+	nodeExporter, nodeExporterOK := nodeExporterMap[node.NodeType]
+	if !nodeExporterOK {
+		err = ErrorUnsupportedNodeType
+		return
+	}
+	return nodeExporter.GetRepoNode(node)
+}

exporter/page.go

@@ -0,0 +1,54 @@
+package exporter
+
+import (
+	"github.com/foomo/contentserver/content"
+	"github.com/foomo/neoscontentrepository/model"
+)
+
+// --------------------------------------------------------------------------------------------
+// ~ Constants
+// --------------------------------------------------------------------------------------------
+
+const NodeTypePage NodeType = "Neos.NodeTypes:Page"
+const MimeTypePage = "application/neos+page"
+
+type Page struct{}
+
+// --------------------------------------------------------------------------------------------
+// ~ Public methods
+// --------------------------------------------------------------------------------------------
+
+func (p *Page) GetRepoNode(node *model.NodeData) (repoNode *content.RepoNode, export bool, err error) {
+
+	repoNode, properties, errRepoNode := createRepoNode(node)
+	if errRepoNode != nil {
+		err = errRepoNode
+		return
+	}
+
+	// is not visible if node is marked as "hidden" or "hiddenBeforeDateTime" and "hiddenAfterDateTime" did not match the current time
+	// if(!$node->isVisible()) {
+	// 	return false;
+	// }
+	// if(!parent::map($node)) {
+	// 	return false;
+	// }
+
+	// if ('' != $prop = $node->getProperty('layout')) {
+	// 	$layout = $prop;
+	// } else if ('' != $prop = RepoNode::getParentProperty($node, 'subpageLayout')) {
+	// 	$layout = $prop;
+	// } else {
+	// 	$layout = 'default';
+	// }
+	// $this->addData('layout', $layout);
+
+	// add data: youtubeID
+	youtubeID := GetPropertyString("youtubeId", properties)
+	if youtubeID != "" {
+		repoNode.Data["youtubeId"] = youtubeID
+	}
+
+	export = true
+	return
+}

exporter/tree.go

@@ -0,0 +1,48 @@
+package exporter
+
+import (
+	"errors"
+	"sync"
+
+	"github.com/foomo/contentserver/content"
+	"github.com/foomo/neoscontentrepository/model"
+)
+
+func BuildTree(node *model.NodeData, tree map[string][]*model.NodeData) (repoNode *content.RepoNode, err error) {
+	repoNode, exportRootNodeOk, errRepoNode := Export(node)
+	if errRepoNode != nil {
+		err = errRepoNode
+		return
+	}
+	if !exportRootNodeOk {
+		err = errors.New("root node export not ok")
+		return
+	}
+
+	// childNodes, childNodesOk := tree[node.Path]
+	// if !childNodesOk {
+	// 	err = errors.New("no nodes for path found: " + node.Path)
+	// 	return
+	// }
+
+	wg := &sync.WaitGroup{}
+	lock := &sync.RWMutex{}
+	for _, childNode := range tree[node.Path] {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+			childRepoNode, errChildRepoNode := BuildTree(childNode, tree)
+			if errChildRepoNode != nil {
+				err = errChildRepoNode
+				return
+			}
+			// @todo keep sorting index order
+			lock.Lock()
+			repoNode.Index = append(repoNode.Index, childRepoNode.ID)
+			repoNode.AddNode(childRepoNode.ID, childRepoNode)
+			lock.Unlock()
+		}()
+	}
+	wg.Wait()
+	return
+}

exporter/utils.go

@@ -0,0 +1,68 @@
+package exporter
+
+import (
+	"encoding/json"
+	"time"
+
+	"github.com/foomo/contentserver/content"
+	"github.com/foomo/neoscontentrepository/model"
+)
+
+const RepoNodeDefaultLayout = "default"
+
+func createRepoNode(node *model.NodeData) (repoNode *content.RepoNode, properties map[string]interface{}, err error) {
+
+	var errProperties error
+	properties, errProperties = parseProperties(node.Properties)
+
+	repoNode = content.NewRepoNode()
+	now := time.Now()
+
+	// set base data
+	repoNode.ID = node.Identifier
+	repoNode.MimeType = MimeTypePage
+	repoNode.Data["createdAt"] = node.CreationDatetime.Unix()
+	repoNode.Data["updatedAt"] = node.LastModificationDatetime.Unix()
+
+	repoNode.Hidden = node.HiddenInIndex || node.Hidden ||
+		(node.HiddenBeforeDatetime != nil && node.HiddenBeforeDatetime.Unix() > 0 && now.Unix() < node.HiddenBeforeDatetime.Unix()) ||
+		(node.HiddenAfterDatetime != nil && node.HiddenAfterDatetime.Unix() > 0 && now.Unix() > node.HiddenAfterDatetime.Unix())
+
+	// add localized properties
+	// $this->URI = $this->getNodeUri($node);
+
+	// handle property errors
+	if errProperties != nil {
+		err = errProperties
+		return
+	}
+
+	// node name
+	repoNode.Name = GetPropertyString("title", properties)
+
+	// layout
+	repoNode.Data["layout"] = RepoNodeDefaultLayout
+	layout := GetPropertyString("layout", properties)
+	if layout != "" {
+		repoNode.Data["layout"] = layout
+	}
+
+	return
+}
+
+// GetPropertyString will return properties value for given key or an empty string
+func GetPropertyString(key string, properties map[string]interface{}) string {
+	if value, ok := properties[key]; ok {
+		return value.(string)
+	}
+	return ""
+}
+
+func parseProperties(props string) (properties map[string]interface{}, err error) {
+	errUnmarshall := json.Unmarshal([]byte(props), &properties)
+	if errUnmarshall != nil {
+		err = errUnmarshall
+		return
+	}
+	return
+}

glide.lock

@@ -0,0 +1,14 @@
+hash: 196ea20faad04f34c72ee4ec0ca7a36c2737374b60efb187cf3c673e1a2e156c
+updated: 2018-12-26T19:15:19.205432+01:00
+imports:
+- name: github.com/foomo/contentserver
+  version: 8f7c23ff4a3c4137e0f3159a2e707553f00c5448
+- name: github.com/go-sql-driver/mysql
+  version: 72cd26f257d44c1114970e19afddcd812016007e
+- name: github.com/stretchr/testify
+  version: f35b8ab0b5a2cef36673838d662e249dd9c94686
+- name: google.golang.org/appengine
+  version: e9657d882bb81064595ca3b56cbe2546bbabf7b1
+  subpackages:
+  - cloudsql
+testImports: []

glide.yaml

@@ -0,0 +1,8 @@
+package: github.com/foomo/neoscontentrepository
+import:
+- package: github.com/foomo/contentserver
+  version: ~1.4.2
+- package: github.com/go-sql-driver/mysql
+  version: ~1.4.1
+- package: github.com/stretchr/testify
+  version: ~1.2.2

model/node.go

@@ -0,0 +1,85 @@
+package model
+
+import (
+	"strings"
+	"time"
+)
+
+/*
++-------------------------------+---------------+------+-----+---------+-------+
+| Field                         | Type          | Null | Key | Default | Extra |
++-------------------------------+---------------+------+-----+---------+-------+
+| persistence_object_identifier | varchar(40)   | NO   | PRI | NULL    |       |
+| workspace                     | varchar(255)  | YES  | MUL | NULL    |       |
+| contentobjectproxy            | varchar(40)   | YES  | MUL | NULL    |       |
+| movedto                       | varchar(40)   | YES  | UNI | NULL    |       |
+| version                       | int(11)       | NO   |     | 1       |       |
+| pathhash                      | varchar(32)   | NO   | MUL | NULL    |       |
+| path                          | varchar(4000) | NO   |     | NULL    |       |
+| parentpathhash                | varchar(32)   | NO   | MUL | NULL    |       |
+| parentpath                    | varchar(4000) | NO   | MUL | NULL    |       |
+| identifier                    | varchar(255)  | NO   | MUL | NULL    |       |
+| sortingindex                  | int(11)       | YES  |     | NULL    |       |
+| removed                       | tinyint(1)    | NO   |     | NULL    |       |
+| dimensionshash                | varchar(32)   | NO   |     | NULL    |       |
+| lastmodificationdatetime      | datetime      | NO   |     | NULL    |       |
+| lastpublicationdatetime       | datetime      | YES  |     | NULL    |       |
+| hiddenbeforedatetime          | datetime      | YES  |     | NULL    |       |
+| hiddenafterdatetime           | datetime      | YES  |     | NULL    |       |
+| dimensionvalues               | longtext      | NO   |     | NULL    |       |
+| properties                    | longtext      | NO   |     | NULL    |       |
+| nodetype                      | varchar(255)  | NO   | MUL | NULL    |       |
+| creationdatetime              | datetime      | NO   |     | NULL    |       |
+| hidden                        | tinyint(1)    | NO   |     | NULL    |       |
+| hiddeninindex                 | tinyint(1)    | NO   |     | NULL    |       |
+| accessroles                   | longtext      | NO   |     | NULL    |       |
++-------------------------------+---------------+------+-----+---------+-------+
+*/
+
+type NodeData struct {
+	PersistenceObjectIdentifier string     `json:"persistence_object_identifier"`
+	Identifier                  string     `json:"identifier"`
+	Version                     uint       `json:"version"`
+	Workspace                   *string    `json:"workspace"`
+	ContentObjectProxy          *string    `json:"contentobjectproxy"`
+	MovedTo                     *string    `json:"movedto"`
+	PathHash                    string     `json:"pathhash"`
+	Path                        string     `json:"path"`
+	ParentPathHash              string     `json:"parentpathhash"`
+	ParentPath                  string     `json:"parentpath"`
+	SortingIndex                *uint      `json:"sortingindex"`
+	Removed                     bool       `json:"removed"`
+	DimensionsHash              string     `json:"dimensionshash"`
+	CreationDatetime            time.Time  `json:"creationdatetime"`
+	LastModificationDatetime    time.Time  `json:"lastmodificationdatetime"`
+	LastPublicationDatetime     *time.Time `json:"lastpublicationdatetime"`
+	HiddenBeforeDatetime        *time.Time `json:"hiddenbeforedatetime"`
+	HiddenAfterDatetime         *time.Time `json:"hiddenafterdatetime"`
+	DimensionValues             string     `json:"dimensionvalues"`
+	Properties                  string     `json:"properties"`
+	NodeType                    string     `json:"nodetype"`
+	Hidden                      bool       `json:"hidden"`
+	HiddenInIndex               bool       `json:"hiddeninindex"`
+	AccessRoles                 string     `json:"accessroles"`
+}
+
+type NodeProperties struct {
+	Title          string `json:"title,omitempty"`
+	URIPathSegment string `json:"uriPathSegment"`
+
+	Layout        string `json:"layout,omitempty"`
+	SubPageLayout string `json:"subpageLayout,omitempty"`
+}
+
+func (node *NodeData) IsVisible() bool {
+	// @todo @implementme
+	return false
+}
+
+func (node *NodeData) GetHash() string {
+	return strings.Join([]string{
+		node.Identifier,
+		*node.Workspace,
+		node.DimensionsHash,
+	}, "___")
+}

store/loader.go

@@ -0,0 +1,134 @@
+package store
+
+import (
+	"database/sql"
+	"fmt"
+	"strings"
+	"sync"
+
+	"github.com/foomo/neoscontentrepository/model"
+
+	// mysql driver import
+	_ "github.com/go-sql-driver/mysql"
+)
+
+func Load(dsn string, nodeTypes []string) (nodes map[string]*model.NodeData, tree map[string][]*model.NodeData, rootPath string, err error) {
+
+	nodesLock := &sync.RWMutex{}
+	nodes = map[string]*model.NodeData{}
+
+	// connect with DB
+	db, errDB := sql.Open("mysql", dsn+"?parseTime=true")
+	if errDB != nil {
+		err = errDB
+		return
+	}
+	defer db.Close()
+
+	nodeTypesSlice := []string{}
+	for _, nodeType := range nodeTypes {
+		nodeTypesSlice = append(nodeTypesSlice, `"`+nodeType+`"`)
+	}
+	nodeTypesString := strings.Join(nodeTypesSlice, ",")
+
+	// sqlStatement, errStatement := db.Prepare(`SELECT persistence_object_identifier, identifier, version, workspace, contentobjectproxy, movedto, pathhash, path, parentpathhash, parentpath, sortingindex, removed, dimensionshash, creationdatetime, lastmodificationdatetime, lastpublicationdatetime, hiddenbeforedatetime, hiddenafterdatetime, dimensionvalues, properties, nodetype, hidden, hiddeninindex, accessroles FROM neos_contentrepository_domain_model_nodedata WHERE nodetype IN (?` + strings.Repeat(`,?`, len(nodeTypes)-1) + `)`)
+	sqlStatement, errStatement := db.Prepare(fmt.Sprintf(`SELECT persistence_object_identifier, identifier, version, workspace, contentobjectproxy, movedto, pathhash, path, parentpathhash, parentpath, sortingindex, removed, dimensionshash, creationdatetime, lastmodificationdatetime, lastpublicationdatetime, hiddenbeforedatetime, hiddenafterdatetime, dimensionvalues, properties, nodetype, hidden, hiddeninindex, accessroles FROM neos_contentrepository_domain_model_nodedata WHERE nodetype IN (%s)`, nodeTypesString))
+	if errStatement != nil {
+		err = errStatement
+		return
+	}
+
+	// execute sql query
+	rows, errRows := sqlStatement.Query()
+	if errRows != nil {
+		err = errRows
+		return
+	}
+
+	idsLock := &sync.RWMutex{}
+	ids := map[string]bool{}
+	tree = map[string][]*model.NodeData{}
+
+	wg := &sync.WaitGroup{}
+	for rows.Next() {
+		wg.Add(1)
+
+		nodeData := &model.NodeData{}
+		// for each row, scan the result into a node data object
+		errScan := rows.Scan(
+			&nodeData.PersistenceObjectIdentifier,
+			&nodeData.Identifier,
+			&nodeData.Version,
+			&nodeData.Workspace,
+			&nodeData.ContentObjectProxy,
+			&nodeData.MovedTo,
+			&nodeData.PathHash,
+			&nodeData.Path,
+			&nodeData.ParentPathHash,
+			&nodeData.ParentPath,
+			&nodeData.SortingIndex,
+			&nodeData.Removed,
+			&nodeData.DimensionsHash,
+			&nodeData.CreationDatetime,
+			&nodeData.LastModificationDatetime,
+			&nodeData.LastPublicationDatetime,
+			&nodeData.HiddenBeforeDatetime,
+			&nodeData.HiddenAfterDatetime,
+			&nodeData.DimensionValues,
+			&nodeData.Properties,
+			&nodeData.NodeType,
+			&nodeData.Hidden,
+			&nodeData.HiddenInIndex,
+			&nodeData.AccessRoles,
+		)
+		if errScan != nil {
+			err = errScan
+			return
+		}
+
+		// find shortest path
+		if rootPath == "" || len(rootPath) > len(nodeData.Path) {
+			rootPath = nodeData.ParentPath
+		}
+
+		go func() {
+			defer wg.Done()
+
+			// skip other workspaces
+			if *nodeData.Workspace != "live" && *nodeData.Workspace != "stage" {
+				return
+			}
+
+			// skip removed nodes
+			if nodeData.Removed {
+				return
+			}
+
+			hash := nodeData.GetHash()
+
+			// collect nodes
+			nodesLock.Lock()
+			nodes[hash] = nodeData
+			if _, ok := tree[nodeData.ParentPath]; !ok {
+				tree[nodeData.ParentPath] = []*model.NodeData{}
+			}
+			tree[nodeData.ParentPath] = append(tree[nodeData.ParentPath], nodeData)
+			nodesLock.Unlock()
+
+			idsLock.RLock()
+			_, ok := ids[hash]
+			idsLock.RUnlock()
+			if !ok {
+				idsLock.Lock()
+				ids[hash] = true
+				idsLock.Unlock()
+			} else {
+				fmt.Println("duplicate ID", nodeData.GetHash())
+			}
+		}()
+	}
+
+	wg.Wait()
+
+	return
+}