feat: generic entries, new docs

This commit is contained in:
Cristian Vidmar 2024-02-01 10:39:26 +01:00
parent 94df0e4aa2
commit 45c34b7254
29 changed files with 1465 additions and 80 deletions

3
docs/_category_.json Normal file
View File

@ -0,0 +1,3 @@
{
"label": "Gocontentful"
}

391
docs/api-reference.md Normal file
View File

@ -0,0 +1,391 @@
---
sidebar_label: API Reference
sidebar_position: 4
---
# API Reference
## Client and cache
```go
NewContentfulClient(
spaceID string,
clientMode string,
clientKey string,
optimisticPageSize uint16,
logFn func(fields map[string]interface{}, level int, args ...interface{}),
logLevel int,
debug bool,
) (*ContentfulClient, error)
```
Creates a Contentful client, [read this](client/basicclientoperations) for an explanation of all parameters.
```go
SetOfflineFallback(filename string) error
```
Sets a path to a space export JSON file to be used as a fallback in case
Contentful is not reachable when you call UpdateCache() on the client. This ensures availability
but can make your content look outdated if the export file is older (and typically it is).
```go
NewOfflineContentfulClient(
file []byte,
logFn func(fields map[string]interface{}, level int, args ...interface{}),
logLevel int,
cacheAssets bool,
textJanitor bool,
) (*ContentfulClient, error)
```
Creates an offline Contentful client that loads space data from a JSON file containing a space export.
```go
(cc *ContentfulClient) SetEnvironment(environment string)
```
Sets the Contentful client's environment. All subsequent API calls will be directed to that environment in the selected
space. Pass an empty string to reset to the _master_ environment.
```go
(cc *ContentfulClient) CacheHasContentType(contentTypeID string) bool
```
Returns true if the specified contentTypeID is cached by the client, false otherwise.
```go
(cc *ContentfulClient) SetCacheUpdateTimeout(seconds int64)
```
Sets the cache update timeout to the specified length. A new client by default times out
caching in 120 seconds. A timeout is used to prevent deadlocks when a service panics and recovers
while the gocontentful goroutines are running and the main caching job is waiting for all
them to finish.
```go
(cc *ContentfulClient) SetSyncMode(mode bool) error
```
Switches on/off the cache sync mode. This method will return an error if called on an offline client.
```go
(cc *ContentfulClient) ResetSync()
```
Resets the sync token: the next call to UpdateCache() will rebuild the cache from scratch.
```go
(cc *ContentfulClient) UpdateCache(ctx context.Context, contentTypes []string, cacheAssets bool) error
```
Builds or re-builds the entire client cache.
```go
(cc *ContentfulClient) UpdateCacheForEntity(ctx context.Context, sysType string, contentType string, entityID string) error
```
Updates a single entry or asset (the sysType can take const sysTypeEntry or sysTypeAsset values) in the cache.
## Content functions and methods
_For these we're assuming a content type named "Person"._
```go
NewCfPerson(contentfulClient ...*ContentfulClient) (cfPerson *CfPerson)
```
Creates a new Person entry. You can manipulate and upsert this later. The contentfulClient parameter is optional but you
might want to pass it most of the times or you won't be able to save the entry.
```go
(cc *ContentfulClient) GetAllPerson() (voMap map[string]*CfPerson, err error)
```
Retrieves all Person entries from the client and returnes a map where the key is the ID of the entry and the value is
the Go value object for that entry.
```go
(cc *ContentfulClient) GetFilteredPerson(query *contentful.Query) (voMap map[string]*CfPerson, err error)
```
Retrieves Person entries matching the specified query.
```go
(cc *ContentfulClient) GetPersonByID(id string, forceNoCache ...bool) (vo *CfPerson, err error)
```
Retrieves the Person entry with the specified ID. The optional _forceNoCache_ parameter, if true,
makes the function bypass the existing cache and load a fresh copy of the entry from Contentful.
```go
(ref ContentfulReferencedEntry) ContentType() (contentType string)
```
Returns the Sys.ID of the content type of the referenced entry
```go
(cc *ContentfulClient) GetContentTypeOfID(ID string) (contentType string)
```
Returns the Contentful content type of an entry ID.
```go
(vo *CfPerson) ToReference() (refSys ContentTypeSys)
```
Converts a value object into a reference that can be added to a reference field of an entry. Note that functions that
retrieve referenced entries return a more flexible and useful _[]\*EntryReference_ (see Quickstart above) but to store
a reference you need a ContentTypeSys.
```go
(vo *CfPerson) GetParents() (parents []EntryReference, err error)
(ref *EntryReference) GetParents(cc *ContentfulClient) (parents []EntryReference, err error)
```
Return a slice of EntryReference objects that represent entries that reference the value object or the entry reference.
Note that in case of parents of an entry reference you need to pass a pointer to a ContentfulClient because
EntryReference objects are generic and can't carry any.
```go
(vo *CfPerson) GetPublishingStatus() string
```
Returns the publishing status of the entry as per the Contentful editor UI.
Value returned is one of the following:
```go
const (
StatusDraft = "draft"
StatusChanged = "changed"
StatusPublished = "published"
)
```
## Entry field getters and setters
Field getters are named after the field ID in Contentful and return the proper type. For example, if the Person content
type has a Symbol (short text) field named 'Name', this will be the getter:
```go
(vo *CfPerson) Name(locale ...string) (string)
```
The locale parameter is optional and if not passed, the function will return the value for the default locale of the
space. If the locale is specified and it's not available for the space, an error is returned. If the locale is valid
but a value doesn't exist for the field and locale, the function will return the value for the default locale if that's
specified as a fallback locale in the space definition in Contentful, otherwise will return an error.
Possible return types are:
- _string_ for fields of types Symbol, Text, Date
- _[]string_ for fields of type List
- _float64_ for fields of type Integer or Number
- _bool_ for fields of type Boolean
- _\*ContentTypeSys_ for single reference fields
- _[]\*ContentTypeSys_ for multiple reference fields
- _\*ContentTypeFieldLocation_ for fields of type Location
- \*interface{} for fields of type Object or RichText
If logLevel is set to LogDebug retrieving the value of a field that is not set and so not available in the API response
even as a fallback to the default locale will log the event. This can become incredibly verbose, use with care.
Field setters are named after the field ID in Contentful and require to pass in the proper type. See FIELD GETTERS above
for a reference. Example:
```go
(vo *CfPerson) SetName(title string, locale ...string) (err error)
```
## Entry write ops (only available for _ClientModeCMA_)
```go
(vo *CfPerson) UpsertEntry(cc *ContentfulClient) (err error)
```
Upserts the entry. This will appear as "Draft" (if it's a new entry) or "Changed" if it's already existing. In the
latter case, you will need to retrieve the entry with one of the Manage\* functions above to acquire the Sys object
that contains the version information. Otherwise the API call will fail with a "Version mismatch" error.
```go
(vo *CfPerson) PublishEntry(cc *ContentfulClient) (err error)
```
Publishes the entry. Note that before publishing you will need to retrieve the entry with one of the Manage\* functions
above to acquire the Sys object that contains the version information. Otherwise the API call will fail with a "Version
mismatch" error. This is needed even if you have just upserted the entry with the function above!
```go
(vo *CfPerson) UnpublishEntry(cc *ContentfulClient) (err error)
```
Unpublishes the entry. Note that before unpublishing you will need to retrieve the entry with one of the Manage\*
functions above to acquire the Sys object that contains the version information. Otherwise the API call will fail with
a "Version mismatch" error. This is needed even if you have just upserted the entry with the function above!
```go
(vo *CfPerson) UpdateEntry(cc *ContentfulClient) (err error)
```
Shortcut function that upserts and publishes the entry. Note that before calling this you will need to retrieve the
entry with one of the Manage\* functions above to acquire the Sys object that contains the version information. Otherwise
the API call will fail with a "Version mismatch" error. Using this shortcut function avoids retrieving the entry twice.
```go
(vo *CfPerson) DeleteEntry(cc *ContentfulClient) (err error)
```
Unpublishes and deletes the entry
### Generic entries
Generic entries have raw fields in this form:
```go
type RawFields map[string]interface{}
type GenericEntry struct {
Sys ContentfulSys `json:"sys,omitempty"`
RawFields RawFields `json:"fields,omitempty"`
CC *ContentfulClient `json:"-"`
}
```
While these seem to defeat the purpose of the idiomatic API, they are useful in cases where you need to
pass-through entries from Contentful to any recipient without type switching. Each generic entry carries
a reference to the Gocontentful client it was used to retrieve it, so that other operations can benefit from it.
For example, get the corresponding idiomatic entry only when needed for processing.
```go
(cc *ContentfulClient) GetGenericEntry(entryID string) (*GenericEntry, error)
```
Retrieves a generic entry by ID
### Asset functions
```go
(cc *ContentfulClient) DeleteAsset(asset *contentful.Asset) error
```
Deletes an asset from the space (only available in CMA)
```go
(cc *ContentfulClient) DeleteAssetFromCache(key string) error {
```
Deletes an asset from the client's cache
```go
(cc *ContentfulClient) GetAllAssets() (map[string]*contentful.Asset, error)
```
Retrieve all assets from a space
```go
(cc *ContentfulClient) GetAssetByID(id string, forceNoCache ...bool) (*contentful.Asset, error)
```
Retrieve an asset from a space by its ID. The optional _forceNoCache_ parameter, if true,
makes the function bypass the existing cache and load a fresh copy of the asset from Contentful.
```go
NewAssetFromURL(id string, uploadUrl string, imageFileType string, title string, locale ...string) *contentful.Asset
```
Creates an Asset from an URL of an existing file online (you still need to upsert it later).
```go
ToAssetReference(asset *contentful.Asset) (refSys ContentTypeSys)
```
Converts the asset to a reference. You need to do this before you add the asset to a reference field of an entry.
```go
(cc *ContentfulClient) DeleteAsset(asset *contentful.Asset) error
```
Deletes an asset from a space by its ID (only available for _ClientModeCMA_)
### Other helper functions and methods
```go
(cc *ContentfulClient) BrokenReferences() (brokenReferences []BrokenReference)
```
Returns a slice of BrokenReference objects with details of where entries have been
referenced but they are not found in the cache. This might naturally return false
positives for content types that are in the space but not included in the cache.
```go
FieldToObject(jsonField interface{}, targetObject interface{}) error
```
Converts a JSON field into an object. Make sure you pass a pointer to an object which type has JSON definition for all
fields you want to retrieve.
```go
HtmlToRichText(htmlSrc string) *RichTextNode
```
Converts an HTML fragment to a RichTextNode. This is useful to migrate data from third-party systems to Contentful or
support HTML paste operations in Web applications. It currently supports headings, paragraphs, hyperlinks, italic and
bold tags, horizontal rules, blockquote, ordered and unordered lists, code. Unknown tags are stripped. This function
doesn't return any error as it converts the input text into something as good as possible, without validation.
```go
RichTextToHtml(rt interface{}, linkResolver LinkResolverFunc, entryLinkResolver EntryLinkResolverFunc, imageResolver ImageResolverFunc, locale Locale) (string, error) {
```
Converts an interface representing a Contentful RichText value (usually from a field getter) into HTML.
The function takes in three (optional) functions as parameters to resolve
hyperlink URLs, permalinks to entries and to derive IMG tag attributes for embedded image assets. The three functions
return a map of attributes for the HTML tag the RichTextToHtml function will emit (either an A or an IMG) and have the
following signature. Note that the ImageResolverFunc function must return a customHTML value that can be empty but if
set it will substitute the IMG tag with the returned HTML snippet. This allows you to emit custom mark-up for your
images, e.g. a PICTURE tag.
```go
type LinkResolverFunc func(url string) (resolvedAttrs map[string]string, resolveError error)
type EntryLinkResolverFunc func(entryID string, locale Locale) (resolvedAttrs map[string]string, resolveError error)
type ImageResolverFunc func(assetID string, locale Locale) (attrs map[string]string, customHTML string, resolveError error)
type EmbeddedEntryResolverFunc func(entryID string, locale Locale) (html string, resolveError error)
```
All the three functions above can be passed as nil with different levels of graceful degrading.
### Constants and global variables
Each generated content type library file exports a constant with the Contentful ID of the content type itself, for
example in _contentful_vo_lib_person.go_:
```go
const ContentTypePerson = "person"
```
Constants are available for each locale supported by the space at the time of code generation, e.g.:
```go
const SpaceLocaleGerman Locale = "de"
const SpaceLocaleFrench Locale = "fr"
const defaultLocale Locale = SpaceLocaleGerman
```
Four levels of logging are supported (even if only partially used at this time):
```go
const (
LogDebug = 0
LogInfo = 1
LogWarn = 2
LogError = 3
)
```
A global variable named _SpaceContentTypeInfoMap_ contains an ID-indexed map of all content types
with their names and descriptions

104
docs/caching.md Normal file
View File

@ -0,0 +1,104 @@
---
sidebar_label: Caching
sidebar_position: 3
---
# Caching
Caching is a fundamental part of working with remote data across the Internet,
where access is severely impacted by latency and transfer time. In real-world scenarios,
you'll always need to keep all the data you need close and sync the changes with the remote
CMS when they happen.
Gocontentful supports caching out of the box. The client can maintain a cache of an entire space
or a subset of the content types that can be initialized with a single method call:
```go
contentTypes := []string{"person", "pet"}
err = cc.UpdateCache(context, contentTypes, true)
```
This makes sense for client modes `ClientModeCDA` and `ClientModeCPA` and not for the management API.
The client will download all the entries, convert and store them in the case as
native Go value objects. This makes subsequent accesses to the space data an in-memory operation removing all the HTTP
overhead.
The first parameter is the context. If you don't use a context in your application or service just pass _context.Background()_
The third parameter of UpdateCache toggles asset caching on or off. If you deal with assets you want this to be always on.
## Full cache init and rebuild
By default the client will cache the whole space using 4 parallel workers to speed up the process.
This is safe since Contentful allows up to 5 concurrent connections.
If you have content types that have a lot of entries, it might make sense to keep them close to each other
in the content types slice passed to UpdateCache(), so that they will run in parallel and not one after the other.
All gocontentful functions that query the space cache-transparent: if a cache is available data will be loaded from
there, otherwise it will be sourced from Contentful. This doesn't apply to _GetFilteredXYZ()_ calls that
always need to pass the query to Contentful.
Gocontentful also supports selective entry and asset cache updates through the following method:
```go
err = cc.UpdateCacheForEntity(context, sysType, contentType, entityID string)
```
When something changes in the space at Contentful you need to update the cache. For this to happen you need to set
up a webhook at Contentful and handle its calls in your service through a public HTTP listener.
When a webhook call gets in, you have the choice of updating your cache in different ways:
- You can regenerate the entire CDA cache when something is published because you want production data to
be 100% up to date in your application. This can get slow and expensive.
- You can alternatively update a single entry in the cache. This is usually the case for the CPA cache because
it's a lot faster and that works well for preview features.
- You can use the Sync API, but only limited to `ClientModeCDA`, as explained in the following paragraph.
In any case, if an update fails the previous cache is preserved to prevent service disruption.
In the unfortunate case a service or application needs to start and Contentful is not available, Gocontentful can work
in an offline mode if you call _SetOfflineFallback_ on the client after you create it passing the path to a space export file.
The gocontentful API can work entirely offline too. In this case a cache is created from a space export file and most of the
features are available (pretty obviously, those that don't require live access to the space, like custom queries). If you update
the export file periodically you can even update the cache from the updated file.
## Sync API support
In versions v1.0.12 and newer, gocontentful supports the Contentful Sync API and that's now the recommended way to cache spaces and manage updates.
Sync is enabled by default when you create a client with CDA mode.
To enable or disable support for the Sync API explicitly, you can call the SetSyncMode method on the client:
```go
cc.SetSyncMode(true)
```
With sync on, the cache updates will happen transparently through downloads of incremental changes.
The syntax to update the cache doesn't change, just call _UpdateCache_ on the client as usual.
The initialization of the cache will be slower when _SyncMode_ is on compared to the legacy full cache init because sync calls cannot be parallelized.
Subsequent updates though will be much faster because only changes in the space from the previous sync will be downloaded.
This includes entries and assets that were deleted. In case of need you can call _ResetSync()_ to start over from a fresh empty cache.
Note that the Sync API is not officially supported by Contentful on the Preview API. At the time of this writing it seems to work but use it at your own risk.
## Cache timeout
Cache update operations time out by default after 120 seconds. This makes sure that no
routine is left hanging, blocking subsequent updates in case the main application or service
recovers from a panic. If you need to increase this limit because you have a huge space with
a lot of entries you can use the _SetCacheUpdateTimeout_ method. See the [API Reference](./api-reference) for details.
## Asset caching
If you use assets in your space, then you absolutely need to enable them in the _UpdateCache_ call.
Otherwise, every time an entry needs to resolve a reference to an asset that single asset will be downloaded
and that for large spaces with thousands of assets can lead to incredibly slow operation.
## When to use and not use caching
Simple answer is: you should almost always use caching. The only scenario where not using
a cache on the client is better is when you only need to download a very limited amount
of entries (in the order of less than some hundreds) and do that at significant distance in time
(e.g. every hour). In this case your application code can be simpler and there won't be any
performance penalty. The other case is when you need to run a lot of custom queries or
use XPath, which is currently not supported by gocontentful directly.

View File

@ -0,0 +1,4 @@
{
"label": "The client at work",
"position": 2
}

33
docs/client/assets.md Normal file
View File

@ -0,0 +1,33 @@
---
sidebar_label: Assets
sidebar_position: 3
---
# Assets
Contentful allows upload and reference of binary assets and gocontentful fully supports them.
Assuming the dog entry references a picture in a field you can get it with:
```go
picture := dog.Picture() // you can pass a locale to this function as usual
```
This returns a \*contenful.AssetNoLocale object handling localization for you in two ways.
First, the field itself could be localized in the model, referencing two different assets altogether.
Secondly, the asset itself can have different files uploaded for different locales.
No matter what, the gocontentful API will return the right file:
```go
// Get the asset's URL at Contentful's CDN
if picture != nil && picture.Fields != nil && picture.Fields.File != nil {
theURL := picture.Fields.File.URL
// ...then do something with it
}
```
There are various functions and methods to work with assets, for example to create an asset
starting from an URL or to convert an asset to a reference to store it in a parent entry
field. See the [API Reference](./api-reference) chapter for details.
Note: there is no function to create a new asset in the generated code because the type `AssetNoLocale`
is from the _github.com/foomo/contentful_ package, just instantiate one if you need a blank asset.

View File

@ -0,0 +1,172 @@
---
sidebar_label: Basic client operations
sidebar_position: 1
---
# Basic client operations
Let's consider a very simple use case. You have a Contentful space where you store information
about people and their pets.
To generate a go package to manipulate those entries, run the following in your terminal (refer to the [Setup section](setup) for details):
```shell
$ gocontentful -spaceid YOUR_SPACE_ID -cmakey YOUR_CMA_API_TOKEN -contenttypes person,pet path/to/your/go/project/folder/people
```
The **-contenttypes** parameter is optional. If not specified, gocontentful will generate an API that supports all the content types of the space.
Gocontentful will scan the space, download locales and content types and generate the Go API files in the target path:
```shell
path/to/your/go/project/folder/people
|-gocontentfulvobase.go
|-gocontentfulvolib_person.go // One file for each content type
|-gocontentfulvolib_pet.go // One file for each content type
|-gocontentfulvolib.go
|-gocontentfulvo.go
```
[We recommend](setup) not passing the _-cmakey_ parameter but rather log in first using the Contentful CLI.
This will be remembered in all subsequent runs. See the [Setup section]
Note: Never modify the generated files. If you change the content model in Contentful, run gocontentful
again. This will update the files for you.
### Get a client
The generated files will be in the "people" subdirectory of your project. Your go program can get a Contentful
client from them:
```go
cc, err := people.NewContentfulClient(YOUR_SPACE_ID, people.ClientModeCDA, YOUR_API_KEY, 1000, contentfulLogger, people.LogDebug,false)
```
The parameters to pass to NewContentfulClient are:
- _spaceID_ (string)
- _clientMode_ (string) supports the constants ClientModeCDA, ClientModeCPA and ClientModeCMA. If you need to operate
on multiple APIs (e.g. one for reading and CMA for writing) you need to get two clients
- _clientKey_ (string) is your API key (generate one for your API at Contentful)
- _optimisticPageSize_ (uint16) is the page size the client will use to download entries from the space for caching.
Contentful's default is 100 but you can specify up to 1000: this might get you into an error because Contentful
limits the payload response size to 70 KB but the client will handle the error and reduce the page size automatically
until it finds a proper value. Hint: using a big page size that always fails is a waste of time and resources because
a lot of initial calls will fail, whereas a too small one will not leverage the full download bandwidth. It's a
trial-and-error and you need to find the best value for your case. For simple content types you can start with 1000,
for very complex ones that include fat fields you might want to get down to 100 or even less.
- _logFn_ is a func(fields map[string]interface{}, level int, args ...interface{}) that the client will call whenever
it needs to log something. It can be nil if you don't need logging and that will be handled gracefully but it's not
recommended. A simple function you can pass that uses the https://github.com/Sirupsen/logrus package might look
something like this:
```go
contentfulLogger := func(fields map[string]interface{}, level int, args ...interface{}) {
switch level {
case people.LogDebug:
log.WithFields(fields).Debug(args)
case people.LogInfo:
log.WithFields(fields).Info(args)
case people.LogWarn:
log.WithFields(fields).Warn(args)
case people.LogError:
log.WithFields(fields).Error(args)
default:
return
}
}
```
- _logLevel_ (int) is the debug level (see function above). Please note that LogDebug is very verbose and even logs
when you request a field value but that is not set for the entry.
- _debug_ (bool) is the Contentful API client debug switch. If set to _true_ it will log on stdout all the CURL calls
to Contentful. This is extremely verbose and extremely valuable when something fails in a call to the API because
it's the only way to see the REST API response.
_NOTE:_ Gocontentful provides an offline version of the client that can load data from a JSON space export file
(as exported by the _contentful_ CLI tool). This is the way you can write unit tests against your generated API that
don't require to be online and the management of a safe API key storage. See the [API Reference](./api-reference)
### Environments support
Gocontentful supports Contentful environments in two ways:
- Code can be generated loading the content model from an environment other than master.
This is done passing the -environment flag on the command line to specify the environment you want to load the model from.
- The gocontentful client in your application can be switched to any environment with the SetEnvironment method.
For example, if your space has an extra environment named "devplayground" you can switch the API to use it with:
```go
cc.SetEnvironment("devplayground")
```
To reset the environment to master pass an empty string.
### Working with RichText
Contentful supports Rich Text fields. Behind the scenes, these are JSON objects that represent
the content through a Contentful-specific data model. Sooner or later you might want to convert such values to and from HTML.
Gocontentful supports the conversion both ways. For instance, you want a person's resume to be converted to HTML:
```go
htmlText := people.RichTextToHtml(person.Resume(), linkResolver, entryLinkResolver, imageResolver, embeddedEntryResolver locale)
```
The parameters linkResolver, entryLinkResolver, embeddedEntryResolver and imageResolver are all functions that you can pass
to convert various elements inserted by the user into the RichText field:
- linkResolver will allow you to create custom HTML tags for hyperlinks. If left blank, RichTextToHtml will just output an A tag.
- entryLinkResolver is used to create hyperlinks with custom URLs when the destination in Contentful is another entry.
If you allow such links to be created in the editor then you must pass this function.
- imageResolver and embebbedEntryResolver are needed when the field accepts assets and entries embedded into the content, to turn
these into actual HTML snippets
The conversion works the other way around too, when you need to source data from outside and create Contentful entries:
```go
myRichText := HtmlToRichText(htmlSrc)
```
See the [API Reference](./api-reference) for more details about these functions.
### More on references
When working with references it's often useful to know if there are any broken ones in the space.
This happens when a published entry references another that has been deleted after the parent
was published. This might create issues if your application code doesn't degrade content gracefully.
To get a report of all broken references you can use the following function:
```go
(cc *ContentfulClient) BrokenReferences() (brokenReferences []BrokenReference)
```
Note that this only works with cached clients. See [the next chapter on caching](./caching).
Also on references: when you want to reference entry B from entry A, you cannot assign
the value object of entry B to the reference field in A. First you need to convert the
object to a `ContentTypeSys` object because that's what Contentful expects in reference fields:
```go
(vo *CfPerson) ToReference() (refSys ContentTypeSys)
```
Finally, you can get the parents (AKA referring) entries of either an entry or
an EntryReference with the _GetParents()_ method. This returns a slice of `[]EntryReference`:
```go
(vo *CfPerson) GetParents() (parents []EntryReference, err error)
(ref *EntryReference) GetParents(cc *ContentfulClient) (parents []EntryReference, err error)
```
### Other useful functions
Another thing you might want to know is the content type of an entry with a given ID:
```go
(cc *ContentfulClient) GetContentTypeOfID(ID string) (contentType string)
```
### Caveats and limitations
- Avoid creating content types that have field IDs equal to reserved Go words (e.g. "type").
Gocontentful won't scan for them and the generated code will break.

76
docs/client/entries.md Normal file
View File

@ -0,0 +1,76 @@
---
sidebar_label: Entries
sidebar_position: 2
---
# Working with entries
Refer to the [Getting started section](../gettingstarted) for an introduction on entry operations.
With your newly created client you can do things like:
```go
// Load all persons
persons, err := cc.GetAllPerson()
// Load a specific person
person, err := cc.GetPersonByID(THE_PERSON_ID)
// or pass a query
person, err := GetFilteredPerson(&contentful.Query{
"contentType":"person",
"exists": []string{"fields.resume"}
})
// The person's name
name := person.Name()
// The work title in a different localization. Available locales are generated as constants.
// If a space is configured to have a fallback from one locale to the default one,
// the getter functions will return that if the value is not set for locale passed to the function.
name := person.Title(people.SpaceLocaleItalian)
// Get references to the person's pets
petRefs := person.Pets()
// Deal with pets
for _, pet := range petRefs {
switch pet.ContentType {
case people.ContentTypeDog: // you have these constants in the generated code
dog := pet.VO.(*people.Dog)
// do something with dog
case people.ContentTypeCat:
// ...
}
```
Once you have loaded an entry, you can use any of the setter methods to alter the fields. For example:
```go
dog.SetAge(7)
```
This will only affect the Go object and doesn't automatically propagate to the space.
To save the entry to Contentful you need to explicitly call one of these methods:
```go
// Upsert (save) an entry
err := dog.UpsertEntry()
// Publish it (after it's been upserted)
err := dog.PublishEntry() // change your mind with err := dog.UnpublishEntry()
// Or do it in one step
err := dog.UpdateEntry() // upserts and publishes
// And delete it
err := dog.DeleteEntry()
```
If you want to know the publication status of an entry as represented in Contentful's UI you
can use the `GetPublishingStatus()` method on the entry itself. Possible return values are the
predefined constants `StatusDraft`, `StatusChanged` and `StatusPublished`.
When saving, publishing or deleting entries:
- You need a client that uses mode _ClientModeCMA_. Entries retrieved with ClientModeCDA
or ClientModeCPA can be saved in memory (for example if you need to enrich the built-in cache) but not persisted to
Contentful.
- Make sure you Get a fresh copy of the entry right before you manipulate it and upsert it / publish it to Contentful. In case it's
saved by someone else in the meantime, the upsert will fail with a version mismatch error.
In case you need a completely new entry just create it, Contentful will fill in the technical details for you:
```go
NewCfPerson(contentfulClient ...*ContentfulClient) (cfPerson *CfPerson)
```

8
docs/client/index.md Normal file
View File

@ -0,0 +1,8 @@
---
sidebar_label: The Client At Work
sidebar_position: 0
---
# The Client At Work
This section explains how to work with the Gocontentful client to create, retrieve, update and delete entities, such as entries and assets. It also walks you through working with references to handle a graph-like structure in a Contentful space. Finally, it dives into some more specific functionalities, like converting to/from RichText and HTML.

View File

@ -0,0 +1,17 @@
---
sidebar_label: Other functions
sidebar_position: 6
---
# Other useful functions
Another thing you might want to know is the content type of an entry with a given ID:
```go
(cc *ContentfulClient) GetContentTypeOfID(ID string) (contentType string)
```
## Caveats and limitations
- Avoid creating content types that have field IDs equal to reserved Go words (e.g. "type").
Gocontentful won't scan for them and the generated code will break.

33
docs/client/references.md Normal file
View File

@ -0,0 +1,33 @@
---
sidebar_label: References
sidebar_position: 5
---
# More on references
When working with references it's often useful to know if there are any broken ones in the space.
This happens when a published entry references another that has been deleted after the parent
was published. This might create issues if your application code doesn't degrade content gracefully.
To get a report of all broken references you can use the following function:
```go
(cc *ContentfulClient) BrokenReferences() (brokenReferences []BrokenReference)
```
Note that this only works with cached clients. See [the next chapter on caching](./caching).
Also on references: when you want to reference entry B from entry A, you cannot assign
the value object of entry B to the reference field in A. First you need to convert the
object to a `ContentTypeSys` object because that's what Contentful expects in reference fields:
```go
(vo *CfPerson) ToReference() (refSys ContentTypeSys)
```
Finally, you can get the parents (AKA referring) entries of either an entry or
an EntryReference with the _GetParents()_ method. This returns a slice of `[]EntryReference`:
```go
(vo *CfPerson) GetParents() (parents []EntryReference, err error)
(ref *EntryReference) GetParents(cc *ContentfulClient) (parents []EntryReference, err error)
```

31
docs/client/richtext.md Normal file
View File

@ -0,0 +1,31 @@
---
sidebar_label: Richtext
sidebar_position: 4
---
# RichText support
Contentful supports Rich Text fields. Behind the scenes, these are JSON objects that represent
the content through a Contentful-specific data model. Sooner or later you might want to convert such values to and from HTML.
Gocontentful supports the conversion both ways. For instance, you want a person's resume to be converted to HTML:
```go
htmlText := people.RichTextToHtml(person.Resume(), linkResolver, entryLinkResolver, imageResolver, embeddedEntryResolver locale)
```
The parameters linkResolver, entryLinkResolver, embeddedEntryResolver and imageResolver are all functions that you can pass
to convert various elements inserted by the user into the RichText field:
- linkResolver will allow you to create custom HTML tags for hyperlinks. If left blank, RichTextToHtml will just output an A tag.
- entryLinkResolver is used to create hyperlinks with custom URLs when the destination in Contentful is another entry.
If you allow such links to be created in the editor then you must pass this function.
- imageResolver and embebbedEntryResolver are needed when the field accepts assets and entries embedded into the content, to turn
these into actual HTML snippets
The conversion works the other way around too, when you need to source data from outside and create Contentful entries:
```go
myRichText := HtmlToRichText(htmlSrc)
```
See the [API Reference](./api-reference) for more details about these functions.

149
docs/gettingstarted.md Normal file
View File

@ -0,0 +1,149 @@
---
sidebar_label: Getting started
sidebar_position: 0
---
# Getting Started
Before you install Gocontentful as a command-line tool to use it in your projects, we suggest you get a taste of how it works by playing with the test API from the Gocontentful repository. This doesn't yet require you to have access to Contentful.
## How to play with the test API
Clone the gocontentful repository from [https://github.com/foomo/gocontentful](https://github.com/foomo/gocontentful) and open it
in your IDE.
The repository includes an offline representation of a Contentful space that can is used for testing gocontentful
without depending on an online connection and an existing Contentful space.
Create a file in the repository home directory and name it `untracked_test.go`. This ensures it's not tracked by git.
Paste the following into the file:
```go
package main
import (
"testing"
"github.com/foomo/gocontentful/test"
"github.com/foomo/gocontentful/test/testapi"
"github.com/sirupsen/logrus"
"github.com/stretchr/testify/require"
)
func TestTheAPI(t *testing.T) {
testLogger := logrus.StandardLogger()
cc, errClient := testapi.NewOfflineContentfulClient("./test/test-space-export.json",
test.GetContenfulLogger(testLogger),
test.LogDebug,
true)
require.NoError(t, errClient)
prods, errProds := cc.GetAllProduct()
require.NoError(t, errProds)
testLogger.WithField("prods", len(prods)).Info("Loaded products")
}
```
The first two lines in the unit test create a logger and an offline gocontentful client. This also
caches the content of the space in memory and lets you play with the API. The space includes three
content types (`brand`, `product` and `category`) and their relative entries.
A product has a reference to a brand and to one or more categories. If you open the `./test/test-space-export.json` file
you can see how the JSON representation of those entries is.
Getting all the products using the Contentful
Content Delivery API would normally require dealing with the connection, query and JSON payload, having
value object defined for all content types and functions to convert from/to those structs. With the Go API generated
by gocontentful all you need to do to load all the products is one single line:
```go
prods, errProds := cc.GetAllProduct()
```
Open a terminal and from the repository home directory run the test. Your output should looks similar to this:
```shell
$ go test -run TestTheAPI
INFO[0000] loading space from local file assets=12 entries=9
INFO[0000] contentful cache update queued task=UpdateCache
INFO[0000] contentful cache worker starting task=UpdateCache
INFO[0000] gonna use a local file task=UpdateCache
INFO[0000] cached all entries of content type contentType=product method=updateCacheForContentType size=4
INFO[0000] cached all assets contentType=asset method=updateCacheForContentType size=12
INFO[0000] cached all entries of content type contentType=brand method=updateCacheForContentType size=3
INFO[0000] cached all entries of content type contentType=category method=updateCacheForContentType size=2
INFO[0000] space caching done, time recorded task=UpdateCache time elapsed=179.357792ms
INFO[0000] contentful cache update returning task=UpdateCache
INFO[0000] contentful cache update returning task=UpdateCache
INFO[0000] Loaded products prods=4
PASS
ok github.com/foomo/gocontentful 0.484s
```
The last line shows that we loaded 4 products. Let's go ahead and play with the API.
We'll load a specific product and log its name. Add this at the end of the unit test:
```go
prod, errProd := cc.GetProductByID("6dbjWqNd9SqccegcqYq224")
require.NoError(t, errProd)
prodName := prod.ProductName("de")
testLogger.WithField("name", prodName).Info("Product loaded")
```
This will be the output at the end of the log when you run the test:
```shell
INFO[0000] Product loaded name="Whisk Beater"
```
The first line loads the product from the space. This is a `*testapi.CfProduct` pointer. The type is generated
and carries all the getter and setter methods to access all the fields and more, e.g. ProductName().
Note that when calling ProductName() we passed `"de"` as a parameter. This is the locale and it's
entirely optional and useful when your space supports multiple locales for translation.
If you omit it, the default space locale will be used.
Let's load the product's brand:
```go
// Get the brand
brandReference := prod.Brand()
brand := brandReference.VO.(*testapi.CfBrand)
testLogger.WithField("name", brand.CompanyName()).Info("Brand")
```
Note that:
- The product has a Brand() method that represents and retrieves the reference from the product entry to the brand entry
- The returned object is not a `*testapi.CfBrand` pointer as you might expect. This is because a reference field in Contentful
can point to entries of multiple content types and that doesn't play nice with Go's static typing.
The object returned is a generic `*testapi.EntryReference` that, among other, includes an `interface{}` attribute (VO) that
is the actual `*testapi.CfBrand`. That's why in the second line we had to cast it.
The test now logs the brand company name:
```shell
INFO[0000] Brand name="Normann Copenhagen"
```
What if we want to follow the reference the other way around and find out which entries link to this brand?
```go
parentRefs, errParents := brand.GetParents()
require.NoError(t, errParents)
testLogger.WithField("parent count", len(parentRefs)).Info("Parents")
for _, parentRef := range parentRefs {
switch parentRef.ContentType {
case testapi.ContentTypeProduct:
parentProduct := parentRef.VO.(*testapi.CfProduct)
testLogger.WithField("name", parentProduct.ProductName()).Info("Parent product")
}
}
```
Again, the `GetParents()` method returns references and not objects. It's a good idea to use the reference `ContentType` attribute
to switch before casting the VO to the type, because as we just said referenced objects can come in different types and casting
to the wrong one would make the runtime panic. Running the test we find out the two products that belong to this brand:
```shell
INFO[0000] Parents parent count=2
INFO[0000] Parent product name="Whisk Beater"
INFO[0000] Parent product name="Hudson Wall Cup"
```

47
docs/index.md Normal file
View File

@ -0,0 +1,47 @@
---
sidebar_label: Gocontentful
sidebar_position: 1
---
# Gocontentful documentation
Gocontentful is a command line tool that generates a set of APIs for the [Go Language](https://go.dev) to interact with a [Contentful](https://www.contentful.com) CMS space.
Unlike the plain Contentful API for Go, the Gocontentful API is idiomatic. Go types are provided with names that mirror the content types of the Contentful space, and get/set methods are named after each field.
In addition, Gocontentful supports in-memory caching and updates of spaces. This way, the space is always accessible through fast Go function calls, even offline.
## What is Contentful
[Contentful](https://www.contentful.com/) is a content platform (often referred to as headless CMS) for [micro-content](https://www.contentful.com/r/knowledgebase/content-as-a-microservice/).
Unlike traditional CMSes, there's no pages or content trees in Contentful. The data model is built from scratch for the purpose of the consuming application, is completely flexible and can be created and hot-changed through the same Web UI that the content editors use. The model dictates which content types can reference others and the final structure is a graph.
## How applications interact with Contentful
Contentful hosts several APIs that remote applications use to create, retrieve, update and delete content. Content is any of the following:
- **Entries**, each with a content type name and a list of data fields as defined by the developer in the content model editor at Contentful
- **Assets** (images, videos, other binary files)
The Contentful APIs exist as either REST or GraphQL endpoints. Gocontentful only supports the REST APIs.
The REST APIs used to manage and retrieve content use standard HTTP verbs (GET, POST, PUT and DELETE) and a JSON payload for both the request (where needed) and the response.
## What is gocontentful
A golang API code generator that simplifies interacting with a Contentful space. The generated API:
- Supports most of the Contentful APIs to perform all read/write operation on entries and assets
- Hides the complexity of the Contentful REST/JSON APIs behind an idiomatic set of golang functions and methods
- Allows for in-memory caching of an entire Contentful space
## Why we need a Go API generator
While it's perfectly fine to call a REST service and receive data in JSON format, in Go that is not very practical. For each content type, the developer needs to maintan type definitions by hand and decode the JSON coming from the Contentful server into the value object.
In addition, calling a remote API across the Internet each time a piece of content is needed, even multiple times for a single page rendering, can have significant impact on performance.
Gocontentful generates a Go API that handles both issues above and can be regenerated every time the content model changes. The developer never needs to update the types by hand, or deal with the complexity of caching content locally. It all happens auytomatically in the generated client.
> **NOTE** - _How much code does Gocontentful generate? In a real-world production scenario where Gocontentful is in use as of 2022, a space content model with 11 content types ranging from 3 to over 40 fields each generates around 50,000 lines of Go code._

110
docs/setup.md Normal file
View File

@ -0,0 +1,110 @@
---
sidebar_label: Setup
sidebar_position: 1
---
# Gocontentful Setup
## Installation
Prerequisite: you need Go 1.21. Upgrade if you still haven't, then run:
```shell
go get github.com/foomo/gocontentful
```
You can run gocontentful from the repository's main folder with
```
go run main.go LIST_OF_PARAMS
```
or build the binary yourself. If you trust us there are [precompiled versions on Github](https://github.com/foomo/gocontentful/releases)
On Mac OS:
```shell
brew install foomo/gocontentful/gocontentful
```
Test the installation (make sure $GOPATH/bin is in your $PATH):
```shell
$ gocontentful -version
v1.1.0
```
## Optional tools
Gocontentful requires a CMA API key to scan the Contentful space and generate the model.
This can be passed as a CLI parameter but that's tedious, and your management key will remain in your shell history.
A better approach is to log in to Contentful using the official _Contentful CLI_. Gocontentful will get the key automatically.
To install the CLI refer to the [official documentation at Contentful.com](https://www.contentful.com/developers/docs/tutorials/cli/installation/).
After installing the CLI, log in inside your terminal with:
```shell
$ contentful login
```
After a roundtrip through the Web authentication pages at Contentful, you'll be logged in:
```
A browser window will open where you will log in (or sign up if you dont have an account), authorize this CLI tool and paste your CMA token here:
? Continue login on the browser? Yes
? Paste your token here: *******************************************
Great! You've successfully logged in!
╭─────────────────────────────────────────────────────────────────────────╮
│ │
│ Your management token: ************** │
│ Stored at: /Users/yourusername/.contentfulrc.json │
│ │
╰─────────────────────────────────────────────────────────────────────────╯
```
## Generate a client for your space
The gocontentful command accepts the following parameters:
```shell
$ gocontentful -help
ERROR: Please specify either a Contentful Space ID and CMA access token or an export file name
SYNOPSIS
gocontentful -spaceid SpaceID -cmakey CMAKey [-contenttypes firsttype,secondtype...lasttype] path/to/target/package
-cmakey string
[Optional] Contentful CMA key
-contenttypes string
[Optional] Content type IDs to parse, comma separated
-environment string
[Optional] Contentful space environment
-exportfile string
Space export file to generate the API from
-help
Print version and exit
-spaceid string
Contentful space ID
-version
Print version and exit
Notes:
- The last segment of the path/to/target/package will be used as package name
- The -cmakey parameter can be omitted if you logged in with the Contentful CLI
```
Notes:
- The last segment of the path/to/target/package will be used as package name
- You need to pass gocontentful either cmakey/spaceid (and optional environment) to generate the API from a live space or exportfile to generate it from a local space export file. The cmakey can be omitted if you are logged in through the Contentful CLI.
Assuming you are logged in with the Contentful CLI and your space id at Contentful is xyz123, you can now generate the Gocontentful client files for a package named "myclient" with:
```
gocontentful -spaceid xyz123 myclient
```
This will create all the necessary Go files inside the `myclient` directory. When that happens without errors, your setup is complete.

View File

@ -33,6 +33,12 @@ type RawItem struct {
Fields RawFields `json:"fields"` Fields RawFields `json:"fields"`
} }
type GenericEntry struct {
Sys ContentfulSys `json:"sys,omitempty"`
RawFields RawFields `json:"fields,omitempty"`
CC *ContentfulClient `json:"-"`
}
type ContentfulReferencedEntry struct { type ContentfulReferencedEntry struct {
Entry *contentful.Entry Entry *contentful.Entry
Col *contentful.Collection Col *contentful.Collection

View File

@ -9,7 +9,6 @@ import (
"fmt" "fmt"
"html" "html"
"io" "io"
"os"
"regexp" "regexp"
"strings" "strings"
"sync" "sync"
@ -30,6 +29,7 @@ type ContentfulCache struct {
assets assetCacheMap assets assetCacheMap
contentTypes []string contentTypes []string
entryMaps cacheEntryMaps entryMaps cacheEntryMaps
genericEntries map[string]*GenericEntry
idContentTypeMap map[string]string idContentTypeMap map[string]string
parentMap map[string][]EntryReference parentMap map[string][]EntryReference
} }
@ -40,6 +40,7 @@ type ContentfulCacheMutex struct {
assetsGcLock sync.RWMutex assetsGcLock sync.RWMutex
idContentTypeMapGcLock sync.RWMutex idContentTypeMapGcLock sync.RWMutex
parentMapGcLock sync.RWMutex parentMapGcLock sync.RWMutex
genericEntriesGcLock sync.RWMutex
{{ range $index , $contentType := $contentTypes }} {{ $contentType.Sys.ID }}GcLock sync.RWMutex {{ range $index , $contentType := $contentTypes }} {{ $contentType.Sys.ID }}GcLock sync.RWMutex
{{ end }} {{ end }}
} }
@ -160,19 +161,23 @@ var (
InfoCachedAllAssets = "cached all assets" InfoCachedAllAssets = "cached all assets"
InfoFallingBackToFile = "gonna use a local file" InfoFallingBackToFile = "gonna use a local file"
InfoLoadingFromFile = "loading space from local file" InfoLoadingFromFile = "loading space from local file"
InfoCacheIsNil = "contentful cache is nil"
InfoCacheWorkerStart = "contentful cache worker starting" InfoCacheWorkerStart = "contentful cache worker starting"
InfoCacheUpdateQueued = "contentful cache update queued" InfoCacheUpdateQueued = "contentful cache update queued"
InfoCacheUpdateCanceled = "contentful cache update canceled" InfoCacheUpdateCanceled = "contentful cache update canceled"
InfoCacheUpdateDone = "contentful cache update returning" InfoCacheUpdateDone = "contentful cache update returning"
InfoCacheUpdateSkipped = "contentful cache update skipped, already one in the queue" InfoCacheUpdateSkipped = "contentful cache update skipped, already one in the queue"
InfoCacheSyncOp = "contentful cache sync op done"
InfoOfflineEntitiesLoaded = "downloaded entries and assets from offline file" InfoOfflineEntitiesLoaded = "downloaded entries and assets from offline file"
InfoPreservingExistingCache = "could not connect for cache update, preserving the existing cache" InfoPreservingExistingCache = "could not connect for cache update, preserving the existing cache"
InfoUpdateCacheTime = "space caching done, time recorded" InfoUpdateCacheTime = "space caching done, time recorded"
ErrorEnvironmentSetToMaster = "environment was empty string, set to master" ErrorEnvironmentSetToMaster = "environment was empty string, set to master"
ErrorEntryNotFound = "entry not found"
ErrorEntryIsNil = "entry is nil" ErrorEntryIsNil = "entry is nil"
ErrorEntrySysIsNil = "entry.Sys is nil" ErrorEntrySysIsNil = "entry.Sys is nil"
ErrorEntrySysContentTypeIsNil = "entry.Sys.ContentType is nil" ErrorEntrySysContentTypeIsNil = "entry.Sys.ContentType is nil"
ErrorEntrySysContentTypeSysIsNil = "entry.Sys.ContentType.Sys is nil" ErrorEntrySysContentTypeSysIsNil = "entry.Sys.ContentType.Sys is nil"
ErrorEntryCachingFailed = "entry caching failed"
) )
var spaceContentTypes = []string{ {{ range $index , $contentType := $contentTypes }}ContentType{{ firstCap $contentType.Sys.ID }}, {{ end }} } var spaceContentTypes = []string{ {{ range $index , $contentType := $contentTypes }}ContentType{{ firstCap $contentType.Sys.ID }}, {{ end }} }
@ -246,7 +251,10 @@ func (cc *ContentfulClient) ClientStats() {
for _, parents := range cc.Cache.parentMap { for _, parents := range cc.Cache.parentMap {
referenceCount += len(parents) referenceCount += len(parents)
} }
fieldsMap["cache parentMap parents"] = referenceCount fieldsMap["cache parentMap parents"] = referenceCount
cc.cacheMutex.genericEntriesGcLock.RLock()
defer cc.cacheMutex.genericEntriesGcLock.RUnlock()
fieldsMap["cache genericEntries"] = len(cc.Cache.genericEntries)
} }
cc.logFn(fieldsMap, LogInfo, "Contentful ClientStats") cc.logFn(fieldsMap, LogInfo, "Contentful ClientStats")
} }
@ -475,6 +483,7 @@ func NewContentfulClient(spaceID string, clientMode ClientMode, clientKey string
assets: assetCacheMap{}, assets: assetCacheMap{},
contentTypes : []string{}, contentTypes : []string{},
entryMaps: cacheEntryMaps{}, entryMaps: cacheEntryMaps{},
genericEntries: map[string]*GenericEntry{},
idContentTypeMap: map[string]string{}, idContentTypeMap: map[string]string{},
parentMap: map[string][]EntryReference{}, parentMap: map[string][]EntryReference{},
}, },
@ -484,6 +493,7 @@ func NewContentfulClient(spaceID string, clientMode ClientMode, clientKey string
assetsGcLock: sync.RWMutex{}, assetsGcLock: sync.RWMutex{},
idContentTypeMapGcLock: sync.RWMutex{}, idContentTypeMapGcLock: sync.RWMutex{},
parentMapGcLock: sync.RWMutex{}, parentMapGcLock: sync.RWMutex{},
genericEntriesGcLock: sync.RWMutex{},
{{ range $index , $contentType := $contentTypes }} {{ $contentType.Sys.ID }}GcLock: sync.RWMutex{}, {{ range $index , $contentType := $contentTypes }} {{ $contentType.Sys.ID }}GcLock: sync.RWMutex{},
{{ end }} {{ end }}
}, },
@ -508,8 +518,8 @@ func NewContentfulClient(spaceID string, clientMode ClientMode, clientKey string
return cc, nil return cc, nil
} }
func NewOfflineContentfulClient(filename string, logFn func(fields map[string]interface{}, level int, args ...interface{}), logLevel int, cacheAssets bool, textJanitor bool) (*ContentfulClient, error) { func NewOfflineContentfulClient(file []byte, logFn func(fields map[string]interface{}, level int, args ...interface{}), logLevel int, cacheAssets bool, textJanitor bool) (*ContentfulClient, error) {
offlineTemp, err := getOfflineSpaceFromFile(filename) offlineTemp, err := getOfflineSpaceFromFile(file)
if err != nil { if err != nil {
return nil, fmt.Errorf("NewOfflineContentfulClient could not parse space export file: %v", err) return nil, fmt.Errorf("NewOfflineContentfulClient could not parse space export file: %v", err)
} }
@ -519,6 +529,7 @@ func NewOfflineContentfulClient(filename string, logFn func(fields map[string]in
Cache: &ContentfulCache{ Cache: &ContentfulCache{
contentTypes: []string{}, contentTypes: []string{},
idContentTypeMap: map[string]string{}, idContentTypeMap: map[string]string{},
genericEntries: map[string]*GenericEntry{},
parentMap: map[string][]EntryReference{}, parentMap: map[string][]EntryReference{},
}, },
cacheMutex: &ContentfulCacheMutex{ cacheMutex: &ContentfulCacheMutex{
@ -575,6 +586,22 @@ func RichTextToHtml(rt interface{}, linkResolver LinkResolverFunc, entryLinkReso
return out, nil return out, nil
} }
func (cc *ContentfulClient) GetGenericEntry(entryID string) (*GenericEntry, error) {
if cc.Cache == nil {
if cc.logFn != nil && cc.logLevel <= LogWarn {
cc.logFn(map[string]interface{}{"task": "GetGenericEntry", "entryId":entryID}, LogWarn, InfoCacheIsNil)
}
return nil, errors.New(InfoCacheIsNil)
}
if _, ok := cc.Cache.genericEntries[entryID]; !ok {
if cc.logFn != nil && cc.logLevel <= LogWarn {
cc.logFn(map[string]interface{}{"task": "GetGenericEntry", "entryId":entryID}, LogWarn, ErrorEntryNotFound)
}
return nil, errors.New(ErrorEntryNotFound)
}
return cc.Cache.genericEntries[entryID], nil
}
func (cc *ContentfulClient) SetCacheUpdateTimeout(seconds int64) { func (cc *ContentfulClient) SetCacheUpdateTimeout(seconds int64) {
cc.cacheUpdateTimeout = seconds cc.cacheUpdateTimeout = seconds
} }
@ -590,8 +617,8 @@ func (cc *ContentfulClient) SetEnvironment(environment string) {
cc.Client.Environment = environment cc.Client.Environment = environment
} }
func (cc *ContentfulClient) SetOfflineFallback(filename string) error { func (cc *ContentfulClient) SetOfflineFallback(file []byte) error {
offlineTemp, err := getOfflineSpaceFromFile(filename) offlineTemp, err := getOfflineSpaceFromFile(file)
if err != nil { if err != nil {
return err return err
} }
@ -721,30 +748,62 @@ func (cc *ContentfulClient) syncCache(ctx context.Context, contentTypes []string
entries = append(entries, entry) entries = append(entries, entry)
} }
} }
syncEntryCount := map[string]int{}
for _, entry := range entries { for _, entry := range entries {
switch entry.Sys.Type { switch entry.Sys.Type {
case sysTypeEntry: case sysTypeEntry:
if !stringSliceContains(spaceContentTypes, entry.Sys.ContentType.Sys.ID) { if !stringSliceContains(spaceContentTypes, entry.Sys.ContentType.Sys.ID) {
continue continue
} }
updateCacheForContentTypeAndEntity(ctx, cc, entry.Sys.ContentType.Sys.ID, entry.Sys.ID, entry, false) errUpdate := updateCacheForContentTypeAndEntity(ctx, cc, entry.Sys.ContentType.Sys.ID, entry.Sys.ID, entry, false)
if errUpdate != nil && cc.logFn != nil && cc.logLevel <= LogInfo {
cc.logFn(map[string]interface{}{"task": "syncCache", "error": errUpdate, "contentType": entry.Sys.ContentType.Sys.ID, "entryId": entry.Sys.ID},
LogWarn, ErrorEntryCachingFailed)
} else {
syncEntryCount[entry.Sys.ContentType.Sys.ID]++
}
case sysTypeDeletedEntry: case sysTypeDeletedEntry:
cc.cacheMutex.idContentTypeMapGcLock.RLock() cc.cacheMutex.idContentTypeMapGcLock.RLock()
contentType := cc.Cache.idContentTypeMap[entry.Sys.ID] contentType := cc.Cache.idContentTypeMap[entry.Sys.ID]
cc.cacheMutex.idContentTypeMapGcLock.RUnlock() cc.cacheMutex.idContentTypeMapGcLock.RUnlock()
updateCacheForContentTypeAndEntity(ctx, cc, contentType, entry.Sys.ID, entry, true) errUpdate := updateCacheForContentTypeAndEntity(ctx, cc, contentType, entry.Sys.ID, entry, true)
if errUpdate != nil && cc.logFn != nil && cc.logLevel <= LogInfo {
cc.logFn(map[string]interface{}{"task": "syncCache", "error": errUpdate, "op":"delete", "contentType": entry.Sys.ContentType.Sys.ID, "entryId": entry.Sys.ID},
LogWarn, ErrorEntryCachingFailed)
} else {
syncEntryCount["deletedEntry"]++
}
default: default:
} }
} }
if cc.logFn != nil && len(entries) > 0 && cc.logLevel <= LogInfo {
cc.logFn(map[string]interface{}{"task": "syncCache", "syncEntryCount": syncEntryCount}, LogInfo, InfoCacheSyncOp)
}
var syncAssetCount int
for _, asset := range assets { for _, asset := range assets {
switch asset.Sys.Type { switch asset.Sys.Type {
case sysTypeAsset: case sysTypeAsset:
updateCacheForContentTypeAndEntity(ctx, cc, assetWorkerType, asset.Sys.ID, asset, false) errUpdate := updateCacheForContentTypeAndEntity(ctx, cc, assetWorkerType, asset.Sys.ID, asset, false)
if errUpdate != nil && cc.logFn != nil && cc.logLevel <= LogInfo {
cc.logFn(map[string]interface{}{"task": "syncCache", "error": errUpdate, "assetId": asset.Sys.ID},
LogWarn, ErrorEntryCachingFailed)
} else {
syncAssetCount++
}
case sysTypeDeletedAsset: case sysTypeDeletedAsset:
updateCacheForContentTypeAndEntity(ctx, cc, assetWorkerType, asset.Sys.ID, nil, true) errUpdate := updateCacheForContentTypeAndEntity(ctx, cc, assetWorkerType, asset.Sys.ID, nil, true)
if errUpdate != nil && cc.logFn != nil && cc.logLevel <= LogInfo {
cc.logFn(map[string]interface{}{"task": "syncCache", "error": errUpdate, "op":"delete", "assetId": asset.Sys.ID},
LogWarn, ErrorEntryCachingFailed)
} else {
syncAssetCount++
}
default: default:
} }
} }
if cc.logFn != nil && len(assets) > 0 && cc.logLevel <= LogInfo {
cc.logFn(map[string]interface{}{"task": "syncCache", "syncAssetCount": syncAssetCount}, LogInfo, InfoCacheSyncOp)
}
} }
} }
@ -762,6 +821,7 @@ func (cc *ContentfulClient) cacheSpace(ctx context.Context, contentTypes []strin
tempCache := &ContentfulCache{ tempCache := &ContentfulCache{
contentTypes: contentTypes, contentTypes: contentTypes,
idContentTypeMap: map[string]string{}, idContentTypeMap: map[string]string{},
genericEntries: map[string]*GenericEntry{},
parentMap: map[string][]EntryReference{}, parentMap: map[string][]EntryReference{},
} }
if cacheAssets { if cacheAssets {
@ -843,6 +903,7 @@ func (cc *ContentfulClient) cacheSpace(ctx context.Context, contentTypes []strin
cc.cacheMutex.assetsGcLock.Lock() cc.cacheMutex.assetsGcLock.Lock()
cc.cacheMutex.idContentTypeMapGcLock.Lock() cc.cacheMutex.idContentTypeMapGcLock.Lock()
cc.cacheMutex.parentMapGcLock.Lock() cc.cacheMutex.parentMapGcLock.Lock()
cc.cacheMutex.genericEntriesGcLock.Lock()
{{ range $index , $contentType := $contentTypes }}cc.cacheMutex.{{ $contentType.Sys.ID }}GcLock.Lock() {{ range $index , $contentType := $contentTypes }}cc.cacheMutex.{{ $contentType.Sys.ID }}GcLock.Lock()
{{ end }} {{ end }}
cc.Cache = tempCache cc.Cache = tempCache
@ -854,6 +915,7 @@ func (cc *ContentfulClient) cacheSpace(ctx context.Context, contentTypes []strin
cc.cacheMutex.assetsGcLock.Unlock() cc.cacheMutex.assetsGcLock.Unlock()
cc.cacheMutex.sharedDataGcLock.Unlock() cc.cacheMutex.sharedDataGcLock.Unlock()
cc.cacheMutex.fullCacheGcLock.Unlock() cc.cacheMutex.fullCacheGcLock.Unlock()
cc.cacheMutex.genericEntriesGcLock.Unlock()
} }
func ToAssetReference(asset *contentful.Asset) (refSys ContentTypeSys) { func ToAssetReference(asset *contentful.Asset) (refSys ContentTypeSys) {
@ -1014,13 +1076,9 @@ func (cc *ContentfulClient) getAllAssets(tryCacheFirst bool) (map[string]*conten
return assets, nil return assets, nil
} }
func getOfflineSpaceFromFile(filename string) (*offlineTemp, error) { func getOfflineSpaceFromFile(file []byte) (*offlineTemp, error) {
fileBytes, err := os.ReadFile(filename)
if err != nil {
return nil, fmt.Errorf("getOfflineSpaceFromFile could not read space export file: %v", err)
}
offlineTemp := &offlineTemp{} offlineTemp := &offlineTemp{}
err = json.Unmarshal(fileBytes, offlineTemp) err := json.Unmarshal(file, offlineTemp)
if err != nil { if err != nil {
return nil, fmt.Errorf("getOfflineSpaceFromFile could not parse space export file: %v", err) return nil, fmt.Errorf("getOfflineSpaceFromFile could not parse space export file: %v", err)
} }
@ -1471,6 +1529,15 @@ func updateCacheForContentType(ctx context.Context, results chan ContentTypeResu
return errors.New("updateCacheForContentType failed for contentType {{ $contentType.Sys.ID }}: "+err.Error()) return errors.New("updateCacheForContentType failed for contentType {{ $contentType.Sys.ID }}: "+err.Error())
} }
tempCache.entryMaps.{{ $contentType.Sys.ID }} = all{{ firstCap $contentType.Sys.ID }} tempCache.entryMaps.{{ $contentType.Sys.ID }} = all{{ firstCap $contentType.Sys.ID }}
cc.cacheMutex.genericEntriesGcLock.Lock()
for _, {{ $contentType.Sys.ID }} := range all{{ firstCap $contentType.Sys.ID }} {
tempCache.genericEntries[{{ $contentType.Sys.ID }}.Sys.ID] = &GenericEntry{
Sys: {{ $contentType.Sys.ID }}.Sys,
RawFields: {{ $contentType.Sys.ID }}.RawFields,
CC: {{ $contentType.Sys.ID }}.CC,
}
}
cc.cacheMutex.genericEntriesGcLock.Unlock()
if cc.logFn != nil && cc.logLevel <= LogInfo { if cc.logFn != nil && cc.logLevel <= LogInfo {
cc.logFn(map[string]interface{}{"contentType":"{{ $contentType.Sys.ID }}", "method": "updateCacheForContentType", "size":len(all{{ firstCap $contentType.Sys.ID }})}, LogInfo, InfoCachedAllEntries) cc.logFn(map[string]interface{}{"contentType":"{{ $contentType.Sys.ID }}", "method": "updateCacheForContentType", "size":len(all{{ firstCap $contentType.Sys.ID }})}, LogInfo, InfoCachedAllEntries)
} }
@ -1530,7 +1597,7 @@ func updateCacheForContentTypeAndEntity(ctx context.Context, cc *ContentfulClien
return nil return nil
} }
func commonGetParents(cc *ContentfulClient, id string, contentType []string) (parents []EntryReference, err error) { func commonGetParents(cc *ContentfulClient, id string, contentTypes []string) (parents []EntryReference, err error) {
parents = []EntryReference{} parents = []EntryReference{}
cc.cacheMutex.sharedDataGcLock.RLock() cc.cacheMutex.sharedDataGcLock.RLock()
cacheInit := cc.cacheInit cacheInit := cc.cacheInit
@ -1538,10 +1605,12 @@ func commonGetParents(cc *ContentfulClient, id string, contentType []string) (pa
if cacheInit { if cacheInit {
cc.cacheMutex.parentMapGcLock.RLock() cc.cacheMutex.parentMapGcLock.RLock()
defer cc.cacheMutex.parentMapGcLock.RUnlock() defer cc.cacheMutex.parentMapGcLock.RUnlock()
if len(contentType) != 0 { if len(contentTypes) != 0 {
for _, parent := range cc.Cache.parentMap[id] { for _, parent := range cc.Cache.parentMap[id] {
if parent.ContentType == contentType[0] { for _, contentType := range contentTypes {
parents = append(parents, parent) if parent.ContentType == contentType {
parents = append(parents, parent)
}
} }
} }
return parents, nil return parents, nil
@ -1564,7 +1633,7 @@ func commonGetParents(cc *ContentfulClient, id string, contentType []string) (pa
if err != nil { if err != nil {
return nil, err return nil, err
} }
if len(contentType) == 1 && contentType[0] != entry.Sys.ContentType.Sys.ID { if len(contentTypes) == 1 && contentTypes[0] != entry.Sys.ContentType.Sys.ID {
continue continue
} }
switch entry.Sys.ContentType.Sys.ID { {{ range $index , $contentType := $contentTypes }} switch entry.Sys.ContentType.Sys.ID { {{ range $index , $contentType := $contentTypes }}

View File

@ -735,6 +735,8 @@ func (cc *ContentfulClient) cache{{ firstCap $contentType.Sys.ID }}ByID(ctx cont
defer cc.cacheMutex.idContentTypeMapGcLock.Unlock() defer cc.cacheMutex.idContentTypeMapGcLock.Unlock()
cc.cacheMutex.parentMapGcLock.Lock() cc.cacheMutex.parentMapGcLock.Lock()
defer cc.cacheMutex.parentMapGcLock.Unlock() defer cc.cacheMutex.parentMapGcLock.Unlock()
cc.cacheMutex.genericEntriesGcLock.Lock()
defer cc.cacheMutex.genericEntriesGcLock.Unlock()
var col *contentful.Collection var col *contentful.Collection
if entryPayload != nil { if entryPayload != nil {
@ -757,6 +759,7 @@ func (cc *ContentfulClient) cache{{ firstCap $contentType.Sys.ID }}ByID(ctx cont
} }
// It was deleted // It was deleted
if col != nil && len(col.Items) == 0 || entryDelete { if col != nil && len(col.Items) == 0 || entryDelete {
delete(cc.Cache.genericEntries, id)
delete(cc.Cache.entryMaps.{{ $contentType.Sys.ID }}, id) delete(cc.Cache.entryMaps.{{ $contentType.Sys.ID }}, id)
delete(cc.Cache.idContentTypeMap, id) delete(cc.Cache.idContentTypeMap, id)
// delete as child // delete as child
@ -782,7 +785,12 @@ func (cc *ContentfulClient) cache{{ firstCap $contentType.Sys.ID }}ByID(ctx cont
cc.Cache.entryMaps.{{ $contentType.Sys.ID }} = map[string]*Cf{{ firstCap $contentType.Sys.ID }}{} cc.Cache.entryMaps.{{ $contentType.Sys.ID }} = map[string]*Cf{{ firstCap $contentType.Sys.ID }}{}
} }
cc.Cache.entryMaps.{{ $contentType.Sys.ID }}[id] = {{ $contentType.Sys.ID }} cc.Cache.entryMaps.{{ $contentType.Sys.ID }}[id] = {{ $contentType.Sys.ID }}
cc.Cache.idContentTypeMap[id] = {{ $contentType.Sys.ID }}.Sys.ContentType.Sys.ID cc.Cache.genericEntries[id] = &GenericEntry{
Sys: {{ $contentType.Sys.ID }}.Sys,
RawFields: {{ $contentType.Sys.ID }}.RawFields,
CC: {{ $contentType.Sys.ID }}.CC,
}
cc.Cache.idContentTypeMap[id] = {{ $contentType.Sys.ID }}.Sys.ContentType.Sys.ID
allChildrensIds := map[string]bool{} allChildrensIds := map[string]bool{}
{{ range $fieldIndex, $field := $contentType.Fields }} {{ range $fieldIndex, $field := $contentType.Fields }}
{{ if fieldIsMultipleReference $field }} {{ if fieldIsMultipleReference $field }}

11
go.mod
View File

@ -1,15 +1,22 @@
module github.com/foomo/gocontentful module github.com/foomo/gocontentful
go 1.16 go 1.21
require ( require (
github.com/foomo/contentful v0.3.6 github.com/foomo/contentful v0.3.6
github.com/sirupsen/logrus v1.8.1 github.com/sirupsen/logrus v1.8.1
github.com/smartystreets/goconvey v1.6.4 // indirect
github.com/stretchr/testify v1.7.0 github.com/stretchr/testify v1.7.0
golang.org/x/sync v0.1.0 golang.org/x/sync v0.1.0
golang.org/x/text v0.14.0 golang.org/x/text v0.14.0
golang.org/x/tools v0.6.0 golang.org/x/tools v0.6.0
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c
)
require (
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/smartystreets/goconvey v1.6.4 // indirect
golang.org/x/mod v0.8.0 // indirect
golang.org/x/sys v0.5.0 // indirect
moul.io/http2curl v1.0.0 // indirect moul.io/http2curl v1.0.0 // indirect
) )

23
go.sum
View File

@ -19,45 +19,22 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=

View File

@ -14,7 +14,7 @@ import (
"github.com/foomo/gocontentful/erm" "github.com/foomo/gocontentful/erm"
) )
var VERSION = "v1.0.26" var VERSION = "v1.1.0"
type contentfulRc struct { type contentfulRc struct {
ManagementToken string `json:"managementToken"` ManagementToken string `json:"managementToken"`
@ -103,8 +103,10 @@ func main() {
if cmaKey == "" && *flagGenerateFromExport == "" { if cmaKey == "" && *flagGenerateFromExport == "" {
cmaKey = getCmaKeyFromRcFile() cmaKey = getCmaKeyFromRcFile()
} }
if conf.ExportFile == "" && (conf.SpaceID == "" || cmaKey == "") || if conf.ExportFile == "" && conf.SpaceID == "" ||
conf.ExportFile != "" && (conf.SpaceID != "" || cmaKey != "") { conf.ExportFile != "" && conf.SpaceID != "" {
byt, _ := json.MarshalIndent(conf, "", " ")
fmt.Println(string(byt))
usageError("Please provide either a Contentful Space ID and CMA access token or an export file name") usageError("Please provide either a Contentful Space ID and CMA access token or an export file name")
} }
var path string var path string

View File

@ -82,7 +82,9 @@ func TestCacheIfNewEntry(t *testing.T) {
stats, err := contentfulClient.GetCacheStats() stats, err := contentfulClient.GetCacheStats()
require.NoError(t, err) require.NoError(t, err)
require.Equal(t, 9, stats.EntryCount) require.Equal(t, 9, stats.EntryCount)
err = contentfulClient.SetOfflineFallback("./test-space-export-newer.json") testFile, err := GetTestFile("./test-space-export-newer.json")
require.NoError(t, err)
err = contentfulClient.SetOfflineFallback(testFile)
require.NoError(t, err) require.NoError(t, err)
err = contentfulClient.UpdateCache(context.Background(), nil, false) err = contentfulClient.UpdateCache(context.Background(), nil, false)
require.NoError(t, err) require.NoError(t, err)
@ -94,7 +96,9 @@ func TestCacheIfNewEntry(t *testing.T) {
func TestPreserveCacheIfNewer(t *testing.T) { func TestPreserveCacheIfNewer(t *testing.T) {
contentfulClient, err := getTestClient() contentfulClient, err := getTestClient()
require.NoError(t, err) require.NoError(t, err)
err = contentfulClient.SetOfflineFallback("./test-space-export-older.json") testFile, err := GetTestFile("./test-space-export-older.json")
require.NoError(t, err)
err = contentfulClient.SetOfflineFallback(testFile)
require.NoError(t, err) require.NoError(t, err)
err = contentfulClient.UpdateCache(context.TODO(), nil, false) err = contentfulClient.UpdateCache(context.TODO(), nil, false)
require.NoError(t, err) require.NoError(t, err)

View File

@ -27,7 +27,9 @@ func TestPublishingStatus(t *testing.T) {
func TestCleanUpUnicode(t *testing.T) { func TestCleanUpUnicode(t *testing.T) {
testLogger := logrus.StandardLogger() testLogger := logrus.StandardLogger()
cc, errClient := testapi.NewOfflineContentfulClient("./test-space-export.json", testFile, err := GetTestFile("./test-space-export.json")
require.NoError(t, err)
cc, errClient := testapi.NewOfflineContentfulClient(testFile,
GetContenfulLogger(testLogger), GetContenfulLogger(testLogger),
LogDebug, LogDebug,
true, true,

View File

@ -1,4 +1,4 @@
// Code generated by https://github.com/foomo/gocontentful v1.0.26 - DO NOT EDIT. // Code generated by https://github.com/foomo/gocontentful v1.1.0 - DO NOT EDIT.
package testapi package testapi
import ( import (

View File

@ -1,4 +1,4 @@
// Code generated by https://github.com/foomo/gocontentful v1.0.26 - DO NOT EDIT. // Code generated by https://github.com/foomo/gocontentful v1.1.0 - DO NOT EDIT.
package testapi package testapi
import "github.com/foomo/contentful" import "github.com/foomo/contentful"
@ -33,6 +33,12 @@ type RawItem struct {
Fields RawFields `json:"fields"` Fields RawFields `json:"fields"`
} }
type GenericEntry struct {
Sys ContentfulSys `json:"sys,omitempty"`
RawFields RawFields `json:"fields,omitempty"`
CC *ContentfulClient `json:"-"`
}
type ContentfulReferencedEntry struct { type ContentfulReferencedEntry struct {
Entry *contentful.Entry Entry *contentful.Entry
Col *contentful.Collection Col *contentful.Collection

View File

@ -1,4 +1,4 @@
// Code generated by https://github.com/foomo/gocontentful v1.0.26 - DO NOT EDIT. // Code generated by https://github.com/foomo/gocontentful v1.1.0 - DO NOT EDIT.
package testapi package testapi
import ( import (
@ -9,7 +9,6 @@ import (
"fmt" "fmt"
"html" "html"
"io" "io"
"os"
"regexp" "regexp"
"strings" "strings"
"sync" "sync"
@ -32,6 +31,7 @@ type ContentfulCache struct {
assets assetCacheMap assets assetCacheMap
contentTypes []string contentTypes []string
entryMaps cacheEntryMaps entryMaps cacheEntryMaps
genericEntries map[string]*GenericEntry
idContentTypeMap map[string]string idContentTypeMap map[string]string
parentMap map[string][]EntryReference parentMap map[string][]EntryReference
} }
@ -42,6 +42,7 @@ type ContentfulCacheMutex struct {
assetsGcLock sync.RWMutex assetsGcLock sync.RWMutex
idContentTypeMapGcLock sync.RWMutex idContentTypeMapGcLock sync.RWMutex
parentMapGcLock sync.RWMutex parentMapGcLock sync.RWMutex
genericEntriesGcLock sync.RWMutex
brandGcLock sync.RWMutex brandGcLock sync.RWMutex
categoryGcLock sync.RWMutex categoryGcLock sync.RWMutex
productGcLock sync.RWMutex productGcLock sync.RWMutex
@ -166,19 +167,23 @@ var (
InfoCachedAllAssets = "cached all assets" InfoCachedAllAssets = "cached all assets"
InfoFallingBackToFile = "gonna use a local file" InfoFallingBackToFile = "gonna use a local file"
InfoLoadingFromFile = "loading space from local file" InfoLoadingFromFile = "loading space from local file"
InfoCacheIsNil = "contentful cache is nil"
InfoCacheWorkerStart = "contentful cache worker starting" InfoCacheWorkerStart = "contentful cache worker starting"
InfoCacheUpdateQueued = "contentful cache update queued" InfoCacheUpdateQueued = "contentful cache update queued"
InfoCacheUpdateCanceled = "contentful cache update canceled" InfoCacheUpdateCanceled = "contentful cache update canceled"
InfoCacheUpdateDone = "contentful cache update returning" InfoCacheUpdateDone = "contentful cache update returning"
InfoCacheUpdateSkipped = "contentful cache update skipped, already one in the queue" InfoCacheUpdateSkipped = "contentful cache update skipped, already one in the queue"
InfoCacheSyncOp = "contentful cache sync op done"
InfoOfflineEntitiesLoaded = "downloaded entries and assets from offline file" InfoOfflineEntitiesLoaded = "downloaded entries and assets from offline file"
InfoPreservingExistingCache = "could not connect for cache update, preserving the existing cache" InfoPreservingExistingCache = "could not connect for cache update, preserving the existing cache"
InfoUpdateCacheTime = "space caching done, time recorded" InfoUpdateCacheTime = "space caching done, time recorded"
ErrorEnvironmentSetToMaster = "environment was empty string, set to master" ErrorEnvironmentSetToMaster = "environment was empty string, set to master"
ErrorEntryNotFound = "entry not found"
ErrorEntryIsNil = "entry is nil" ErrorEntryIsNil = "entry is nil"
ErrorEntrySysIsNil = "entry.Sys is nil" ErrorEntrySysIsNil = "entry.Sys is nil"
ErrorEntrySysContentTypeIsNil = "entry.Sys.ContentType is nil" ErrorEntrySysContentTypeIsNil = "entry.Sys.ContentType is nil"
ErrorEntrySysContentTypeSysIsNil = "entry.Sys.ContentType.Sys is nil" ErrorEntrySysContentTypeSysIsNil = "entry.Sys.ContentType.Sys is nil"
ErrorEntryCachingFailed = "entry caching failed"
) )
var spaceContentTypes = []string{ContentTypeBrand, ContentTypeCategory, ContentTypeProduct} var spaceContentTypes = []string{ContentTypeBrand, ContentTypeCategory, ContentTypeProduct}
@ -264,6 +269,9 @@ func (cc *ContentfulClient) ClientStats() {
referenceCount += len(parents) referenceCount += len(parents)
} }
fieldsMap["cache parentMap parents"] = referenceCount fieldsMap["cache parentMap parents"] = referenceCount
cc.cacheMutex.genericEntriesGcLock.RLock()
defer cc.cacheMutex.genericEntriesGcLock.RUnlock()
fieldsMap["cache genericEntries"] = len(cc.Cache.genericEntries)
} }
cc.logFn(fieldsMap, LogInfo, "Contentful ClientStats") cc.logFn(fieldsMap, LogInfo, "Contentful ClientStats")
} }
@ -506,6 +514,7 @@ func NewContentfulClient(spaceID string, clientMode ClientMode, clientKey string
assets: assetCacheMap{}, assets: assetCacheMap{},
contentTypes: []string{}, contentTypes: []string{},
entryMaps: cacheEntryMaps{}, entryMaps: cacheEntryMaps{},
genericEntries: map[string]*GenericEntry{},
idContentTypeMap: map[string]string{}, idContentTypeMap: map[string]string{},
parentMap: map[string][]EntryReference{}, parentMap: map[string][]EntryReference{},
}, },
@ -515,6 +524,7 @@ func NewContentfulClient(spaceID string, clientMode ClientMode, clientKey string
assetsGcLock: sync.RWMutex{}, assetsGcLock: sync.RWMutex{},
idContentTypeMapGcLock: sync.RWMutex{}, idContentTypeMapGcLock: sync.RWMutex{},
parentMapGcLock: sync.RWMutex{}, parentMapGcLock: sync.RWMutex{},
genericEntriesGcLock: sync.RWMutex{},
brandGcLock: sync.RWMutex{}, brandGcLock: sync.RWMutex{},
categoryGcLock: sync.RWMutex{}, categoryGcLock: sync.RWMutex{},
productGcLock: sync.RWMutex{}, productGcLock: sync.RWMutex{},
@ -540,8 +550,8 @@ func NewContentfulClient(spaceID string, clientMode ClientMode, clientKey string
return cc, nil return cc, nil
} }
func NewOfflineContentfulClient(filename string, logFn func(fields map[string]interface{}, level int, args ...interface{}), logLevel int, cacheAssets bool, textJanitor bool) (*ContentfulClient, error) { func NewOfflineContentfulClient(file []byte, logFn func(fields map[string]interface{}, level int, args ...interface{}), logLevel int, cacheAssets bool, textJanitor bool) (*ContentfulClient, error) {
offlineTemp, err := getOfflineSpaceFromFile(filename) offlineTemp, err := getOfflineSpaceFromFile(file)
if err != nil { if err != nil {
return nil, fmt.Errorf("NewOfflineContentfulClient could not parse space export file: %v", err) return nil, fmt.Errorf("NewOfflineContentfulClient could not parse space export file: %v", err)
} }
@ -551,6 +561,7 @@ func NewOfflineContentfulClient(filename string, logFn func(fields map[string]in
Cache: &ContentfulCache{ Cache: &ContentfulCache{
contentTypes: []string{}, contentTypes: []string{},
idContentTypeMap: map[string]string{}, idContentTypeMap: map[string]string{},
genericEntries: map[string]*GenericEntry{},
parentMap: map[string][]EntryReference{}, parentMap: map[string][]EntryReference{},
}, },
cacheMutex: &ContentfulCacheMutex{ cacheMutex: &ContentfulCacheMutex{
@ -609,6 +620,22 @@ func RichTextToHtml(rt interface{}, linkResolver LinkResolverFunc, entryLinkReso
return out, nil return out, nil
} }
func (cc *ContentfulClient) GetGenericEntry(entryID string) (*GenericEntry, error) {
if cc.Cache == nil {
if cc.logFn != nil && cc.logLevel <= LogWarn {
cc.logFn(map[string]interface{}{"task": "GetGenericEntry", "entryId": entryID}, LogWarn, InfoCacheIsNil)
}
return nil, errors.New(InfoCacheIsNil)
}
if _, ok := cc.Cache.genericEntries[entryID]; !ok {
if cc.logFn != nil && cc.logLevel <= LogWarn {
cc.logFn(map[string]interface{}{"task": "GetGenericEntry", "entryId": entryID}, LogWarn, ErrorEntryNotFound)
}
return nil, errors.New(ErrorEntryNotFound)
}
return cc.Cache.genericEntries[entryID], nil
}
func (cc *ContentfulClient) SetCacheUpdateTimeout(seconds int64) { func (cc *ContentfulClient) SetCacheUpdateTimeout(seconds int64) {
cc.cacheUpdateTimeout = seconds cc.cacheUpdateTimeout = seconds
} }
@ -624,8 +651,8 @@ func (cc *ContentfulClient) SetEnvironment(environment string) {
cc.Client.Environment = environment cc.Client.Environment = environment
} }
func (cc *ContentfulClient) SetOfflineFallback(filename string) error { func (cc *ContentfulClient) SetOfflineFallback(file []byte) error {
offlineTemp, err := getOfflineSpaceFromFile(filename) offlineTemp, err := getOfflineSpaceFromFile(file)
if err != nil { if err != nil {
return err return err
} }
@ -755,30 +782,62 @@ func (cc *ContentfulClient) syncCache(ctx context.Context, contentTypes []string
entries = append(entries, entry) entries = append(entries, entry)
} }
} }
syncEntryCount := map[string]int{}
for _, entry := range entries { for _, entry := range entries {
switch entry.Sys.Type { switch entry.Sys.Type {
case sysTypeEntry: case sysTypeEntry:
if !stringSliceContains(spaceContentTypes, entry.Sys.ContentType.Sys.ID) { if !stringSliceContains(spaceContentTypes, entry.Sys.ContentType.Sys.ID) {
continue continue
} }
updateCacheForContentTypeAndEntity(ctx, cc, entry.Sys.ContentType.Sys.ID, entry.Sys.ID, entry, false) errUpdate := updateCacheForContentTypeAndEntity(ctx, cc, entry.Sys.ContentType.Sys.ID, entry.Sys.ID, entry, false)
if errUpdate != nil && cc.logFn != nil && cc.logLevel <= LogInfo {
cc.logFn(map[string]interface{}{"task": "syncCache", "error": errUpdate, "contentType": entry.Sys.ContentType.Sys.ID, "entryId": entry.Sys.ID},
LogWarn, ErrorEntryCachingFailed)
} else {
syncEntryCount[entry.Sys.ContentType.Sys.ID]++
}
case sysTypeDeletedEntry: case sysTypeDeletedEntry:
cc.cacheMutex.idContentTypeMapGcLock.RLock() cc.cacheMutex.idContentTypeMapGcLock.RLock()
contentType := cc.Cache.idContentTypeMap[entry.Sys.ID] contentType := cc.Cache.idContentTypeMap[entry.Sys.ID]
cc.cacheMutex.idContentTypeMapGcLock.RUnlock() cc.cacheMutex.idContentTypeMapGcLock.RUnlock()
updateCacheForContentTypeAndEntity(ctx, cc, contentType, entry.Sys.ID, entry, true) errUpdate := updateCacheForContentTypeAndEntity(ctx, cc, contentType, entry.Sys.ID, entry, true)
if errUpdate != nil && cc.logFn != nil && cc.logLevel <= LogInfo {
cc.logFn(map[string]interface{}{"task": "syncCache", "error": errUpdate, "op": "delete", "contentType": entry.Sys.ContentType.Sys.ID, "entryId": entry.Sys.ID},
LogWarn, ErrorEntryCachingFailed)
} else {
syncEntryCount["deletedEntry"]++
}
default: default:
} }
} }
if cc.logFn != nil && len(entries) > 0 && cc.logLevel <= LogInfo {
cc.logFn(map[string]interface{}{"task": "syncCache", "syncEntryCount": syncEntryCount}, LogInfo, InfoCacheSyncOp)
}
var syncAssetCount int
for _, asset := range assets { for _, asset := range assets {
switch asset.Sys.Type { switch asset.Sys.Type {
case sysTypeAsset: case sysTypeAsset:
updateCacheForContentTypeAndEntity(ctx, cc, assetWorkerType, asset.Sys.ID, asset, false) errUpdate := updateCacheForContentTypeAndEntity(ctx, cc, assetWorkerType, asset.Sys.ID, asset, false)
if errUpdate != nil && cc.logFn != nil && cc.logLevel <= LogInfo {
cc.logFn(map[string]interface{}{"task": "syncCache", "error": errUpdate, "assetId": asset.Sys.ID},
LogWarn, ErrorEntryCachingFailed)
} else {
syncAssetCount++
}
case sysTypeDeletedAsset: case sysTypeDeletedAsset:
updateCacheForContentTypeAndEntity(ctx, cc, assetWorkerType, asset.Sys.ID, nil, true) errUpdate := updateCacheForContentTypeAndEntity(ctx, cc, assetWorkerType, asset.Sys.ID, nil, true)
if errUpdate != nil && cc.logFn != nil && cc.logLevel <= LogInfo {
cc.logFn(map[string]interface{}{"task": "syncCache", "error": errUpdate, "op": "delete", "assetId": asset.Sys.ID},
LogWarn, ErrorEntryCachingFailed)
} else {
syncAssetCount++
}
default: default:
} }
} }
if cc.logFn != nil && len(assets) > 0 && cc.logLevel <= LogInfo {
cc.logFn(map[string]interface{}{"task": "syncCache", "syncAssetCount": syncAssetCount}, LogInfo, InfoCacheSyncOp)
}
} }
} }
@ -796,6 +855,7 @@ func (cc *ContentfulClient) cacheSpace(ctx context.Context, contentTypes []strin
tempCache := &ContentfulCache{ tempCache := &ContentfulCache{
contentTypes: contentTypes, contentTypes: contentTypes,
idContentTypeMap: map[string]string{}, idContentTypeMap: map[string]string{},
genericEntries: map[string]*GenericEntry{},
parentMap: map[string][]EntryReference{}, parentMap: map[string][]EntryReference{},
} }
if cacheAssets { if cacheAssets {
@ -877,6 +937,7 @@ func (cc *ContentfulClient) cacheSpace(ctx context.Context, contentTypes []strin
cc.cacheMutex.assetsGcLock.Lock() cc.cacheMutex.assetsGcLock.Lock()
cc.cacheMutex.idContentTypeMapGcLock.Lock() cc.cacheMutex.idContentTypeMapGcLock.Lock()
cc.cacheMutex.parentMapGcLock.Lock() cc.cacheMutex.parentMapGcLock.Lock()
cc.cacheMutex.genericEntriesGcLock.Lock()
cc.cacheMutex.brandGcLock.Lock() cc.cacheMutex.brandGcLock.Lock()
cc.cacheMutex.categoryGcLock.Lock() cc.cacheMutex.categoryGcLock.Lock()
cc.cacheMutex.productGcLock.Lock() cc.cacheMutex.productGcLock.Lock()
@ -892,6 +953,7 @@ func (cc *ContentfulClient) cacheSpace(ctx context.Context, contentTypes []strin
cc.cacheMutex.assetsGcLock.Unlock() cc.cacheMutex.assetsGcLock.Unlock()
cc.cacheMutex.sharedDataGcLock.Unlock() cc.cacheMutex.sharedDataGcLock.Unlock()
cc.cacheMutex.fullCacheGcLock.Unlock() cc.cacheMutex.fullCacheGcLock.Unlock()
cc.cacheMutex.genericEntriesGcLock.Unlock()
} }
func ToAssetReference(asset *contentful.Asset) (refSys ContentTypeSys) { func ToAssetReference(asset *contentful.Asset) (refSys ContentTypeSys) {
@ -1062,13 +1124,9 @@ func (cc *ContentfulClient) getAllAssets(tryCacheFirst bool) (map[string]*conten
return assets, nil return assets, nil
} }
func getOfflineSpaceFromFile(filename string) (*offlineTemp, error) { func getOfflineSpaceFromFile(file []byte) (*offlineTemp, error) {
fileBytes, err := os.ReadFile(filename)
if err != nil {
return nil, fmt.Errorf("getOfflineSpaceFromFile could not read space export file: %v", err)
}
offlineTemp := &offlineTemp{} offlineTemp := &offlineTemp{}
err = json.Unmarshal(fileBytes, offlineTemp) err := json.Unmarshal(file, offlineTemp)
if err != nil { if err != nil {
return nil, fmt.Errorf("getOfflineSpaceFromFile could not parse space export file: %v", err) return nil, fmt.Errorf("getOfflineSpaceFromFile could not parse space export file: %v", err)
} }
@ -1519,6 +1577,15 @@ func updateCacheForContentType(ctx context.Context, results chan ContentTypeResu
return errors.New("updateCacheForContentType failed for contentType brand: " + err.Error()) return errors.New("updateCacheForContentType failed for contentType brand: " + err.Error())
} }
tempCache.entryMaps.brand = allBrand tempCache.entryMaps.brand = allBrand
cc.cacheMutex.genericEntriesGcLock.Lock()
for _, brand := range allBrand {
tempCache.genericEntries[brand.Sys.ID] = &GenericEntry{
Sys: brand.Sys,
RawFields: brand.RawFields,
CC: brand.CC,
}
}
cc.cacheMutex.genericEntriesGcLock.Unlock()
if cc.logFn != nil && cc.logLevel <= LogInfo { if cc.logFn != nil && cc.logLevel <= LogInfo {
cc.logFn(map[string]interface{}{"contentType": "brand", "method": "updateCacheForContentType", "size": len(allBrand)}, LogInfo, InfoCachedAllEntries) cc.logFn(map[string]interface{}{"contentType": "brand", "method": "updateCacheForContentType", "size": len(allBrand)}, LogInfo, InfoCachedAllEntries)
} }
@ -1529,6 +1596,15 @@ func updateCacheForContentType(ctx context.Context, results chan ContentTypeResu
return errors.New("updateCacheForContentType failed for contentType category: " + err.Error()) return errors.New("updateCacheForContentType failed for contentType category: " + err.Error())
} }
tempCache.entryMaps.category = allCategory tempCache.entryMaps.category = allCategory
cc.cacheMutex.genericEntriesGcLock.Lock()
for _, category := range allCategory {
tempCache.genericEntries[category.Sys.ID] = &GenericEntry{
Sys: category.Sys,
RawFields: category.RawFields,
CC: category.CC,
}
}
cc.cacheMutex.genericEntriesGcLock.Unlock()
if cc.logFn != nil && cc.logLevel <= LogInfo { if cc.logFn != nil && cc.logLevel <= LogInfo {
cc.logFn(map[string]interface{}{"contentType": "category", "method": "updateCacheForContentType", "size": len(allCategory)}, LogInfo, InfoCachedAllEntries) cc.logFn(map[string]interface{}{"contentType": "category", "method": "updateCacheForContentType", "size": len(allCategory)}, LogInfo, InfoCachedAllEntries)
} }
@ -1539,6 +1615,15 @@ func updateCacheForContentType(ctx context.Context, results chan ContentTypeResu
return errors.New("updateCacheForContentType failed for contentType product: " + err.Error()) return errors.New("updateCacheForContentType failed for contentType product: " + err.Error())
} }
tempCache.entryMaps.product = allProduct tempCache.entryMaps.product = allProduct
cc.cacheMutex.genericEntriesGcLock.Lock()
for _, product := range allProduct {
tempCache.genericEntries[product.Sys.ID] = &GenericEntry{
Sys: product.Sys,
RawFields: product.RawFields,
CC: product.CC,
}
}
cc.cacheMutex.genericEntriesGcLock.Unlock()
if cc.logFn != nil && cc.logLevel <= LogInfo { if cc.logFn != nil && cc.logLevel <= LogInfo {
cc.logFn(map[string]interface{}{"contentType": "product", "method": "updateCacheForContentType", "size": len(allProduct)}, LogInfo, InfoCachedAllEntries) cc.logFn(map[string]interface{}{"contentType": "product", "method": "updateCacheForContentType", "size": len(allProduct)}, LogInfo, InfoCachedAllEntries)
} }
@ -1622,7 +1707,7 @@ func updateCacheForContentTypeAndEntity(ctx context.Context, cc *ContentfulClien
return nil return nil
} }
func commonGetParents(cc *ContentfulClient, id string, contentType []string) (parents []EntryReference, err error) { func commonGetParents(cc *ContentfulClient, id string, contentTypes []string) (parents []EntryReference, err error) {
parents = []EntryReference{} parents = []EntryReference{}
cc.cacheMutex.sharedDataGcLock.RLock() cc.cacheMutex.sharedDataGcLock.RLock()
cacheInit := cc.cacheInit cacheInit := cc.cacheInit
@ -1630,10 +1715,12 @@ func commonGetParents(cc *ContentfulClient, id string, contentType []string) (pa
if cacheInit { if cacheInit {
cc.cacheMutex.parentMapGcLock.RLock() cc.cacheMutex.parentMapGcLock.RLock()
defer cc.cacheMutex.parentMapGcLock.RUnlock() defer cc.cacheMutex.parentMapGcLock.RUnlock()
if len(contentType) != 0 { if len(contentTypes) != 0 {
for _, parent := range cc.Cache.parentMap[id] { for _, parent := range cc.Cache.parentMap[id] {
if parent.ContentType == contentType[0] { for _, contentType := range contentTypes {
parents = append(parents, parent) if parent.ContentType == contentType {
parents = append(parents, parent)
}
} }
} }
return parents, nil return parents, nil
@ -1656,7 +1743,7 @@ func commonGetParents(cc *ContentfulClient, id string, contentType []string) (pa
if err != nil { if err != nil {
return nil, err return nil, err
} }
if len(contentType) == 1 && contentType[0] != entry.Sys.ContentType.Sys.ID { if len(contentTypes) == 1 && contentTypes[0] != entry.Sys.ContentType.Sys.ID {
continue continue
} }
switch entry.Sys.ContentType.Sys.ID { switch entry.Sys.ContentType.Sys.ID {

View File

@ -1,4 +1,4 @@
// Code generated by https://github.com/foomo/gocontentful v1.0.26 - DO NOT EDIT. // Code generated by https://github.com/foomo/gocontentful v1.1.0 - DO NOT EDIT.
package testapi package testapi
import ( import (
@ -795,6 +795,8 @@ func (cc *ContentfulClient) cacheBrandByID(ctx context.Context, id string, entry
defer cc.cacheMutex.idContentTypeMapGcLock.Unlock() defer cc.cacheMutex.idContentTypeMapGcLock.Unlock()
cc.cacheMutex.parentMapGcLock.Lock() cc.cacheMutex.parentMapGcLock.Lock()
defer cc.cacheMutex.parentMapGcLock.Unlock() defer cc.cacheMutex.parentMapGcLock.Unlock()
cc.cacheMutex.genericEntriesGcLock.Lock()
defer cc.cacheMutex.genericEntriesGcLock.Unlock()
var col *contentful.Collection var col *contentful.Collection
if entryPayload != nil { if entryPayload != nil {
@ -817,6 +819,7 @@ func (cc *ContentfulClient) cacheBrandByID(ctx context.Context, id string, entry
} }
// It was deleted // It was deleted
if col != nil && len(col.Items) == 0 || entryDelete { if col != nil && len(col.Items) == 0 || entryDelete {
delete(cc.Cache.genericEntries, id)
delete(cc.Cache.entryMaps.brand, id) delete(cc.Cache.entryMaps.brand, id)
delete(cc.Cache.idContentTypeMap, id) delete(cc.Cache.idContentTypeMap, id)
// delete as child // delete as child
@ -842,6 +845,11 @@ func (cc *ContentfulClient) cacheBrandByID(ctx context.Context, id string, entry
cc.Cache.entryMaps.brand = map[string]*CfBrand{} cc.Cache.entryMaps.brand = map[string]*CfBrand{}
} }
cc.Cache.entryMaps.brand[id] = brand cc.Cache.entryMaps.brand[id] = brand
cc.Cache.genericEntries[id] = &GenericEntry{
Sys: brand.Sys,
RawFields: brand.RawFields,
CC: brand.CC,
}
cc.Cache.idContentTypeMap[id] = brand.Sys.ContentType.Sys.ID cc.Cache.idContentTypeMap[id] = brand.Sys.ContentType.Sys.ID
allChildrensIds := map[string]bool{} allChildrensIds := map[string]bool{}

View File

@ -1,4 +1,4 @@
// Code generated by https://github.com/foomo/gocontentful v1.0.26 - DO NOT EDIT. // Code generated by https://github.com/foomo/gocontentful v1.1.0 - DO NOT EDIT.
package testapi package testapi
import ( import (
@ -559,6 +559,8 @@ func (cc *ContentfulClient) cacheCategoryByID(ctx context.Context, id string, en
defer cc.cacheMutex.idContentTypeMapGcLock.Unlock() defer cc.cacheMutex.idContentTypeMapGcLock.Unlock()
cc.cacheMutex.parentMapGcLock.Lock() cc.cacheMutex.parentMapGcLock.Lock()
defer cc.cacheMutex.parentMapGcLock.Unlock() defer cc.cacheMutex.parentMapGcLock.Unlock()
cc.cacheMutex.genericEntriesGcLock.Lock()
defer cc.cacheMutex.genericEntriesGcLock.Unlock()
var col *contentful.Collection var col *contentful.Collection
if entryPayload != nil { if entryPayload != nil {
@ -581,6 +583,7 @@ func (cc *ContentfulClient) cacheCategoryByID(ctx context.Context, id string, en
} }
// It was deleted // It was deleted
if col != nil && len(col.Items) == 0 || entryDelete { if col != nil && len(col.Items) == 0 || entryDelete {
delete(cc.Cache.genericEntries, id)
delete(cc.Cache.entryMaps.category, id) delete(cc.Cache.entryMaps.category, id)
delete(cc.Cache.idContentTypeMap, id) delete(cc.Cache.idContentTypeMap, id)
// delete as child // delete as child
@ -606,6 +609,11 @@ func (cc *ContentfulClient) cacheCategoryByID(ctx context.Context, id string, en
cc.Cache.entryMaps.category = map[string]*CfCategory{} cc.Cache.entryMaps.category = map[string]*CfCategory{}
} }
cc.Cache.entryMaps.category[id] = category cc.Cache.entryMaps.category[id] = category
cc.Cache.genericEntries[id] = &GenericEntry{
Sys: category.Sys,
RawFields: category.RawFields,
CC: category.CC,
}
cc.Cache.idContentTypeMap[id] = category.Sys.ContentType.Sys.ID cc.Cache.idContentTypeMap[id] = category.Sys.ContentType.Sys.ID
allChildrensIds := map[string]bool{} allChildrensIds := map[string]bool{}

View File

@ -1,4 +1,4 @@
// Code generated by https://github.com/foomo/gocontentful v1.0.26 - DO NOT EDIT. // Code generated by https://github.com/foomo/gocontentful v1.1.0 - DO NOT EDIT.
package testapi package testapi
import ( import (
@ -1321,6 +1321,8 @@ func (cc *ContentfulClient) cacheProductByID(ctx context.Context, id string, ent
defer cc.cacheMutex.idContentTypeMapGcLock.Unlock() defer cc.cacheMutex.idContentTypeMapGcLock.Unlock()
cc.cacheMutex.parentMapGcLock.Lock() cc.cacheMutex.parentMapGcLock.Lock()
defer cc.cacheMutex.parentMapGcLock.Unlock() defer cc.cacheMutex.parentMapGcLock.Unlock()
cc.cacheMutex.genericEntriesGcLock.Lock()
defer cc.cacheMutex.genericEntriesGcLock.Unlock()
var col *contentful.Collection var col *contentful.Collection
if entryPayload != nil { if entryPayload != nil {
@ -1343,6 +1345,7 @@ func (cc *ContentfulClient) cacheProductByID(ctx context.Context, id string, ent
} }
// It was deleted // It was deleted
if col != nil && len(col.Items) == 0 || entryDelete { if col != nil && len(col.Items) == 0 || entryDelete {
delete(cc.Cache.genericEntries, id)
delete(cc.Cache.entryMaps.product, id) delete(cc.Cache.entryMaps.product, id)
delete(cc.Cache.idContentTypeMap, id) delete(cc.Cache.idContentTypeMap, id)
// delete as child // delete as child
@ -1368,6 +1371,11 @@ func (cc *ContentfulClient) cacheProductByID(ctx context.Context, id string, ent
cc.Cache.entryMaps.product = map[string]*CfProduct{} cc.Cache.entryMaps.product = map[string]*CfProduct{}
} }
cc.Cache.entryMaps.product[id] = product cc.Cache.entryMaps.product[id] = product
cc.Cache.genericEntries[id] = &GenericEntry{
Sys: product.Sys,
RawFields: product.RawFields,
CC: product.CC,
}
cc.Cache.idContentTypeMap[id] = product.Sys.ContentType.Sys.ID cc.Cache.idContentTypeMap[id] = product.Sys.ContentType.Sys.ID
allChildrensIds := map[string]bool{} allChildrensIds := map[string]bool{}

View File

@ -1,6 +1,9 @@
package test package test
import ( import (
"fmt"
"os"
"github.com/foomo/gocontentful/test/testapi" "github.com/foomo/gocontentful/test/testapi"
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
) )
@ -15,7 +18,19 @@ const (
var testLogger = logrus.StandardLogger() var testLogger = logrus.StandardLogger()
func getTestClient() (*testapi.ContentfulClient, error) { func getTestClient() (*testapi.ContentfulClient, error) {
return testapi.NewOfflineContentfulClient("./test-space-export.json", GetContenfulLogger(testLogger), LogDebug, true, true) testFile, err := GetTestFile("./test-space-export.json")
if err != nil {
return nil, fmt.Errorf("getTestClient could not read space export file: %v", err)
}
return testapi.NewOfflineContentfulClient(testFile, GetContenfulLogger(testLogger), LogDebug, true, true)
}
func GetTestFile(filename string) ([]byte, error) {
fileBytes, err := os.ReadFile(filename)
if err != nil {
return nil, fmt.Errorf("getTestFile could not read space export file: %v", err)
}
return fileBytes, nil
} }
func GetContenfulLogger(log *logrus.Logger) func(fields map[string]interface{}, level int, args ...interface{}) { func GetContenfulLogger(log *logrus.Logger) func(fields map[string]interface{}, level int, args ...interface{}) {