foomo/gocontentful
1
0
Fork 0
mirror of https://github.com/foomo/gocontentful.git synced 2025-10-16 12:25:39 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

feat: non-concurrent cache updates

Browse Source
This commit is contained in:
Cristian Vidmar 2022-05-17 12:16:24 +02:00
parent 62b99e2013
commit ea560cfdc4
15 changed files with 387 additions and 269 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
erm/space.go
View File

@ -22,6 +22,7 @@ type spaceConf struct {
Locales []Locale
ContentTypes []ContentType
ContentType ContentType
Version string
}
// GetLocales retrieves locale definition from Contentful
@ -127,7 +128,7 @@ func getData(spaceID, cmaKey, environment, exportFile string, flagContentTypes [
}
// GenerateAPI calls the generators
func GenerateAPI(dir, packageName, spaceID, cmaKey, environment, exportFile string, flagContentTypes []string) (err error) {
func GenerateAPI(dir, packageName, spaceID, cmaKey, environment, exportFile string, flagContentTypes []string, version string) (err error) {
contentTypes, locales, errGetData := getData(spaceID, cmaKey, environment, exportFile, flagContentTypes)
if errGetData != nil {
return errGetData
@ -145,6 +146,7 @@ func GenerateAPI(dir, packageName, spaceID, cmaKey, environment, exportFile stri
PackageName: packageName,
Locales: locales,
ContentTypes: contentTypes,
Version: version,
}
return generateCode(conf)
}

2
erm/templates/contentful_vo.gotmpl
View File

@ -1,4 +1,4 @@
// Package {{ .PackageName }} - DO NOT EDIT THIS FILE: Auto-generated code by https://github.com/foomo/gocontentful
// Package {{ .PackageName }} - DO NOT EDIT THIS FILE: Auto-generated code by https://github.com/foomo/gocontentful {{ .Version }}
{{ $cfg := . }}{{ $contentTypes := .ContentTypes }}package {{ .PackageName }}
import "github.com/foomo/contentful"

2
erm/templates/contentful_vo_base.gotmpl
View File

@ -1,4 +1,4 @@
// Package {{ .PackageName }} - DO NOT EDIT THIS FILE: Auto-generated code by https://github.com/foomo/gocontentful
// Package {{ .PackageName }} - DO NOT EDIT THIS FILE: Auto-generated code by https://github.com/foomo/gocontentful {{ .Version }}
{{ $cfg := . }}package {{ .PackageName }}
import "github.com/foomo/contentful"

315
erm/templates/contentful_vo_lib.gotmpl
View File

@ -1,4 +1,4 @@
// Package {{ .PackageName }} - DO NOT EDIT THIS FILE: Auto-generated code by https://github.com/foomo/gocontentful
// Package {{ .PackageName }} - DO NOT EDIT THIS FILE: Auto-generated code by https://github.com/foomo/gocontentful {{ .Version }}
{{ $cfg := . }}{{ $contentTypes := .ContentTypes }}{{ $locales := .Locales }}package {{ .PackageName }}
import (
@ -46,15 +46,19 @@ type ContentfulCacheMutex struct {
type assetCacheMap map[string]*contentful.Asset
type ContentfulClient struct {
Cache *ContentfulCache
CacheMutex *ContentfulCacheMutex
clientMode ClientMode
Client *contentful.Contentful
locales []Locale
logFn func(
fields map[string]interface{},
level int,
args ...interface{},
Cache *ContentfulCache
cacheInit bool
cacheMutex *ContentfulCacheMutex
cacheQueue chan struct{}
cacheDone chan struct{}
cacheWorkerOnce sync.Once
clientMode ClientMode
Client *contentful.Contentful
locales []Locale
logFn func(
fields map[string]interface{},
level int,
args ...interface{},
)
logLevel int
optimisticPageSize uint16 // Start downloading entries at this page size
@ -144,8 +148,12 @@ var (
InfoUpdatedEntityCache = "updated cache for entity"
InfoCachedAllEntries = "cached all entries of content type"
InfoCachedAllAssets = "cached all assets"
InfoFallingBackToFile = "could not download space data from Contentful, falling back to local file"
InfoFallingBackToFile = "gonna use a local file"
InfoLoadingFromFile = "loading space from local file"
InfoCacheWorkerStart = "cache worker starting"
InfoCacheUpdateQueued = "cache update queued"
InfoCacheUpdateDone = "cache update returning"
InfoCacheUpdateSkipped = "cache update skipped, already one in the queue"
InfoOfflineEntitiesLoaded = "downloaded entries and assets from offline file"
InfoPreservingExistingCache = "could not connect for cache update, preserving the existing cache"
InfoUpdateCacheTime = "space caching done, time recorded"
@ -160,13 +168,13 @@ var SpaceContentTypeInfoMap = ContentTypeInfoMap{ {{ range $index , $contentType
},{{ end }}
}
func (cc *ContentfulClient) BrokenReferences() (brokenReferences []BrokenReference) {
if cc.Cache == nil || cc.CacheMutex == nil {
if cc.Cache == nil || cc.cacheMutex == nil {
return
}
cc.CacheMutex.parentMapGcLock.Lock()
defer cc.CacheMutex.parentMapGcLock.Unlock()
cc.CacheMutex.idContentTypeMapGcLock.Lock()
defer cc.CacheMutex.idContentTypeMapGcLock.Unlock()
cc.cacheMutex.parentMapGcLock.Lock()
defer cc.cacheMutex.parentMapGcLock.Unlock()
cc.cacheMutex.idContentTypeMapGcLock.Lock()
defer cc.cacheMutex.idContentTypeMapGcLock.Unlock()
for childID, parents := range cc.Cache.parentMap {
if _, okGotEntry := cc.Cache.idContentTypeMap[childID]; !okGotEntry {
for _, parent := range parents {
@ -200,19 +208,15 @@ func (cc *ContentfulClient) ClientStats() {
return
}
if cc.logFn != nil {
cached := false
if cc.Cache != nil {
cached = true
}
fieldsMap := map[string]interface{}{
"space ID": cc.SpaceID,
"environment": cc.Client.Environment,
"clientMode": cc.clientMode,
"contentTypes": strings.Join(cc.Cache.contentTypes, ","),
"locales": cc.locales,
"cached": cached,
"cached": cc.cacheInit,
}
if cached {
if cc.cacheInit {
fieldsMap["cache asset count"] = len(cc.Cache.assets)
fieldsMap["cache entry count"] = len(cc.Cache.idContentTypeMap)
fieldsMap["cache parentMap length"] = len(cc.Cache.parentMap)
@ -255,10 +259,10 @@ func (cc *ContentfulClient) GetAssetByID(id string, forceNoCache ...bool) (*cont
if cc == nil || cc.Client == nil {
return nil, errors.New("GetAssetByID: No client available")
}
if cc.Cache != nil && cc.Cache.assets != nil && (len(forceNoCache) == 0 || !forceNoCache[0]) {
cc.CacheMutex.assetsGcLock.Lock()
if cc.cacheInit && cc.Cache.assets != nil && (len(forceNoCache) == 0 || !forceNoCache[0]) {
cc.cacheMutex.assetsGcLock.Lock()
asset, okAsset := cc.Cache.assets[id]
cc.CacheMutex.assetsGcLock.Unlock()
cc.cacheMutex.assetsGcLock.Unlock()
if okAsset {
return asset, nil
}
@ -294,12 +298,12 @@ func (cc *ContentfulClient) GetContentTypeOfID(id string) (string, error) {
if cc == nil || cc.Client == nil {
return "", errors.New("GetContentTypeOfID: No client available")
}
if cc.Cache != nil && cc.CacheMutex != nil {
if cc.cacheInit {
okVo := false
{{ range $index , $contentType := $contentTypes }}
cc.CacheMutex.{{ $contentType.Sys.ID }}GcLock.Lock()
cc.cacheMutex.{{ $contentType.Sys.ID }}GcLock.Lock()
_, okVo = cc.Cache.entryMaps.{{ $contentType.Sys.ID }}[id]
cc.CacheMutex.{{ $contentType.Sys.ID }}GcLock.Unlock()
cc.cacheMutex.{{ $contentType.Sys.ID }}GcLock.Unlock()
if okVo {
return ContentType{{ firstCap $contentType.Sys.ID }}, nil
}
@ -424,6 +428,22 @@ func NewContentfulClient(spaceID string, clientMode ClientMode, clientKey string
cc := &ContentfulClient{
clientMode: clientMode,
Client: apiClient,
Cache: &ContentfulCache{
contentTypes: []string{},
idContentTypeMap: map[string]string{},
parentMap: map[string][]EntryReference{},
},
cacheMutex: &ContentfulCacheMutex{
fullCacheGcLock: sync.RWMutex{},
sharedDataGcLock: sync.RWMutex{},
assetsGcLock: sync.RWMutex{},
idContentTypeMapGcLock: sync.RWMutex{},
parentMapGcLock: sync.RWMutex{},
{{ range $index , $contentType := $contentTypes }} {{ $contentType.Sys.ID }}GcLock: sync.RWMutex{},
{{ end }}
},
cacheQueue: make(chan struct{}, 1),
cacheDone: make(chan struct{}, 1),
locales: []Locale{ {{ range $index , $locale := $locales }}SpaceLocale{{ onlyLetters $locale.Name }}, {{ end }} },
logFn: logFn,
logLevel: logLevel,
@ -449,6 +469,22 @@ func NewOfflineContentfulClient(filename string, logFn func(fields map[string]in
cc := &ContentfulClient{
clientMode: ClientModeCDA,
Client: contentful.NewCDA(""),
Cache: &ContentfulCache{
contentTypes: []string{},
idContentTypeMap: map[string]string{},
parentMap: map[string][]EntryReference{},
},
cacheMutex: &ContentfulCacheMutex{
fullCacheGcLock: sync.RWMutex{},
sharedDataGcLock: sync.RWMutex{},
assetsGcLock: sync.RWMutex{},
idContentTypeMapGcLock: sync.RWMutex{},
parentMapGcLock: sync.RWMutex{},
{{ range $index , $contentType := $contentTypes }} {{ $contentType.Sys.ID }}GcLock: sync.RWMutex{},
{{ end }}
},
cacheQueue: make(chan struct{}, 1),
cacheDone: make(chan struct{}, 1),
locales: []Locale{
{{ range $index , $locale := $locales }}SpaceLocale{{ onlyLetters $locale.Name }},
{{ end}}},
@ -505,7 +541,6 @@ func (cc *ContentfulClient) SetOfflineFallback(filename string) error {
}
func (cc *ContentfulClient) UpdateCache(ctx context.Context, contentTypes []string, cacheAssets bool) error {
start := time.Now()
if contentTypes == nil {
contentTypes = spaceContentTypes
} else {
@ -515,101 +550,125 @@ func (cc *ContentfulClient) UpdateCache(ctx context.Context, contentTypes []stri
}
}
}
if cc.CacheMutex == nil {
cc.CacheMutex = &ContentfulCacheMutex{
fullCacheGcLock: sync.RWMutex{},
sharedDataGcLock: sync.RWMutex{},
assetsGcLock: sync.RWMutex{},
idContentTypeMapGcLock: sync.RWMutex{},
parentMapGcLock: sync.RWMutex{},
{{ range $index , $contentType := $contentTypes }} {{ $contentType.Sys.ID }}GcLock: sync.RWMutex{},
{{ end }}
}
cc.cacheWorkerOnce.Do(func() {
go cc.cacheWorker(ctx, contentTypes, cacheAssets)
})
if len(cc.cacheQueue) == 0 {
if cc.logFn != nil && cc.logLevel <= LogInfo {
cc.logFn(map[string]interface{}{"task":"UpdateCache"}, LogInfo, InfoCacheUpdateQueued)
}
cc.cacheQueue <- struct{}{}
<-cc.cacheDone
cc.cacheInit = true
if cc.logFn != nil && cc.logLevel <= LogInfo {
cc.logFn(map[string]interface{}{"task":"UpdateCache"}, LogInfo, InfoCacheUpdateDone)
}
return nil
}
tempCache := &ContentfulCache{
contentTypes: contentTypes,
idContentTypeMap: map[string]string{},
parentMap: map[string][]EntryReference{},
}
if cacheAssets {
contentTypes = append([]string{assetWorkerType}, contentTypes...)
}
_, errCanWeEvenConnect := cc.Client.Spaces.Get(cc.SpaceID)
cc.CacheMutex.sharedDataGcLock.RLock()
offlinePreviousState := cc.offline
cc.CacheMutex.sharedDataGcLock.RUnlock()
if errCanWeEvenConnect != nil {
if len(cc.offlineTemp.Entries) > 0 && (cc.Cache == nil || offlinePreviousState == true) {
if cc.logFn != nil && cc.logLevel <= LogInfo {
cc.logFn(map[string]interface{}{"method": "UpdateCache"}, LogInfo, InfoFallingBackToFile)
}
cc.CacheMutex.sharedDataGcLock.Lock()
cc.offline = true
cc.CacheMutex.sharedDataGcLock.Unlock()
} else {
if cc.logFn != nil && cc.logLevel <= LogInfo {
cc.logFn(map[string]interface{}{"method": "UpdateCache"}, LogInfo, InfoPreservingExistingCache)
}
}
}
results := make(chan ContentTypeResult, 16)
resultsDone := make(chan struct{})
contentTypeChan := make(chan string)
group, gctx := errgroup.WithContext(ctx)
for i := 0; i < cacheUpdateConcurrency; i++ {
group.Go(func() error {
for contentType := range contentTypeChan {
err := updateCacheForContentType(gctx, results, cc, tempCache, contentType)
if err != nil {
if cc.logFn != nil && cc.logLevel <= LogInfo {
cc.logFn(map[string]interface{}{"method": "UpdateCache", "contentType": contentType}, LogError, err.Error())
}
return err
}
}
return nil
})
}
go func() {
for _, contentType := range contentTypes {
contentTypeChan <- contentType
}
close(contentTypeChan)
}()
go func() {
for res := range results {
tempCache.idContentTypeMap[res.EntryID] = res.ContentType
for childID, references := range res.References {
tempCache.parentMap[childID] = append(tempCache.parentMap[childID], references...)
}
}
resultsDone <- struct{}{}
}()
err := group.Wait()
close(results)
if err != nil {
// drain contentTypeChan
for _ = range contentTypeChan { }
cc.CacheMutex.sharedDataGcLock.Lock()
cc.offline = offlinePreviousState
cc.CacheMutex.sharedDataGcLock.Unlock()
return err
}
// Signal that the cache build is done
<-resultsDone
if cc.logFn != nil && cc.logLevel <= LogInfo {
cc.logFn(map[string]interface{}{"time elapsed":fmt.Sprint(time.Since(start)), "method": "UpdateCache"}, LogInfo, InfoUpdateCacheTime)
}
cc.CacheMutex.fullCacheGcLock.Lock()
defer cc.CacheMutex.fullCacheGcLock.Unlock()
cc.Cache = tempCache
cc.CacheMutex.sharedDataGcLock.Lock()
cc.offline = offlinePreviousState
cc.CacheMutex.sharedDataGcLock.Unlock()
if cc.logFn != nil && cc.logLevel <= LogInfo {
cc.logFn(map[string]interface{}{"task":"UpdateCache"}, LogInfo, InfoCacheUpdateSkipped)
}
return nil
}
func (cc *ContentfulClient) cacheWorker(ctx context.Context, contentTypes []string, cacheAssets bool) {
if cc.logFn != nil && cc.logLevel <= LogInfo {
cc.logFn(map[string]interface{}{"task":"UpdateCache"}, LogInfo, InfoCacheWorkerStart)
}
for _ = range cc.cacheQueue {
cc.cacheSpace(ctx, contentTypes, cacheAssets)
cc.cacheDone <- struct{}{}
}
}
func (cc *ContentfulClient) cacheSpace(ctx context.Context, contentTypes []string, cacheAssets bool) {
start := time.Now()
tempCache := &ContentfulCache{
contentTypes: contentTypes,
idContentTypeMap: map[string]string{},
parentMap: map[string][]EntryReference{},
}
if cacheAssets {
contentTypes = append([]string{assetWorkerType}, contentTypes...)
}
_, errCanWeEvenConnect := cc.Client.Spaces.Get(cc.SpaceID)
cc.cacheMutex.sharedDataGcLock.RLock()
offlinePreviousState := cc.offline
cc.cacheMutex.sharedDataGcLock.RUnlock()
if errCanWeEvenConnect != nil {
if len(cc.offlineTemp.Entries) > 0 && (cc.Cache == nil || offlinePreviousState == true) {
if cc.logFn != nil && cc.logLevel <= LogInfo {
cc.logFn(map[string]interface{}{"task": "UpdateCache"}, LogInfo, InfoFallingBackToFile)
}
cc.cacheMutex.sharedDataGcLock.Lock()
cc.offline = true
cc.cacheMutex.sharedDataGcLock.Unlock()
} else {
if cc.logFn != nil && cc.logLevel <= LogInfo {
cc.logFn(map[string]interface{}{"task": "UpdateCache"}, LogInfo, InfoPreservingExistingCache)
}
}
}
results := make(chan ContentTypeResult, 16)
resultsDone := make(chan struct{})
contentTypeChan := make(chan string)
group, gctx := errgroup.WithContext(ctx)
for i := 0; i < cacheUpdateConcurrency; i++ {
group.Go(func() error {
for contentType := range contentTypeChan {
err := updateCacheForContentType(gctx, results, cc, tempCache, contentType)
if err != nil {
if cc.logFn != nil && cc.logLevel <= LogInfo {
cc.logFn(map[string]interface{}{"task": "UpdateCache", "contentType": contentType}, LogError, err.Error())
}
return err
}
}
return nil
})
}
go func() {
for _, contentType := range contentTypes {
contentTypeChan <- contentType
}
close(contentTypeChan)
}()
go func() {
for res := range results {
tempCache.idContentTypeMap[res.EntryID] = res.ContentType
for childID, references := range res.References {
tempCache.parentMap[childID] = append(tempCache.parentMap[childID], references...)
}
}
resultsDone <- struct{}{}
}()
err := group.Wait()
close(results)
if err != nil {
// drain contentTypeChan
for _ = range contentTypeChan {
}
cc.cacheMutex.sharedDataGcLock.Lock()
cc.offline = offlinePreviousState
cc.cacheMutex.sharedDataGcLock.Unlock()
if cc.logFn != nil && cc.logLevel <= LogInfo {
cc.logFn(map[string]interface{}{"task": "UpdateCache"}, LogError, err.Error())
}
return
}
// Signal that the cache build is done
<-resultsDone
if cc.logFn != nil && cc.logLevel <= LogInfo {
cc.logFn(map[string]interface{}{"time elapsed": fmt.Sprint(time.Since(start)), "task": "UpdateCache"}, LogInfo, InfoUpdateCacheTime)
}
cc.cacheMutex.fullCacheGcLock.Lock()
defer cc.cacheMutex.fullCacheGcLock.Unlock()
cc.Cache = tempCache
cc.cacheMutex.sharedDataGcLock.Lock()
cc.offline = offlinePreviousState
cc.cacheMutex.sharedDataGcLock.Unlock()
}
func ToAssetReference(asset *contentful.Asset) (refSys ContentTypeSys) {
refSys.Sys.ID = asset.Sys.ID
refSys.Sys.Type = FieldTypeLink
@ -674,23 +733,23 @@ func (cc *ContentfulClient) cacheGcAssetByID(ctx context.Context, id string) err
asset.Fields.File[string(loc)].URL = "https:" + asset.Fields.File[string(loc)].URL
}
}
cc.CacheMutex.assetsGcLock.Lock()
cc.cacheMutex.assetsGcLock.Lock()
cc.Cache.assets[id] = &asset
cc.CacheMutex.assetsGcLock.Unlock()
cc.cacheMutex.assetsGcLock.Unlock()
return nil
}
func (cc *ContentfulClient) deleteAssetFromCache(key string) error {
if cc.Cache == nil || cc.CacheMutex == nil {
if cc.Cache == nil || cc.cacheMutex == nil {
return errors.New("no cache available")
}
cc.CacheMutex.assetsGcLock.Lock()
cc.cacheMutex.assetsGcLock.Lock()
if _, ok := cc.Cache.assets[key]; ok {
delete(cc.Cache.assets, key)
cc.CacheMutex.assetsGcLock.Unlock()
cc.cacheMutex.assetsGcLock.Unlock()
return nil
}
cc.CacheMutex.assetsGcLock.Unlock()
cc.cacheMutex.assetsGcLock.Unlock()
return errors.New("asset not found in cache, could not delete")
}
@ -723,7 +782,7 @@ func (cc *ContentfulClient) getAllAssets(tryCacheFirst bool) (map[string]*conten
if cc == nil || cc.Client == nil {
return nil, errors.New("getAllAssets: No client available")
}
if cc.Cache != nil && cc.Cache.assets != nil && tryCacheFirst {
if cc.cacheInit && cc.Cache.assets != nil && tryCacheFirst {
return cc.Cache.assets, nil
}
allItems := []interface{}{}
@ -1256,7 +1315,7 @@ func updateCacheForContentTypeAndEntity(ctx context.Context, cc *ContentfulClien
func commonGetParents(cc *ContentfulClient, id string, contentType []string) (parents []EntryReference, err error) {
parents = []EntryReference{}
if cc.Cache != nil {
if cc.cacheInit {
if len(contentType) != 0 {
for _, parent := range cc.Cache.parentMap[id] {
if parent.ContentType == contentType[0] {

28
erm/templates/contentful_vo_lib_contenttype.gotmpl
View File

@ -1,4 +1,4 @@
// Package {{ .PackageName }} - DO NOT EDIT THIS FILE: Auto-generated code by https://github.com/foomo/gocontentful
// Package {{ .PackageName }} - DO NOT EDIT THIS FILE: Auto-generated code by https://github.com/foomo/gocontentful {{ .Version }}
{{ $cfg := . }}{{ $contentTypes := .ContentTypes }}{{ $contentType := .ContentType }}package {{ .PackageName }}
import (
@ -20,7 +20,7 @@ func (cc *ContentfulClient) GetAll{{ firstCap $contentType.Sys.ID }}() (voMap ma
if cc == nil {
return nil, errors.New("GetAll{{ firstCap $contentType.Sys.ID }}: No client available")
}
if cc.Cache != nil {
if cc.cacheInit {
return cc.Cache.entryMaps.{{ $contentType.Sys.ID }}, nil
}
col, err := cc.optimisticPageSizeGetAll("{{ $contentType.Sys.ID }}", cc.optimisticPageSize)
@ -66,9 +66,9 @@ func (cc *ContentfulClient) Get{{ firstCap $contentType.Sys.ID }}ByID(id string,
if cc == nil || cc.Client == nil {
return nil, errors.New("Get{{ firstCap $contentType.Sys.ID }}ByID: No client available")
}
if cc.Cache != nil && cc.CacheMutex != nil && (len(forceNoCache) == 0 || !forceNoCache[0]) {
cc.CacheMutex.{{ $contentType.Sys.ID }}GcLock.RLock()
defer cc.CacheMutex.{{ $contentType.Sys.ID }}GcLock.RUnlock()
if cc.cacheInit && (len(forceNoCache) == 0 || !forceNoCache[0]) {
cc.cacheMutex.{{ $contentType.Sys.ID }}GcLock.RLock()
defer cc.cacheMutex.{{ $contentType.Sys.ID }}GcLock.RUnlock()
vo, ok := cc.Cache.entryMaps.{{ $contentType.Sys.ID }}[id]
if ok {
return vo, nil
@ -631,7 +631,7 @@ func (cc *ContentfulClient) cacheAll{{ firstCap $contentType.Sys.ID }}(ctx conte
col := &contentful.Collection{
Items: []interface{}{},
}
cc.CacheMutex.sharedDataGcLock.RLock()
cc.cacheMutex.sharedDataGcLock.RLock()
if cc.offline {
for _, entry := range cc.offlineTemp.Entries {
if entry.Sys.ContentType.Sys.ID == ContentType{{ firstCap $contentType.Sys.ID }} {
@ -644,14 +644,14 @@ func (cc *ContentfulClient) cacheAll{{ firstCap $contentType.Sys.ID }}(ctx conte
return nil, err
}
}
cc.CacheMutex.sharedDataGcLock.RUnlock()
cc.cacheMutex.sharedDataGcLock.RUnlock()
all{{ firstCap $contentType.Sys.ID }}, err = colToCf{{ firstCap $contentType.Sys.ID }}(col,cc)
if err != nil {
return nil, err
}
{{ $contentType.Sys.ID }}Map := map[string]*Cf{{ firstCap $contentType.Sys.ID }}{}
for _, {{ $contentType.Sys.ID }} := range all{{ firstCap $contentType.Sys.ID }} {
if cc.Cache != nil {
if cc.cacheInit {
existing{{ firstCap $contentType.Sys.ID }}, err := cc.Get{{ firstCap $contentType.Sys.ID }}ByID({{ $contentType.Sys.ID }}.Sys.ID)
if err == nil && existing{{ firstCap $contentType.Sys.ID }} != nil && existing{{ firstCap $contentType.Sys.ID }}.Sys.Version > {{ $contentType.Sys.ID }}.Sys.Version {
return nil, fmt.Errorf("cache update canceled because {{ firstCap $contentType.Sys.ID }} entry %s is newer in cache", {{ $contentType.Sys.ID }}.Sys.ID)
@ -716,12 +716,12 @@ func (cc *ContentfulClient) cache{{ firstCap $contentType.Sys.ID }}ByID(ctx cont
if err != nil {
return err
}
cc.CacheMutex.{{ $contentType.Sys.ID }}GcLock.Lock()
defer cc.CacheMutex.{{ $contentType.Sys.ID }}GcLock.Unlock()
cc.CacheMutex.idContentTypeMapGcLock.Lock()
defer cc.CacheMutex.idContentTypeMapGcLock.Unlock()
cc.CacheMutex.parentMapGcLock.Lock()
defer cc.CacheMutex.parentMapGcLock.Unlock()
cc.cacheMutex.{{ $contentType.Sys.ID }}GcLock.Lock()
defer cc.cacheMutex.{{ $contentType.Sys.ID }}GcLock.Unlock()
cc.cacheMutex.idContentTypeMapGcLock.Lock()
defer cc.cacheMutex.idContentTypeMapGcLock.Unlock()
cc.cacheMutex.parentMapGcLock.Lock()
defer cc.cacheMutex.parentMapGcLock.Unlock()
// It was deleted
if len(col.Items) == 0 {
delete(cc.Cache.entryMaps.{{ $contentType.Sys.ID }}, id)

4
main.go
View File

@ -14,7 +14,7 @@ import (
"github.com/foomo/gocontentful/erm"
)
var VERSION = "v1.0.10"
var VERSION = "v1.0.11"
type contentfulRc struct {
ManagementToken string `json:"managementToken"`
@ -108,7 +108,7 @@ func main() {
}
}
err = erm.GenerateAPI(filepath.Dir(path), packageName, *flagSpaceID, cmaKey, *flagEnvironment, *flagGenerateFromExport, flagContentTypesSlice)
err = erm.GenerateAPI(filepath.Dir(path), packageName, *flagSpaceID, cmaKey, *flagEnvironment, *flagGenerateFromExport, flagContentTypesSlice, VERSION)
if err != nil {
fatal("Something went horribly wrong...", err)
}

2
test/cache_test.go
View File

@ -84,7 +84,7 @@ func TestPreserveCacheIfNewer(t *testing.T) {
err = contentfulClient.SetOfflineFallback("./test-space-export-older.json")
require.NoError(t, err)
err = contentfulClient.UpdateCache(context.TODO(), nil, false)
require.Error(t, err)
require.NoError(t, err)
brand, err := contentfulClient.GetBrandByID("JrePkDVYomE8AwcuCUyMi")
require.NoError(t, err)
require.Equal(t, 2.0, brand.Sys.Version)

2
test/other_test.go
View File

@ -4,11 +4,13 @@ import (
"github.com/foomo/gocontentful/test/testapi"
"github.com/stretchr/testify/require"
"testing"
"time"
)
func TestPublishingStatus(t *testing.T) {
contentfulClient, err := getTestClient()
require.NoError(t, err)
time.Sleep(time.Second)
draft, err := contentfulClient.GetProductByID("6dbjWqNd9SqccegcqYq224")
require.NoError(t, err)
require.Equal(t, testapi.StatusDraft, draft.GetPublishingStatus())

12
test/testapi/apiextensions.go
View File

@ -25,12 +25,12 @@ func (cc *ContentfulClient) SetProductInCache(product *CfProduct) {
if cc.Cache == nil {
return
}
cc.CacheMutex.productGcLock.Lock()
defer cc.CacheMutex.productGcLock.Unlock()
cc.CacheMutex.idContentTypeMapGcLock.Lock()
defer cc.CacheMutex.idContentTypeMapGcLock.Unlock()
cc.CacheMutex.parentMapGcLock.Lock()
defer cc.CacheMutex.parentMapGcLock.Unlock()
cc.cacheMutex.productGcLock.Lock()
defer cc.cacheMutex.productGcLock.Unlock()
cc.cacheMutex.idContentTypeMapGcLock.Lock()
defer cc.cacheMutex.idContentTypeMapGcLock.Unlock()
cc.cacheMutex.parentMapGcLock.Lock()
defer cc.cacheMutex.parentMapGcLock.Unlock()
cc.Cache.entryMaps.product[product.Sys.ID] = product
cc.Cache.idContentTypeMap[product.Sys.ID] = product.Sys.ContentType.Sys.ID
return

2
test/testapi/gocontentfulvo.go
View File

@ -1,4 +1,4 @@
// Package testapi - DO NOT EDIT THIS FILE: Auto-generated code by https://github.com/foomo/gocontentful
// Package testapi - DO NOT EDIT THIS FILE: Auto-generated code by https://github.com/foomo/gocontentful v1.0.11
package testapi
import (

2
test/testapi/gocontentfulvobase.go
View File

@ -1,4 +1,4 @@
// Package testapi - DO NOT EDIT THIS FILE: Auto-generated code by https://github.com/foomo/gocontentful
// Package testapi - DO NOT EDIT THIS FILE: Auto-generated code by https://github.com/foomo/gocontentful v1.0.11
package testapi
import "github.com/foomo/contentful"

197
test/testapi/gocontentfulvolib.go
View File

@ -1,4 +1,4 @@
// Package testapi - DO NOT EDIT THIS FILE: Auto-generated code by https://github.com/foomo/gocontentful
// Package testapi - DO NOT EDIT THIS FILE: Auto-generated code by https://github.com/foomo/gocontentful v1.0.11
package testapi
import (
@ -49,12 +49,16 @@ type ContentfulCacheMutex struct {
type assetCacheMap map[string]*contentful.Asset
type ContentfulClient struct {
Cache *ContentfulCache
CacheMutex *ContentfulCacheMutex
clientMode ClientMode
Client *contentful.Contentful
locales []Locale
logFn func(
Cache *ContentfulCache
cacheInit bool
cacheMutex *ContentfulCacheMutex
cacheQueue chan struct{}
cacheDone chan struct{}
cacheWorkerOnce sync.Once
clientMode ClientMode
Client *contentful.Contentful
locales []Locale
logFn func(
fields map[string]interface{},
level int,
args ...interface{},
@ -148,8 +152,12 @@ var (
InfoUpdatedEntityCache = "updated cache for entity"
InfoCachedAllEntries = "cached all entries of content type"
InfoCachedAllAssets = "cached all assets"
InfoFallingBackToFile = "could not download space data from Contentful, falling back to local file"
InfoFallingBackToFile = "gonna use a local file"
InfoLoadingFromFile = "loading space from local file"
InfoCacheWorkerStart = "cache worker starting"
InfoCacheUpdateQueued = "cache update queued"
InfoCacheUpdateDone = "cache update returning"
InfoCacheUpdateSkipped = "cache update skipped, already one in the queue"
InfoOfflineEntitiesLoaded = "downloaded entries and assets from offline file"
InfoPreservingExistingCache = "could not connect for cache update, preserving the existing cache"
InfoUpdateCacheTime = "space caching done, time recorded"
@ -175,13 +183,13 @@ var SpaceContentTypeInfoMap = ContentTypeInfoMap{
}
func (cc *ContentfulClient) BrokenReferences() (brokenReferences []BrokenReference) {
if cc.Cache == nil || cc.CacheMutex == nil {
if cc.Cache == nil || cc.cacheMutex == nil {
return
}
cc.CacheMutex.parentMapGcLock.Lock()
defer cc.CacheMutex.parentMapGcLock.Unlock()
cc.CacheMutex.idContentTypeMapGcLock.Lock()
defer cc.CacheMutex.idContentTypeMapGcLock.Unlock()
cc.cacheMutex.parentMapGcLock.Lock()
defer cc.cacheMutex.parentMapGcLock.Unlock()
cc.cacheMutex.idContentTypeMapGcLock.Lock()
defer cc.cacheMutex.idContentTypeMapGcLock.Unlock()
for childID, parents := range cc.Cache.parentMap {
if _, okGotEntry := cc.Cache.idContentTypeMap[childID]; !okGotEntry {
for _, parent := range parents {
@ -215,19 +223,15 @@ func (cc *ContentfulClient) ClientStats() {
return
}
if cc.logFn != nil {
cached := false
if cc.Cache != nil {
cached = true
}
fieldsMap := map[string]interface{}{
"space ID": cc.SpaceID,
"environment": cc.Client.Environment,
"clientMode": cc.clientMode,
"contentTypes": strings.Join(cc.Cache.contentTypes, ","),
"locales": cc.locales,
"cached": cached,
"cached": cc.cacheInit,
}
if cached {
if cc.cacheInit {
fieldsMap["cache asset count"] = len(cc.Cache.assets)
fieldsMap["cache entry count"] = len(cc.Cache.idContentTypeMap)
fieldsMap["cache parentMap length"] = len(cc.Cache.parentMap)
@ -270,10 +274,10 @@ func (cc *ContentfulClient) GetAssetByID(id string, forceNoCache ...bool) (*cont
if cc == nil || cc.Client == nil {
return nil, errors.New("GetAssetByID: No client available")
}
if cc.Cache != nil && cc.Cache.assets != nil && (len(forceNoCache) == 0 || !forceNoCache[0]) {
cc.CacheMutex.assetsGcLock.Lock()
if cc.cacheInit && cc.Cache.assets != nil && (len(forceNoCache) == 0 || !forceNoCache[0]) {
cc.cacheMutex.assetsGcLock.Lock()
asset, okAsset := cc.Cache.assets[id]
cc.CacheMutex.assetsGcLock.Unlock()
cc.cacheMutex.assetsGcLock.Unlock()
if okAsset {
return asset, nil
}
@ -309,26 +313,26 @@ func (cc *ContentfulClient) GetContentTypeOfID(id string) (string, error) {
if cc == nil || cc.Client == nil {
return "", errors.New("GetContentTypeOfID: No client available")
}
if cc.Cache != nil && cc.CacheMutex != nil {
if cc.cacheInit {
okVo := false
cc.CacheMutex.brandGcLock.Lock()
cc.cacheMutex.brandGcLock.Lock()
_, okVo = cc.Cache.entryMaps.brand[id]
cc.CacheMutex.brandGcLock.Unlock()
cc.cacheMutex.brandGcLock.Unlock()
if okVo {
return ContentTypeBrand, nil
}
cc.CacheMutex.categoryGcLock.Lock()
cc.cacheMutex.categoryGcLock.Lock()
_, okVo = cc.Cache.entryMaps.category[id]
cc.CacheMutex.categoryGcLock.Unlock()
cc.cacheMutex.categoryGcLock.Unlock()
if okVo {
return ContentTypeCategory, nil
}
cc.CacheMutex.productGcLock.Lock()
cc.cacheMutex.productGcLock.Lock()
_, okVo = cc.Cache.entryMaps.product[id]
cc.CacheMutex.productGcLock.Unlock()
cc.cacheMutex.productGcLock.Unlock()
if okVo {
return ContentTypeProduct, nil
}
@ -385,12 +389,8 @@ func HtmlToRichText(htmlSrc string) *RichTextNode {
if isBasic {
basicRich := &RichTextNode{
NodeType: RichTextNodeDocument,
Content: rtnd.Content,
}
par := RichTextNode{
NodeType: RichTextNodeParagraph,
}
par.Content = rtnd.Content
basicRich.Content = append(basicRich.Content, par)
return basicRich
}
return rtnd
@ -455,8 +455,25 @@ func NewContentfulClient(spaceID string, clientMode ClientMode, clientKey string
}
apiClient.Debug = debug
cc := &ContentfulClient{
clientMode: clientMode,
Client: apiClient,
clientMode: clientMode,
Client: apiClient,
Cache: &ContentfulCache{
contentTypes: []string{},
idContentTypeMap: map[string]string{},
parentMap: map[string][]EntryReference{},
},
cacheMutex: &ContentfulCacheMutex{
fullCacheGcLock: sync.RWMutex{},
sharedDataGcLock: sync.RWMutex{},
assetsGcLock: sync.RWMutex{},
idContentTypeMapGcLock: sync.RWMutex{},
parentMapGcLock: sync.RWMutex{},
brandGcLock: sync.RWMutex{},
categoryGcLock: sync.RWMutex{},
productGcLock: sync.RWMutex{},
},
cacheQueue: make(chan struct{}, 1),
cacheDone: make(chan struct{}, 1),
locales: []Locale{SpaceLocaleGerman, SpaceLocaleFrench},
logFn: logFn,
logLevel: logLevel,
@ -482,6 +499,23 @@ func NewOfflineContentfulClient(filename string, logFn func(fields map[string]in
cc := &ContentfulClient{
clientMode: ClientModeCDA,
Client: contentful.NewCDA(""),
Cache: &ContentfulCache{
contentTypes: []string{},
idContentTypeMap: map[string]string{},
parentMap: map[string][]EntryReference{},
},
cacheMutex: &ContentfulCacheMutex{
fullCacheGcLock: sync.RWMutex{},
sharedDataGcLock: sync.RWMutex{},
assetsGcLock: sync.RWMutex{},
idContentTypeMapGcLock: sync.RWMutex{},
parentMapGcLock: sync.RWMutex{},
brandGcLock: sync.RWMutex{},
categoryGcLock: sync.RWMutex{},
productGcLock: sync.RWMutex{},
},
cacheQueue: make(chan struct{}, 1),
cacheDone: make(chan struct{}, 1),
locales: []Locale{
SpaceLocaleGerman,
SpaceLocaleFrench,
@ -539,7 +573,6 @@ func (cc *ContentfulClient) SetOfflineFallback(filename string) error {
}
func (cc *ContentfulClient) UpdateCache(ctx context.Context, contentTypes []string, cacheAssets bool) error {
start := time.Now()
if contentTypes == nil {
contentTypes = spaceContentTypes
} else {
@ -549,18 +582,38 @@ func (cc *ContentfulClient) UpdateCache(ctx context.Context, contentTypes []stri
}
}
}
if cc.CacheMutex == nil {
cc.CacheMutex = &ContentfulCacheMutex{
fullCacheGcLock: sync.RWMutex{},
sharedDataGcLock: sync.RWMutex{},
assetsGcLock: sync.RWMutex{},
idContentTypeMapGcLock: sync.RWMutex{},
parentMapGcLock: sync.RWMutex{},
brandGcLock: sync.RWMutex{},
categoryGcLock: sync.RWMutex{},
productGcLock: sync.RWMutex{},
cc.cacheWorkerOnce.Do(func() {
go cc.cacheWorker(ctx, contentTypes, cacheAssets)
})
if len(cc.cacheQueue) == 0 {
if cc.logFn != nil && cc.logLevel <= LogInfo {
cc.logFn(map[string]interface{}{"task": "UpdateCache"}, LogInfo, InfoCacheUpdateQueued)
}
cc.cacheQueue <- struct{}{}
<-cc.cacheDone
cc.cacheInit = true
if cc.logFn != nil && cc.logLevel <= LogInfo {
cc.logFn(map[string]interface{}{"task": "UpdateCache"}, LogInfo, InfoCacheUpdateDone)
}
return nil
}
if cc.logFn != nil && cc.logLevel <= LogInfo {
cc.logFn(map[string]interface{}{"task": "UpdateCache"}, LogInfo, InfoCacheUpdateSkipped)
}
return nil
}
func (cc *ContentfulClient) cacheWorker(ctx context.Context, contentTypes []string, cacheAssets bool) {
if cc.logFn != nil && cc.logLevel <= LogInfo {
cc.logFn(map[string]interface{}{"task": "UpdateCache"}, LogInfo, InfoCacheWorkerStart)
}
for _ = range cc.cacheQueue {
cc.cacheSpace(ctx, contentTypes, cacheAssets)
cc.cacheDone <- struct{}{}
}
}
func (cc *ContentfulClient) cacheSpace(ctx context.Context, contentTypes []string, cacheAssets bool) {
start := time.Now()
tempCache := &ContentfulCache{
contentTypes: contentTypes,
idContentTypeMap: map[string]string{},
@ -570,20 +623,20 @@ func (cc *ContentfulClient) UpdateCache(ctx context.Context, contentTypes []stri
contentTypes = append([]string{assetWorkerType}, contentTypes...)
}
_, errCanWeEvenConnect := cc.Client.Spaces.Get(cc.SpaceID)
cc.CacheMutex.sharedDataGcLock.RLock()
cc.cacheMutex.sharedDataGcLock.RLock()
offlinePreviousState := cc.offline
cc.CacheMutex.sharedDataGcLock.RUnlock()
cc.cacheMutex.sharedDataGcLock.RUnlock()
if errCanWeEvenConnect != nil {
if len(cc.offlineTemp.Entries) > 0 && (cc.Cache == nil || offlinePreviousState == true) {
if cc.logFn != nil && cc.logLevel <= LogInfo {
cc.logFn(map[string]interface{}{"method": "UpdateCache"}, LogInfo, InfoFallingBackToFile)
cc.logFn(map[string]interface{}{"task": "UpdateCache"}, LogInfo, InfoFallingBackToFile)
}
cc.CacheMutex.sharedDataGcLock.Lock()
cc.cacheMutex.sharedDataGcLock.Lock()
cc.offline = true
cc.CacheMutex.sharedDataGcLock.Unlock()
cc.cacheMutex.sharedDataGcLock.Unlock()
} else {
if cc.logFn != nil && cc.logLevel <= LogInfo {
cc.logFn(map[string]interface{}{"method": "UpdateCache"}, LogInfo, InfoPreservingExistingCache)
cc.logFn(map[string]interface{}{"task": "UpdateCache"}, LogInfo, InfoPreservingExistingCache)
}
}
}
@ -597,7 +650,7 @@ func (cc *ContentfulClient) UpdateCache(ctx context.Context, contentTypes []stri
err := updateCacheForContentType(gctx, results, cc, tempCache, contentType)
if err != nil {
if cc.logFn != nil && cc.logLevel <= LogInfo {
cc.logFn(map[string]interface{}{"method": "UpdateCache", "contentType": contentType}, LogError, err.Error())
cc.logFn(map[string]interface{}{"task": "UpdateCache", "contentType": contentType}, LogError, err.Error())
}
return err
}
@ -626,24 +679,26 @@ func (cc *ContentfulClient) UpdateCache(ctx context.Context, contentTypes []stri
// drain contentTypeChan
for _ = range contentTypeChan {
}
cc.CacheMutex.sharedDataGcLock.Lock()
cc.cacheMutex.sharedDataGcLock.Lock()
cc.offline = offlinePreviousState
cc.CacheMutex.sharedDataGcLock.Unlock()
return err
cc.cacheMutex.sharedDataGcLock.Unlock()
if cc.logFn != nil && cc.logLevel <= LogInfo {
cc.logFn(map[string]interface{}{"task": "UpdateCache"}, LogError, err.Error())
}
return
}
// Signal that the cache build is done
<-resultsDone
if cc.logFn != nil && cc.logLevel <= LogInfo {
cc.logFn(map[string]interface{}{"time elapsed": fmt.Sprint(time.Since(start)), "method": "UpdateCache"}, LogInfo, InfoUpdateCacheTime)
cc.logFn(map[string]interface{}{"time elapsed": fmt.Sprint(time.Since(start)), "task": "UpdateCache"}, LogInfo, InfoUpdateCacheTime)
}
cc.CacheMutex.fullCacheGcLock.Lock()
defer cc.CacheMutex.fullCacheGcLock.Unlock()
cc.cacheMutex.fullCacheGcLock.Lock()
defer cc.cacheMutex.fullCacheGcLock.Unlock()
cc.Cache = tempCache
cc.CacheMutex.sharedDataGcLock.Lock()
cc.cacheMutex.sharedDataGcLock.Lock()
cc.offline = offlinePreviousState
cc.CacheMutex.sharedDataGcLock.Unlock()
return nil
cc.cacheMutex.sharedDataGcLock.Unlock()
}
func ToAssetReference(asset *contentful.Asset) (refSys ContentTypeSys) {
@ -710,23 +765,23 @@ func (cc *ContentfulClient) cacheGcAssetByID(ctx context.Context, id string) err
asset.Fields.File[string(loc)].URL = "https:" + asset.Fields.File[string(loc)].URL
}
}
cc.CacheMutex.assetsGcLock.Lock()
cc.cacheMutex.assetsGcLock.Lock()
cc.Cache.assets[id] = &asset
cc.CacheMutex.assetsGcLock.Unlock()
cc.cacheMutex.assetsGcLock.Unlock()
return nil
}
func (cc *ContentfulClient) deleteAssetFromCache(key string) error {
if cc.Cache == nil || cc.CacheMutex == nil {
if cc.Cache == nil || cc.cacheMutex == nil {
return errors.New("no cache available")
}
cc.CacheMutex.assetsGcLock.Lock()
cc.cacheMutex.assetsGcLock.Lock()
if _, ok := cc.Cache.assets[key]; ok {
delete(cc.Cache.assets, key)
cc.CacheMutex.assetsGcLock.Unlock()
cc.cacheMutex.assetsGcLock.Unlock()
return nil
}
cc.CacheMutex.assetsGcLock.Unlock()
cc.cacheMutex.assetsGcLock.Unlock()
return errors.New("asset not found in cache, could not delete")
}
@ -769,7 +824,7 @@ func (cc *ContentfulClient) getAllAssets(tryCacheFirst bool) (map[string]*conten
if cc == nil || cc.Client == nil {
return nil, errors.New("getAllAssets: No client available")
}
if cc.Cache != nil && cc.Cache.assets != nil && tryCacheFirst {
if cc.cacheInit && cc.Cache.assets != nil && tryCacheFirst {
return cc.Cache.assets, nil
}
allItems := []interface{}{}
@ -1340,7 +1395,7 @@ func updateCacheForContentTypeAndEntity(ctx context.Context, cc *ContentfulClien
func commonGetParents(cc *ContentfulClient, id string, contentType []string) (parents []EntryReference, err error) {
parents = []EntryReference{}
if cc.Cache != nil {
if cc.cacheInit {
if len(contentType) != 0 {
for _, parent := range cc.Cache.parentMap[id] {
if parent.ContentType == contentType[0] {

28
test/testapi/gocontentfulvolibbrand.go
View File

@ -1,4 +1,4 @@
// Package testapi - DO NOT EDIT THIS FILE: Auto-generated code by https://github.com/foomo/gocontentful
// Package testapi - DO NOT EDIT THIS FILE: Auto-generated code by https://github.com/foomo/gocontentful v1.0.11
package testapi
import (
@ -22,7 +22,7 @@ func (cc *ContentfulClient) GetAllBrand() (voMap map[string]*CfBrand, err error)
if cc == nil {
return nil, errors.New("GetAllBrand: No client available")
}
if cc.Cache != nil {
if cc.cacheInit {
return cc.Cache.entryMaps.brand, nil
}
col, err := cc.optimisticPageSizeGetAll("brand", cc.optimisticPageSize)
@ -68,9 +68,9 @@ func (cc *ContentfulClient) GetBrandByID(id string, forceNoCache ...bool) (vo *C
if cc == nil || cc.Client == nil {
return nil, errors.New("GetBrandByID: No client available")
}
if cc.Cache != nil && cc.CacheMutex != nil && (len(forceNoCache) == 0 || !forceNoCache[0]) {
cc.CacheMutex.brandGcLock.RLock()
defer cc.CacheMutex.brandGcLock.RUnlock()
if cc.cacheInit && (len(forceNoCache) == 0 || !forceNoCache[0]) {
cc.cacheMutex.brandGcLock.RLock()
defer cc.cacheMutex.brandGcLock.RUnlock()
vo, ok := cc.Cache.entryMaps.brand[id]
if ok {
return vo, nil
@ -718,7 +718,7 @@ func (cc *ContentfulClient) cacheAllBrand(ctx context.Context, resultChan chan<-
col := &contentful.Collection{
Items: []interface{}{},
}
cc.CacheMutex.sharedDataGcLock.RLock()
cc.cacheMutex.sharedDataGcLock.RLock()
if cc.offline {
for _, entry := range cc.offlineTemp.Entries {
if entry.Sys.ContentType.Sys.ID == ContentTypeBrand {
@ -731,14 +731,14 @@ func (cc *ContentfulClient) cacheAllBrand(ctx context.Context, resultChan chan<-
return nil, err
}
}
cc.CacheMutex.sharedDataGcLock.RUnlock()
cc.cacheMutex.sharedDataGcLock.RUnlock()
allBrand, err = colToCfBrand(col, cc)
if err != nil {
return nil, err
}
brandMap := map[string]*CfBrand{}
for _, brand := range allBrand {
if cc.Cache != nil {
if cc.cacheInit {
existingBrand, err := cc.GetBrandByID(brand.Sys.ID)
if err == nil && existingBrand != nil && existingBrand.Sys.Version > brand.Sys.Version {
return nil, fmt.Errorf("cache update canceled because Brand entry %s is newer in cache", brand.Sys.ID)
@ -776,12 +776,12 @@ func (cc *ContentfulClient) cacheBrandByID(ctx context.Context, id string) error
if err != nil {
return err
}
cc.CacheMutex.brandGcLock.Lock()
defer cc.CacheMutex.brandGcLock.Unlock()
cc.CacheMutex.idContentTypeMapGcLock.Lock()
defer cc.CacheMutex.idContentTypeMapGcLock.Unlock()
cc.CacheMutex.parentMapGcLock.Lock()
defer cc.CacheMutex.parentMapGcLock.Unlock()
cc.cacheMutex.brandGcLock.Lock()
defer cc.cacheMutex.brandGcLock.Unlock()
cc.cacheMutex.idContentTypeMapGcLock.Lock()
defer cc.cacheMutex.idContentTypeMapGcLock.Unlock()
cc.cacheMutex.parentMapGcLock.Lock()
defer cc.cacheMutex.parentMapGcLock.Unlock()
// It was deleted
if len(col.Items) == 0 {
delete(cc.Cache.entryMaps.brand, id)

28
test/testapi/gocontentfulvolibcategory.go
View File

@ -1,4 +1,4 @@
// Package testapi - DO NOT EDIT THIS FILE: Auto-generated code by https://github.com/foomo/gocontentful
// Package testapi - DO NOT EDIT THIS FILE: Auto-generated code by https://github.com/foomo/gocontentful v1.0.11
package testapi
import (
@ -22,7 +22,7 @@ func (cc *ContentfulClient) GetAllCategory() (voMap map[string]*CfCategory, err
if cc == nil {
return nil, errors.New("GetAllCategory: No client available")
}
if cc.Cache != nil {
if cc.cacheInit {
return cc.Cache.entryMaps.category, nil
}
col, err := cc.optimisticPageSizeGetAll("category", cc.optimisticPageSize)
@ -68,9 +68,9 @@ func (cc *ContentfulClient) GetCategoryByID(id string, forceNoCache ...bool) (vo
if cc == nil || cc.Client == nil {
return nil, errors.New("GetCategoryByID: No client available")
}
if cc.Cache != nil && cc.CacheMutex != nil && (len(forceNoCache) == 0 || !forceNoCache[0]) {
cc.CacheMutex.categoryGcLock.RLock()
defer cc.CacheMutex.categoryGcLock.RUnlock()
if cc.cacheInit && (len(forceNoCache) == 0 || !forceNoCache[0]) {
cc.cacheMutex.categoryGcLock.RLock()
defer cc.cacheMutex.categoryGcLock.RUnlock()
vo, ok := cc.Cache.entryMaps.category[id]
if ok {
return vo, nil
@ -482,7 +482,7 @@ func (cc *ContentfulClient) cacheAllCategory(ctx context.Context, resultChan cha
col := &contentful.Collection{
Items: []interface{}{},
}
cc.CacheMutex.sharedDataGcLock.RLock()
cc.cacheMutex.sharedDataGcLock.RLock()
if cc.offline {
for _, entry := range cc.offlineTemp.Entries {
if entry.Sys.ContentType.Sys.ID == ContentTypeCategory {
@ -495,14 +495,14 @@ func (cc *ContentfulClient) cacheAllCategory(ctx context.Context, resultChan cha
return nil, err
}
}
cc.CacheMutex.sharedDataGcLock.RUnlock()
cc.cacheMutex.sharedDataGcLock.RUnlock()
allCategory, err = colToCfCategory(col, cc)
if err != nil {
return nil, err
}
categoryMap := map[string]*CfCategory{}
for _, category := range allCategory {
if cc.Cache != nil {
if cc.cacheInit {
existingCategory, err := cc.GetCategoryByID(category.Sys.ID)
if err == nil && existingCategory != nil && existingCategory.Sys.Version > category.Sys.Version {
return nil, fmt.Errorf("cache update canceled because Category entry %s is newer in cache", category.Sys.ID)
@ -540,12 +540,12 @@ func (cc *ContentfulClient) cacheCategoryByID(ctx context.Context, id string) er
if err != nil {
return err
}
cc.CacheMutex.categoryGcLock.Lock()
defer cc.CacheMutex.categoryGcLock.Unlock()
cc.CacheMutex.idContentTypeMapGcLock.Lock()
defer cc.CacheMutex.idContentTypeMapGcLock.Unlock()
cc.CacheMutex.parentMapGcLock.Lock()
defer cc.CacheMutex.parentMapGcLock.Unlock()
cc.cacheMutex.categoryGcLock.Lock()
defer cc.cacheMutex.categoryGcLock.Unlock()
cc.cacheMutex.idContentTypeMapGcLock.Lock()
defer cc.cacheMutex.idContentTypeMapGcLock.Unlock()
cc.cacheMutex.parentMapGcLock.Lock()
defer cc.cacheMutex.parentMapGcLock.Unlock()
// It was deleted
if len(col.Items) == 0 {
delete(cc.Cache.entryMaps.category, id)

28
test/testapi/gocontentfulvolibproduct.go
View File

@ -1,4 +1,4 @@
// Package testapi - DO NOT EDIT THIS FILE: Auto-generated code by https://github.com/foomo/gocontentful
// Package testapi - DO NOT EDIT THIS FILE: Auto-generated code by https://github.com/foomo/gocontentful v1.0.11
package testapi
import (
@ -22,7 +22,7 @@ func (cc *ContentfulClient) GetAllProduct() (voMap map[string]*CfProduct, err er
if cc == nil {
return nil, errors.New("GetAllProduct: No client available")
}
if cc.Cache != nil {
if cc.cacheInit {
return cc.Cache.entryMaps.product, nil
}
col, err := cc.optimisticPageSizeGetAll("product", cc.optimisticPageSize)
@ -68,9 +68,9 @@ func (cc *ContentfulClient) GetProductByID(id string, forceNoCache ...bool) (vo
if cc == nil || cc.Client == nil {
return nil, errors.New("GetProductByID: No client available")
}
if cc.Cache != nil && cc.CacheMutex != nil && (len(forceNoCache) == 0 || !forceNoCache[0]) {
cc.CacheMutex.productGcLock.RLock()
defer cc.CacheMutex.productGcLock.RUnlock()
if cc.cacheInit && (len(forceNoCache) == 0 || !forceNoCache[0]) {
cc.cacheMutex.productGcLock.RLock()
defer cc.cacheMutex.productGcLock.RUnlock()
vo, ok := cc.Cache.entryMaps.product[id]
if ok {
return vo, nil
@ -1100,7 +1100,7 @@ func (cc *ContentfulClient) cacheAllProduct(ctx context.Context, resultChan chan
col := &contentful.Collection{
Items: []interface{}{},
}
cc.CacheMutex.sharedDataGcLock.RLock()
cc.cacheMutex.sharedDataGcLock.RLock()
if cc.offline {
for _, entry := range cc.offlineTemp.Entries {
if entry.Sys.ContentType.Sys.ID == ContentTypeProduct {
@ -1113,14 +1113,14 @@ func (cc *ContentfulClient) cacheAllProduct(ctx context.Context, resultChan chan
return nil, err
}
}
cc.CacheMutex.sharedDataGcLock.RUnlock()
cc.cacheMutex.sharedDataGcLock.RUnlock()
allProduct, err = colToCfProduct(col, cc)
if err != nil {
return nil, err
}
productMap := map[string]*CfProduct{}
for _, product := range allProduct {
if cc.Cache != nil {
if cc.cacheInit {
existingProduct, err := cc.GetProductByID(product.Sys.ID)
if err == nil && existingProduct != nil && existingProduct.Sys.Version > product.Sys.Version {
return nil, fmt.Errorf("cache update canceled because Product entry %s is newer in cache", product.Sys.ID)
@ -1182,12 +1182,12 @@ func (cc *ContentfulClient) cacheProductByID(ctx context.Context, id string) err
if err != nil {
return err
}
cc.CacheMutex.productGcLock.Lock()
defer cc.CacheMutex.productGcLock.Unlock()
cc.CacheMutex.idContentTypeMapGcLock.Lock()
defer cc.CacheMutex.idContentTypeMapGcLock.Unlock()
cc.CacheMutex.parentMapGcLock.Lock()
defer cc.CacheMutex.parentMapGcLock.Unlock()
cc.cacheMutex.productGcLock.Lock()
defer cc.cacheMutex.productGcLock.Unlock()
cc.cacheMutex.idContentTypeMapGcLock.Lock()
defer cc.cacheMutex.idContentTypeMapGcLock.Unlock()
cc.cacheMutex.parentMapGcLock.Lock()
defer cc.cacheMutex.parentMapGcLock.Unlock()
// It was deleted
if len(col.Items) == 0 {
delete(cc.Cache.entryMaps.product, id)