From 5fed609b131a527172afce671f526586bb2be41b Mon Sep 17 00:00:00 2001 From: David Ashby Date: Sun, 3 Apr 2022 11:26:26 -0400 Subject: [PATCH] add file system persistence of record API calls to avoid hammering the discogs API --- .gitignore | 4 +-- cmd/serve/main.go | 8 +---- config/config.go | 17 ++++++---- database/records.go | 80 ++++++++++++++++++++++++++++++++++++++++----- 4 files changed, 85 insertions(+), 24 deletions(-) diff --git a/.gitignore b/.gitignore index 7edbd02..0742ee2 100644 --- a/.gitignore +++ b/.gitignore @@ -2,5 +2,5 @@ /manager *.properties .DS_Store -*.csv -/vendor \ No newline at end of file +/vendor +.recordsCache \ No newline at end of file diff --git a/cmd/serve/main.go b/cmd/serve/main.go index ada5908..1f71c5b 100644 --- a/cmd/serve/main.go +++ b/cmd/serve/main.go @@ -129,16 +129,10 @@ func main() { log.Fatalln(err) } log.Printf("latest migration: %d; migrations run: %d", latest, run) - discogsCache, err := database.NewDiscogsCache(c.DiscogsToken, time.Hour*24, "delta.mu.alpha") + discogsCache, err := database.NewDiscogsCache(c.DiscogsToken, time.Hour*24, c.DiscogsUser, c.DiscogsPersist, c.DiscogsCacheFile) if err != nil { log.Fatalln(err) } - go func() { - err := discogsCache.FlushCache(context.Background()) - if err != nil { - log.Printf("error loading discogs content: %v", err) - } - }() r := &Router{ static: f, lib: lib, diff --git a/config/config.go b/config/config.go index 3da0fee..b88c747 100644 --- a/config/config.go +++ b/config/config.go @@ -1,11 +1,14 @@ package config type Config struct { - DBUser string - DBPass string - DBHost string - DBPort string - DBName string - DiscogsToken string - Debug bool + DBUser string + DBPass string + DBHost string + DBPort string + DBName string + DiscogsToken string + DiscogsUser string + DiscogsPersist bool + DiscogsCacheFile string `default:".recordsCache"` + Debug bool } diff --git a/database/records.go b/database/records.go index 46aac98..c1f50f6 100644 --- a/database/records.go +++ b/database/records.go @@ -2,8 +2,11 @@ package database import ( "context" + "encoding/gob" + "errors" "fmt" "log" + "os" "strconv" "sync" "time" @@ -20,9 +23,16 @@ type DiscogsCache struct { lastRefresh time.Time client discogs.Discogs username string + persistence bool + persistFile string } -func NewDiscogsCache(token string, maxCacheAge time.Duration, username string) (*DiscogsCache, error) { +type persistence struct { + CachedRecordSlice []media.Record + LastRefresh time.Time +} + +func NewDiscogsCache(token string, maxCacheAge time.Duration, username string, persist bool, persistFile string) (*DiscogsCache, error) { client, err := discogs.New(&discogs.Options{ UserAgent: "library.yetaga.in personal collection cache", Token: token, @@ -30,21 +40,52 @@ func NewDiscogsCache(token string, maxCacheAge time.Duration, username string) ( if err != nil { return nil, err } - return &DiscogsCache{ + cache := &DiscogsCache{ authToken: token, client: client, maxCacheAge: maxCacheAge, username: username, - }, nil + persistence: persist, + persistFile: persistFile, + } + if cache.persistence && cache.persistFile != "" { + p := &persistence{} + f, err := os.Open(cache.persistFile) + if errors.Is(err, os.ErrNotExist) { + log.Printf("%s not found, skipping file load...", cache.persistFile) + return cache, nil + } + if err != nil { + return cache, fmt.Errorf("error opening cache file %s: %w", cache.persistFile, err) + } + err = gob.NewDecoder(f).Decode(p) + if err != nil { + return cache, fmt.Errorf("error readhing from cache file %s: %w", cache.persistFile, err) + } + log.Printf("loaded %d records from %s", len(p.CachedRecordSlice), cache.persistFile) + cache.cache = p.CachedRecordSlice + cache.lastRefresh = p.LastRefresh + if time.Now().After(cache.lastRefresh.Add(cache.maxCacheAge)) { + log.Printf("cache expired, running refresh...") + go func() { + err := cache.FlushCache(context.Background()) + if err != nil { + log.Printf("error loading discogs content: %v", err) + } + }() + } + } + return cache, nil } func (c *DiscogsCache) FlushCache(ctx context.Context) error { c.m.Lock() defer c.m.Unlock() records, err := c.fetchRecords(ctx, nil) - c.cache = records - c.lastRefresh = time.Now() - return err + if err != nil { + return err + } + return c.saveRecordsToCache(ctx, records) } func (c *DiscogsCache) GetAllRecords(ctx context.Context) ([]media.Record, error) { @@ -52,13 +93,36 @@ func (c *DiscogsCache) GetAllRecords(ctx context.Context) ([]media.Record, error defer c.m.Unlock() if time.Now().After(c.lastRefresh.Add(c.maxCacheAge)) { records, err := c.fetchRecords(ctx, nil) - c.cache = records - c.lastRefresh = time.Now() + if err != nil { + return c.cache, err + } + err = c.saveRecordsToCache(ctx, records) return c.cache, err } return c.cache, nil } +func (c *DiscogsCache) saveRecordsToCache(ctx context.Context, records []media.Record) error { + c.cache = records + c.lastRefresh = time.Now() + if c.persistence && c.persistFile != "" { + p := persistence{ + CachedRecordSlice: c.cache, + LastRefresh: c.lastRefresh, + } + f, err := os.OpenFile(c.persistFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0755) + if err != nil { + return fmt.Errorf("error opening cache file %s: %w", c.persistFile, err) + } + err = gob.NewEncoder(f).Encode(p) + if err != nil { + return fmt.Errorf("error writing to cache file %s: %w", c.persistFile, err) + } + log.Printf("wrote %d records to %s", len(c.cache), c.persistFile) + } + return nil +} + func (c *DiscogsCache) fetchRecords(ctx context.Context, pagination *discogs.Pagination) ([]media.Record, error) { records := []media.Record{} if pagination == nil {