Merge pull request 'add file system persistence of record API calls to avoid hammering the discogs API' (#3) from persist into master

Reviewed-on: #3

.gitignore

@@ -2,5 +2,5 @@
 /manager
 *.properties
 .DS_Store
-*.csv
+/vendor
-/vendor
+.recordsCache

cmd/serve/main.go

@@ -129,16 +129,10 @@ func main() {
 		log.Fatalln(err)
 	}
 	log.Printf("latest migration: %d; migrations run: %d", latest, run)
-	discogsCache, err := database.NewDiscogsCache(c.DiscogsToken, time.Hour*24, "delta.mu.alpha")
+	discogsCache, err := database.NewDiscogsCache(c.DiscogsToken, time.Hour*24, c.DiscogsUser, c.DiscogsPersist, c.DiscogsCacheFile)
 	if err != nil {
 		log.Fatalln(err)
 	}
-	go func() {
-		err := discogsCache.FlushCache(context.Background())
-		if err != nil {
-			log.Printf("error loading discogs content: %v", err)
-		}
-	}()
 	r := &Router{
 		static: f,
 		lib:    lib,

config/config.go

@@ -1,11 +1,14 @@
 package config
 
 type Config struct {
-	DBUser       string
+	DBUser           string
-	DBPass       string
+	DBPass           string
-	DBHost       string
+	DBHost           string
-	DBPort       string
+	DBPort           string
-	DBName       string
+	DBName           string
-	DiscogsToken string
+	DiscogsToken     string
-	Debug        bool
+	DiscogsUser      string
+	DiscogsPersist   bool
+	DiscogsCacheFile string `default:".recordsCache"`
+	Debug            bool
 }

database/records.go

@@ -2,8 +2,11 @@ package database
 
 import (
 	"context"
+	"encoding/gob"
+	"errors"
 	"fmt"
 	"log"
+	"os"
 	"strconv"
 	"sync"
 	"time"
@@ -20,9 +23,16 @@ type DiscogsCache struct {
 	lastRefresh time.Time
 	client      discogs.Discogs
 	username    string
+	persistence bool
+	persistFile string
 }
 
-func NewDiscogsCache(token string, maxCacheAge time.Duration, username string) (*DiscogsCache, error) {
+type persistence struct {
+	CachedRecordSlice []media.Record
+	LastRefresh       time.Time
+}
+
+func NewDiscogsCache(token string, maxCacheAge time.Duration, username string, persist bool, persistFile string) (*DiscogsCache, error) {
 	client, err := discogs.New(&discogs.Options{
 		UserAgent: "library.yetaga.in personal collection cache",
 		Token:     token,
@@ -30,21 +40,52 @@ func NewDiscogsCache(token string, maxCacheAge time.Duration, username string) (
 	if err != nil {
 		return nil, err
 	}
-	return &DiscogsCache{
+	cache := &DiscogsCache{
 		authToken:   token,
 		client:      client,
 		maxCacheAge: maxCacheAge,
 		username:    username,
-	}, nil
+		persistence: persist,
+		persistFile: persistFile,
+	}
+	if cache.persistence && cache.persistFile != "" {
+		p := &persistence{}
+		f, err := os.Open(cache.persistFile)
+		if errors.Is(err, os.ErrNotExist) {
+			log.Printf("%s not found, skipping file load...", cache.persistFile)
+			return cache, nil
+		}
+		if err != nil {
+			return cache, fmt.Errorf("error opening cache file %s: %w", cache.persistFile, err)
+		}
+		err = gob.NewDecoder(f).Decode(p)
+		if err != nil {
+			return cache, fmt.Errorf("error readhing from cache file %s: %w", cache.persistFile, err)
+		}
+		log.Printf("loaded %d records from %s", len(p.CachedRecordSlice), cache.persistFile)
+		cache.cache = p.CachedRecordSlice
+		cache.lastRefresh = p.LastRefresh
+		if time.Now().After(cache.lastRefresh.Add(cache.maxCacheAge)) {
+			log.Printf("cache expired, running refresh...")
+			go func() {
+				err := cache.FlushCache(context.Background())
+				if err != nil {
+					log.Printf("error loading discogs content: %v", err)
+				}
+			}()
+		}
+	}
+	return cache, nil
 }
 
 func (c *DiscogsCache) FlushCache(ctx context.Context) error {
 	c.m.Lock()
 	defer c.m.Unlock()
 	records, err := c.fetchRecords(ctx, nil)
-	c.cache = records
+	if err != nil {
-	c.lastRefresh = time.Now()
+		return err
-	return err
+	}
+	return c.saveRecordsToCache(ctx, records)
 }
 
 func (c *DiscogsCache) GetAllRecords(ctx context.Context) ([]media.Record, error) {
@@ -52,13 +93,36 @@ func (c *DiscogsCache) GetAllRecords(ctx context.Context) ([]media.Record, error
 	defer c.m.Unlock()
 	if time.Now().After(c.lastRefresh.Add(c.maxCacheAge)) {
 		records, err := c.fetchRecords(ctx, nil)
-		c.cache = records
+		if err != nil {
-		c.lastRefresh = time.Now()
+			return c.cache, err
+		}
+		err = c.saveRecordsToCache(ctx, records)
 		return c.cache, err
 	}
 	return c.cache, nil
 }
 
+func (c *DiscogsCache) saveRecordsToCache(ctx context.Context, records []media.Record) error {
+	c.cache = records
+	c.lastRefresh = time.Now()
+	if c.persistence && c.persistFile != "" {
+		p := persistence{
+			CachedRecordSlice: c.cache,
+			LastRefresh:       c.lastRefresh,
+		}
+		f, err := os.OpenFile(c.persistFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0755)
+		if err != nil {
+			return fmt.Errorf("error opening cache file %s: %w", c.persistFile, err)
+		}
+		err = gob.NewEncoder(f).Encode(p)
+		if err != nil {
+			return fmt.Errorf("error writing to cache file %s: %w", c.persistFile, err)
+		}
+		log.Printf("wrote %d records to %s", len(c.cache), c.persistFile)
+	}
+	return nil
+}
+
 func (c *DiscogsCache) fetchRecords(ctx context.Context, pagination *discogs.Pagination) ([]media.Record, error) {
 	records := []media.Record{}
 	if pagination == nil {