add file system persistence of record API calls to avoid hammering the discogs API #3

Merged
alazyreader merged 1 commits from persist into master 2022-04-03 15:51:19 +00:00
4 changed files with 85 additions and 24 deletions

2
.gitignore vendored
View File

@ -2,5 +2,5 @@
/manager /manager
*.properties *.properties
.DS_Store .DS_Store
*.csv
/vendor /vendor
.recordsCache

View File

@ -129,16 +129,10 @@ func main() {
log.Fatalln(err) log.Fatalln(err)
} }
log.Printf("latest migration: %d; migrations run: %d", latest, run) log.Printf("latest migration: %d; migrations run: %d", latest, run)
discogsCache, err := database.NewDiscogsCache(c.DiscogsToken, time.Hour*24, "delta.mu.alpha") discogsCache, err := database.NewDiscogsCache(c.DiscogsToken, time.Hour*24, c.DiscogsUser, c.DiscogsPersist, c.DiscogsCacheFile)
if err != nil { if err != nil {
log.Fatalln(err) log.Fatalln(err)
} }
go func() {
err := discogsCache.FlushCache(context.Background())
if err != nil {
log.Printf("error loading discogs content: %v", err)
}
}()
r := &Router{ r := &Router{
static: f, static: f,
lib: lib, lib: lib,

View File

@ -1,11 +1,14 @@
package config package config
type Config struct { type Config struct {
DBUser string DBUser string
DBPass string DBPass string
DBHost string DBHost string
DBPort string DBPort string
DBName string DBName string
DiscogsToken string DiscogsToken string
Debug bool DiscogsUser string
DiscogsPersist bool
DiscogsCacheFile string `default:".recordsCache"`
Debug bool
} }

View File

@ -2,8 +2,11 @@ package database
import ( import (
"context" "context"
"encoding/gob"
"errors"
"fmt" "fmt"
"log" "log"
"os"
"strconv" "strconv"
"sync" "sync"
"time" "time"
@ -20,9 +23,16 @@ type DiscogsCache struct {
lastRefresh time.Time lastRefresh time.Time
client discogs.Discogs client discogs.Discogs
username string username string
persistence bool
persistFile string
} }
func NewDiscogsCache(token string, maxCacheAge time.Duration, username string) (*DiscogsCache, error) { type persistence struct {
CachedRecordSlice []media.Record
LastRefresh time.Time
}
func NewDiscogsCache(token string, maxCacheAge time.Duration, username string, persist bool, persistFile string) (*DiscogsCache, error) {
client, err := discogs.New(&discogs.Options{ client, err := discogs.New(&discogs.Options{
UserAgent: "library.yetaga.in personal collection cache", UserAgent: "library.yetaga.in personal collection cache",
Token: token, Token: token,
@ -30,21 +40,52 @@ func NewDiscogsCache(token string, maxCacheAge time.Duration, username string) (
if err != nil { if err != nil {
return nil, err return nil, err
} }
return &DiscogsCache{ cache := &DiscogsCache{
authToken: token, authToken: token,
client: client, client: client,
maxCacheAge: maxCacheAge, maxCacheAge: maxCacheAge,
username: username, username: username,
}, nil persistence: persist,
persistFile: persistFile,
}
if cache.persistence && cache.persistFile != "" {
p := &persistence{}
f, err := os.Open(cache.persistFile)
if errors.Is(err, os.ErrNotExist) {
log.Printf("%s not found, skipping file load...", cache.persistFile)
return cache, nil
}
if err != nil {
return cache, fmt.Errorf("error opening cache file %s: %w", cache.persistFile, err)
}
err = gob.NewDecoder(f).Decode(p)
if err != nil {
return cache, fmt.Errorf("error readhing from cache file %s: %w", cache.persistFile, err)
}
log.Printf("loaded %d records from %s", len(p.CachedRecordSlice), cache.persistFile)
cache.cache = p.CachedRecordSlice
cache.lastRefresh = p.LastRefresh
if time.Now().After(cache.lastRefresh.Add(cache.maxCacheAge)) {
log.Printf("cache expired, running refresh...")
go func() {
err := cache.FlushCache(context.Background())
if err != nil {
log.Printf("error loading discogs content: %v", err)
}
}()
}
}
return cache, nil
} }
func (c *DiscogsCache) FlushCache(ctx context.Context) error { func (c *DiscogsCache) FlushCache(ctx context.Context) error {
c.m.Lock() c.m.Lock()
defer c.m.Unlock() defer c.m.Unlock()
records, err := c.fetchRecords(ctx, nil) records, err := c.fetchRecords(ctx, nil)
c.cache = records if err != nil {
c.lastRefresh = time.Now() return err
return err }
return c.saveRecordsToCache(ctx, records)
} }
func (c *DiscogsCache) GetAllRecords(ctx context.Context) ([]media.Record, error) { func (c *DiscogsCache) GetAllRecords(ctx context.Context) ([]media.Record, error) {
@ -52,13 +93,36 @@ func (c *DiscogsCache) GetAllRecords(ctx context.Context) ([]media.Record, error
defer c.m.Unlock() defer c.m.Unlock()
if time.Now().After(c.lastRefresh.Add(c.maxCacheAge)) { if time.Now().After(c.lastRefresh.Add(c.maxCacheAge)) {
records, err := c.fetchRecords(ctx, nil) records, err := c.fetchRecords(ctx, nil)
c.cache = records if err != nil {
c.lastRefresh = time.Now() return c.cache, err
}
err = c.saveRecordsToCache(ctx, records)
return c.cache, err return c.cache, err
} }
return c.cache, nil return c.cache, nil
} }
func (c *DiscogsCache) saveRecordsToCache(ctx context.Context, records []media.Record) error {
c.cache = records
c.lastRefresh = time.Now()
if c.persistence && c.persistFile != "" {
p := persistence{
CachedRecordSlice: c.cache,
LastRefresh: c.lastRefresh,
}
f, err := os.OpenFile(c.persistFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0755)
if err != nil {
return fmt.Errorf("error opening cache file %s: %w", c.persistFile, err)
}
err = gob.NewEncoder(f).Encode(p)
if err != nil {
return fmt.Errorf("error writing to cache file %s: %w", c.persistFile, err)
}
log.Printf("wrote %d records to %s", len(c.cache), c.persistFile)
}
return nil
}
func (c *DiscogsCache) fetchRecords(ctx context.Context, pagination *discogs.Pagination) ([]media.Record, error) { func (c *DiscogsCache) fetchRecords(ctx context.Context, pagination *discogs.Pagination) ([]media.Record, error) {
records := []media.Record{} records := []media.Record{}
if pagination == nil { if pagination == nil {