add file system persistence of record API calls to avoid hammering the discogs API
ci/woodpecker/push/woodpecker Pipeline failed Details
ci/woodpecker/pr/woodpecker Pipeline failed Details

This commit is contained in:
David 2022-04-03 11:26:26 -04:00
parent 7b7e8d0058
commit 5fed609b13
4 changed files with 85 additions and 24 deletions

2
.gitignore vendored
View File

@ -2,5 +2,5 @@
/manager
*.properties
.DS_Store
*.csv
/vendor
.recordsCache

View File

@ -129,16 +129,10 @@ func main() {
log.Fatalln(err)
}
log.Printf("latest migration: %d; migrations run: %d", latest, run)
discogsCache, err := database.NewDiscogsCache(c.DiscogsToken, time.Hour*24, "delta.mu.alpha")
discogsCache, err := database.NewDiscogsCache(c.DiscogsToken, time.Hour*24, c.DiscogsUser, c.DiscogsPersist, c.DiscogsCacheFile)
if err != nil {
log.Fatalln(err)
}
go func() {
err := discogsCache.FlushCache(context.Background())
if err != nil {
log.Printf("error loading discogs content: %v", err)
}
}()
r := &Router{
static: f,
lib: lib,

View File

@ -7,5 +7,8 @@ type Config struct {
DBPort string
DBName string
DiscogsToken string
DiscogsUser string
DiscogsPersist bool
DiscogsCacheFile string `default:".recordsCache"`
Debug bool
}

View File

@ -2,8 +2,11 @@ package database
import (
"context"
"encoding/gob"
"errors"
"fmt"
"log"
"os"
"strconv"
"sync"
"time"
@ -20,9 +23,16 @@ type DiscogsCache struct {
lastRefresh time.Time
client discogs.Discogs
username string
persistence bool
persistFile string
}
func NewDiscogsCache(token string, maxCacheAge time.Duration, username string) (*DiscogsCache, error) {
type persistence struct {
CachedRecordSlice []media.Record
LastRefresh time.Time
}
func NewDiscogsCache(token string, maxCacheAge time.Duration, username string, persist bool, persistFile string) (*DiscogsCache, error) {
client, err := discogs.New(&discogs.Options{
UserAgent: "library.yetaga.in personal collection cache",
Token: token,
@ -30,35 +40,89 @@ func NewDiscogsCache(token string, maxCacheAge time.Duration, username string) (
if err != nil {
return nil, err
}
return &DiscogsCache{
cache := &DiscogsCache{
authToken: token,
client: client,
maxCacheAge: maxCacheAge,
username: username,
}, nil
persistence: persist,
persistFile: persistFile,
}
if cache.persistence && cache.persistFile != "" {
p := &persistence{}
f, err := os.Open(cache.persistFile)
if errors.Is(err, os.ErrNotExist) {
log.Printf("%s not found, skipping file load...", cache.persistFile)
return cache, nil
}
if err != nil {
return cache, fmt.Errorf("error opening cache file %s: %w", cache.persistFile, err)
}
err = gob.NewDecoder(f).Decode(p)
if err != nil {
return cache, fmt.Errorf("error readhing from cache file %s: %w", cache.persistFile, err)
}
log.Printf("loaded %d records from %s", len(p.CachedRecordSlice), cache.persistFile)
cache.cache = p.CachedRecordSlice
cache.lastRefresh = p.LastRefresh
if time.Now().After(cache.lastRefresh.Add(cache.maxCacheAge)) {
log.Printf("cache expired, running refresh...")
go func() {
err := cache.FlushCache(context.Background())
if err != nil {
log.Printf("error loading discogs content: %v", err)
}
}()
}
}
return cache, nil
}
func (c *DiscogsCache) FlushCache(ctx context.Context) error {
c.m.Lock()
defer c.m.Unlock()
records, err := c.fetchRecords(ctx, nil)
c.cache = records
c.lastRefresh = time.Now()
if err != nil {
return err
}
return c.saveRecordsToCache(ctx, records)
}
func (c *DiscogsCache) GetAllRecords(ctx context.Context) ([]media.Record, error) {
c.m.Lock()
defer c.m.Unlock()
if time.Now().After(c.lastRefresh.Add(c.maxCacheAge)) {
records, err := c.fetchRecords(ctx, nil)
c.cache = records
c.lastRefresh = time.Now()
if err != nil {
return c.cache, err
}
err = c.saveRecordsToCache(ctx, records)
return c.cache, err
}
return c.cache, nil
}
func (c *DiscogsCache) saveRecordsToCache(ctx context.Context, records []media.Record) error {
c.cache = records
c.lastRefresh = time.Now()
if c.persistence && c.persistFile != "" {
p := persistence{
CachedRecordSlice: c.cache,
LastRefresh: c.lastRefresh,
}
f, err := os.OpenFile(c.persistFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0755)
if err != nil {
return fmt.Errorf("error opening cache file %s: %w", c.persistFile, err)
}
err = gob.NewEncoder(f).Encode(p)
if err != nil {
return fmt.Errorf("error writing to cache file %s: %w", c.persistFile, err)
}
log.Printf("wrote %d records to %s", len(c.cache), c.persistFile)
}
return nil
}
func (c *DiscogsCache) fetchRecords(ctx context.Context, pagination *discogs.Pagination) ([]media.Record, error) {
records := []media.Record{}
if pagination == nil {