add file system persistence of record API calls to avoid hammering the discogs API
Some checks failed
ci/woodpecker/push/woodpecker Pipeline failed
ci/woodpecker/pr/woodpecker Pipeline failed

This commit is contained in:
David 2022-04-03 11:26:26 -04:00
parent 7b7e8d0058
commit 5fed609b13
4 changed files with 85 additions and 24 deletions

4
.gitignore vendored
View File

@ -2,5 +2,5 @@
/manager
*.properties
.DS_Store
*.csv
/vendor
/vendor
.recordsCache

View File

@ -129,16 +129,10 @@ func main() {
log.Fatalln(err)
}
log.Printf("latest migration: %d; migrations run: %d", latest, run)
discogsCache, err := database.NewDiscogsCache(c.DiscogsToken, time.Hour*24, "delta.mu.alpha")
discogsCache, err := database.NewDiscogsCache(c.DiscogsToken, time.Hour*24, c.DiscogsUser, c.DiscogsPersist, c.DiscogsCacheFile)
if err != nil {
log.Fatalln(err)
}
go func() {
err := discogsCache.FlushCache(context.Background())
if err != nil {
log.Printf("error loading discogs content: %v", err)
}
}()
r := &Router{
static: f,
lib: lib,

View File

@ -1,11 +1,14 @@
package config
type Config struct {
DBUser string
DBPass string
DBHost string
DBPort string
DBName string
DiscogsToken string
Debug bool
DBUser string
DBPass string
DBHost string
DBPort string
DBName string
DiscogsToken string
DiscogsUser string
DiscogsPersist bool
DiscogsCacheFile string `default:".recordsCache"`
Debug bool
}

View File

@ -2,8 +2,11 @@ package database
import (
"context"
"encoding/gob"
"errors"
"fmt"
"log"
"os"
"strconv"
"sync"
"time"
@ -20,9 +23,16 @@ type DiscogsCache struct {
lastRefresh time.Time
client discogs.Discogs
username string
persistence bool
persistFile string
}
func NewDiscogsCache(token string, maxCacheAge time.Duration, username string) (*DiscogsCache, error) {
type persistence struct {
CachedRecordSlice []media.Record
LastRefresh time.Time
}
func NewDiscogsCache(token string, maxCacheAge time.Duration, username string, persist bool, persistFile string) (*DiscogsCache, error) {
client, err := discogs.New(&discogs.Options{
UserAgent: "library.yetaga.in personal collection cache",
Token: token,
@ -30,21 +40,52 @@ func NewDiscogsCache(token string, maxCacheAge time.Duration, username string) (
if err != nil {
return nil, err
}
return &DiscogsCache{
cache := &DiscogsCache{
authToken: token,
client: client,
maxCacheAge: maxCacheAge,
username: username,
}, nil
persistence: persist,
persistFile: persistFile,
}
if cache.persistence && cache.persistFile != "" {
p := &persistence{}
f, err := os.Open(cache.persistFile)
if errors.Is(err, os.ErrNotExist) {
log.Printf("%s not found, skipping file load...", cache.persistFile)
return cache, nil
}
if err != nil {
return cache, fmt.Errorf("error opening cache file %s: %w", cache.persistFile, err)
}
err = gob.NewDecoder(f).Decode(p)
if err != nil {
return cache, fmt.Errorf("error readhing from cache file %s: %w", cache.persistFile, err)
}
log.Printf("loaded %d records from %s", len(p.CachedRecordSlice), cache.persistFile)
cache.cache = p.CachedRecordSlice
cache.lastRefresh = p.LastRefresh
if time.Now().After(cache.lastRefresh.Add(cache.maxCacheAge)) {
log.Printf("cache expired, running refresh...")
go func() {
err := cache.FlushCache(context.Background())
if err != nil {
log.Printf("error loading discogs content: %v", err)
}
}()
}
}
return cache, nil
}
func (c *DiscogsCache) FlushCache(ctx context.Context) error {
c.m.Lock()
defer c.m.Unlock()
records, err := c.fetchRecords(ctx, nil)
c.cache = records
c.lastRefresh = time.Now()
return err
if err != nil {
return err
}
return c.saveRecordsToCache(ctx, records)
}
func (c *DiscogsCache) GetAllRecords(ctx context.Context) ([]media.Record, error) {
@ -52,13 +93,36 @@ func (c *DiscogsCache) GetAllRecords(ctx context.Context) ([]media.Record, error
defer c.m.Unlock()
if time.Now().After(c.lastRefresh.Add(c.maxCacheAge)) {
records, err := c.fetchRecords(ctx, nil)
c.cache = records
c.lastRefresh = time.Now()
if err != nil {
return c.cache, err
}
err = c.saveRecordsToCache(ctx, records)
return c.cache, err
}
return c.cache, nil
}
func (c *DiscogsCache) saveRecordsToCache(ctx context.Context, records []media.Record) error {
c.cache = records
c.lastRefresh = time.Now()
if c.persistence && c.persistFile != "" {
p := persistence{
CachedRecordSlice: c.cache,
LastRefresh: c.lastRefresh,
}
f, err := os.OpenFile(c.persistFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0755)
if err != nil {
return fmt.Errorf("error opening cache file %s: %w", c.persistFile, err)
}
err = gob.NewEncoder(f).Encode(p)
if err != nil {
return fmt.Errorf("error writing to cache file %s: %w", c.persistFile, err)
}
log.Printf("wrote %d records to %s", len(c.cache), c.persistFile)
}
return nil
}
func (c *DiscogsCache) fetchRecords(ctx context.Context, pagination *discogs.Pagination) ([]media.Record, error) {
records := []media.Record{}
if pagination == nil {