add file system persistence of record API calls to avoid hammering the discogs API
This commit is contained in:
parent
7b7e8d0058
commit
5fed609b13
2
.gitignore
vendored
2
.gitignore
vendored
@ -2,5 +2,5 @@
|
||||
/manager
|
||||
*.properties
|
||||
.DS_Store
|
||||
*.csv
|
||||
/vendor
|
||||
.recordsCache
|
@ -129,16 +129,10 @@ func main() {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
log.Printf("latest migration: %d; migrations run: %d", latest, run)
|
||||
discogsCache, err := database.NewDiscogsCache(c.DiscogsToken, time.Hour*24, "delta.mu.alpha")
|
||||
discogsCache, err := database.NewDiscogsCache(c.DiscogsToken, time.Hour*24, c.DiscogsUser, c.DiscogsPersist, c.DiscogsCacheFile)
|
||||
if err != nil {
|
||||
log.Fatalln(err)
|
||||
}
|
||||
go func() {
|
||||
err := discogsCache.FlushCache(context.Background())
|
||||
if err != nil {
|
||||
log.Printf("error loading discogs content: %v", err)
|
||||
}
|
||||
}()
|
||||
r := &Router{
|
||||
static: f,
|
||||
lib: lib,
|
||||
|
@ -1,11 +1,14 @@
|
||||
package config
|
||||
|
||||
type Config struct {
|
||||
DBUser string
|
||||
DBPass string
|
||||
DBHost string
|
||||
DBPort string
|
||||
DBName string
|
||||
DiscogsToken string
|
||||
Debug bool
|
||||
DBUser string
|
||||
DBPass string
|
||||
DBHost string
|
||||
DBPort string
|
||||
DBName string
|
||||
DiscogsToken string
|
||||
DiscogsUser string
|
||||
DiscogsPersist bool
|
||||
DiscogsCacheFile string `default:".recordsCache"`
|
||||
Debug bool
|
||||
}
|
||||
|
@ -2,8 +2,11 @@ package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/gob"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"strconv"
|
||||
"sync"
|
||||
"time"
|
||||
@ -20,9 +23,16 @@ type DiscogsCache struct {
|
||||
lastRefresh time.Time
|
||||
client discogs.Discogs
|
||||
username string
|
||||
persistence bool
|
||||
persistFile string
|
||||
}
|
||||
|
||||
func NewDiscogsCache(token string, maxCacheAge time.Duration, username string) (*DiscogsCache, error) {
|
||||
type persistence struct {
|
||||
CachedRecordSlice []media.Record
|
||||
LastRefresh time.Time
|
||||
}
|
||||
|
||||
func NewDiscogsCache(token string, maxCacheAge time.Duration, username string, persist bool, persistFile string) (*DiscogsCache, error) {
|
||||
client, err := discogs.New(&discogs.Options{
|
||||
UserAgent: "library.yetaga.in personal collection cache",
|
||||
Token: token,
|
||||
@ -30,21 +40,52 @@ func NewDiscogsCache(token string, maxCacheAge time.Duration, username string) (
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &DiscogsCache{
|
||||
cache := &DiscogsCache{
|
||||
authToken: token,
|
||||
client: client,
|
||||
maxCacheAge: maxCacheAge,
|
||||
username: username,
|
||||
}, nil
|
||||
persistence: persist,
|
||||
persistFile: persistFile,
|
||||
}
|
||||
if cache.persistence && cache.persistFile != "" {
|
||||
p := &persistence{}
|
||||
f, err := os.Open(cache.persistFile)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
log.Printf("%s not found, skipping file load...", cache.persistFile)
|
||||
return cache, nil
|
||||
}
|
||||
if err != nil {
|
||||
return cache, fmt.Errorf("error opening cache file %s: %w", cache.persistFile, err)
|
||||
}
|
||||
err = gob.NewDecoder(f).Decode(p)
|
||||
if err != nil {
|
||||
return cache, fmt.Errorf("error readhing from cache file %s: %w", cache.persistFile, err)
|
||||
}
|
||||
log.Printf("loaded %d records from %s", len(p.CachedRecordSlice), cache.persistFile)
|
||||
cache.cache = p.CachedRecordSlice
|
||||
cache.lastRefresh = p.LastRefresh
|
||||
if time.Now().After(cache.lastRefresh.Add(cache.maxCacheAge)) {
|
||||
log.Printf("cache expired, running refresh...")
|
||||
go func() {
|
||||
err := cache.FlushCache(context.Background())
|
||||
if err != nil {
|
||||
log.Printf("error loading discogs content: %v", err)
|
||||
}
|
||||
}()
|
||||
}
|
||||
}
|
||||
return cache, nil
|
||||
}
|
||||
|
||||
func (c *DiscogsCache) FlushCache(ctx context.Context) error {
|
||||
c.m.Lock()
|
||||
defer c.m.Unlock()
|
||||
records, err := c.fetchRecords(ctx, nil)
|
||||
c.cache = records
|
||||
c.lastRefresh = time.Now()
|
||||
return err
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return c.saveRecordsToCache(ctx, records)
|
||||
}
|
||||
|
||||
func (c *DiscogsCache) GetAllRecords(ctx context.Context) ([]media.Record, error) {
|
||||
@ -52,13 +93,36 @@ func (c *DiscogsCache) GetAllRecords(ctx context.Context) ([]media.Record, error
|
||||
defer c.m.Unlock()
|
||||
if time.Now().After(c.lastRefresh.Add(c.maxCacheAge)) {
|
||||
records, err := c.fetchRecords(ctx, nil)
|
||||
c.cache = records
|
||||
c.lastRefresh = time.Now()
|
||||
if err != nil {
|
||||
return c.cache, err
|
||||
}
|
||||
err = c.saveRecordsToCache(ctx, records)
|
||||
return c.cache, err
|
||||
}
|
||||
return c.cache, nil
|
||||
}
|
||||
|
||||
func (c *DiscogsCache) saveRecordsToCache(ctx context.Context, records []media.Record) error {
|
||||
c.cache = records
|
||||
c.lastRefresh = time.Now()
|
||||
if c.persistence && c.persistFile != "" {
|
||||
p := persistence{
|
||||
CachedRecordSlice: c.cache,
|
||||
LastRefresh: c.lastRefresh,
|
||||
}
|
||||
f, err := os.OpenFile(c.persistFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0755)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error opening cache file %s: %w", c.persistFile, err)
|
||||
}
|
||||
err = gob.NewEncoder(f).Encode(p)
|
||||
if err != nil {
|
||||
return fmt.Errorf("error writing to cache file %s: %w", c.persistFile, err)
|
||||
}
|
||||
log.Printf("wrote %d records to %s", len(c.cache), c.persistFile)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *DiscogsCache) fetchRecords(ctx context.Context, pagination *discogs.Pagination) ([]media.Record, error) {
|
||||
records := []media.Record{}
|
||||
if pagination == nil {
|
||||
|
Loading…
Reference in New Issue
Block a user