add file system persistence of record API calls to avoid hammering the discogs API

pull/3/head
David Ashby 6 months ago
parent 7b7e8d0058
commit 5fed609b13
  1. 4
      .gitignore
  2. 8
      cmd/serve/main.go
  3. 17
      config/config.go
  4. 80
      database/records.go

4
.gitignore vendored

@ -2,5 +2,5 @@
/manager
*.properties
.DS_Store
*.csv
/vendor
/vendor
.recordsCache

@ -129,16 +129,10 @@ func main() {
log.Fatalln(err)
}
log.Printf("latest migration: %d; migrations run: %d", latest, run)
discogsCache, err := database.NewDiscogsCache(c.DiscogsToken, time.Hour*24, "delta.mu.alpha")
discogsCache, err := database.NewDiscogsCache(c.DiscogsToken, time.Hour*24, c.DiscogsUser, c.DiscogsPersist, c.DiscogsCacheFile)
if err != nil {
log.Fatalln(err)
}
go func() {
err := discogsCache.FlushCache(context.Background())
if err != nil {
log.Printf("error loading discogs content: %v", err)
}
}()
r := &Router{
static: f,
lib: lib,

@ -1,11 +1,14 @@
package config
type Config struct {
DBUser string
DBPass string
DBHost string
DBPort string
DBName string
DiscogsToken string
Debug bool
DBUser string
DBPass string
DBHost string
DBPort string
DBName string
DiscogsToken string
DiscogsUser string
DiscogsPersist bool
DiscogsCacheFile string `default:".recordsCache"`
Debug bool
}

@ -2,8 +2,11 @@ package database
import (
"context"
"encoding/gob"
"errors"
"fmt"
"log"
"os"
"strconv"
"sync"
"time"
@ -20,9 +23,16 @@ type DiscogsCache struct {
lastRefresh time.Time
client discogs.Discogs
username string
persistence bool
persistFile string
}
func NewDiscogsCache(token string, maxCacheAge time.Duration, username string) (*DiscogsCache, error) {
type persistence struct {
CachedRecordSlice []media.Record
LastRefresh time.Time
}
func NewDiscogsCache(token string, maxCacheAge time.Duration, username string, persist bool, persistFile string) (*DiscogsCache, error) {
client, err := discogs.New(&discogs.Options{
UserAgent: "library.yetaga.in personal collection cache",
Token: token,
@ -30,21 +40,52 @@ func NewDiscogsCache(token string, maxCacheAge time.Duration, username string) (
if err != nil {
return nil, err
}
return &DiscogsCache{
cache := &DiscogsCache{
authToken: token,
client: client,
maxCacheAge: maxCacheAge,
username: username,
}, nil
persistence: persist,
persistFile: persistFile,
}
if cache.persistence && cache.persistFile != "" {
p := &persistence{}
f, err := os.Open(cache.persistFile)
if errors.Is(err, os.ErrNotExist) {
log.Printf("%s not found, skipping file load...", cache.persistFile)
return cache, nil
}
if err != nil {
return cache, fmt.Errorf("error opening cache file %s: %w", cache.persistFile, err)
}
err = gob.NewDecoder(f).Decode(p)
if err != nil {
return cache, fmt.Errorf("error readhing from cache file %s: %w", cache.persistFile, err)
}
log.Printf("loaded %d records from %s", len(p.CachedRecordSlice), cache.persistFile)
cache.cache = p.CachedRecordSlice
cache.lastRefresh = p.LastRefresh
if time.Now().After(cache.lastRefresh.Add(cache.maxCacheAge)) {
log.Printf("cache expired, running refresh...")
go func() {
err := cache.FlushCache(context.Background())
if err != nil {
log.Printf("error loading discogs content: %v", err)
}
}()
}
}
return cache, nil
}
func (c *DiscogsCache) FlushCache(ctx context.Context) error {
c.m.Lock()
defer c.m.Unlock()
records, err := c.fetchRecords(ctx, nil)
c.cache = records
c.lastRefresh = time.Now()
return err
if err != nil {
return err
}
return c.saveRecordsToCache(ctx, records)
}
func (c *DiscogsCache) GetAllRecords(ctx context.Context) ([]media.Record, error) {
@ -52,13 +93,36 @@ func (c *DiscogsCache) GetAllRecords(ctx context.Context) ([]media.Record, error
defer c.m.Unlock()
if time.Now().After(c.lastRefresh.Add(c.maxCacheAge)) {
records, err := c.fetchRecords(ctx, nil)
c.cache = records
c.lastRefresh = time.Now()
if err != nil {
return c.cache, err
}
err = c.saveRecordsToCache(ctx, records)
return c.cache, err
}
return c.cache, nil
}
func (c *DiscogsCache) saveRecordsToCache(ctx context.Context, records []media.Record) error {
c.cache = records
c.lastRefresh = time.Now()
if c.persistence && c.persistFile != "" {
p := persistence{
CachedRecordSlice: c.cache,
LastRefresh: c.lastRefresh,
}
f, err := os.OpenFile(c.persistFile, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0755)
if err != nil {
return fmt.Errorf("error opening cache file %s: %w", c.persistFile, err)
}
err = gob.NewEncoder(f).Encode(p)
if err != nil {
return fmt.Errorf("error writing to cache file %s: %w", c.persistFile, err)
}
log.Printf("wrote %d records to %s", len(c.cache), c.persistFile)
}
return nil
}
func (c *DiscogsCache) fetchRecords(ctx context.Context, pagination *discogs.Pagination) ([]media.Record, error) {
records := []media.Record{}
if pagination == nil {

Loading…
Cancel
Save