From 84803f1e3d23da10e11ffd3cdb8975c8f04a52b4 Mon Sep 17 00:00:00 2001 From: David Ashby Date: Sun, 13 Mar 2022 18:04:09 -0400 Subject: [PATCH 1/5] discogs basic backend --- cmd/serve/main.go | 64 ++++++++++++++++++++---- config/config.go | 13 ++--- database/mysql.go | 45 ++++++++--------- database/records.go | 111 ++++++++++++++++++++++++++++++++++++++++++ frontend/files/app.js | 2 +- go.mod | 3 ++ go.sum | 7 +++ media/media.go | 12 +++++ 8 files changed, 214 insertions(+), 43 deletions(-) create mode 100644 database/records.go diff --git a/cmd/serve/main.go b/cmd/serve/main.go index 119b990..447a77a 100644 --- a/cmd/serve/main.go +++ b/cmd/serve/main.go @@ -7,6 +7,7 @@ import ( "log" "net/http" "strings" + "time" "git.yetaga.in/alazyreader/library/config" "git.yetaga.in/alazyreader/library/database" @@ -22,24 +23,44 @@ func max(a, b int) int { return b } +func obscureStr(in string, l int) string { + return in[0:max(l, len(in))] + strings.Repeat("*", max(0, len(in)-l)) +} + type Library interface { GetAllBooks(context.Context) ([]media.Book, error) } +type RecordCollection interface { + GetAllRecords(context.Context) ([]media.Record, error) +} + type Router struct { static fs.FS lib Library + rcol RecordCollection +} + +func writeJSON(w http.ResponseWriter, b []byte, status int) { + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.WriteHeader(status) + w.Write(b) + w.Write([]byte("\n")) } func (r *Router) ServeHTTP(w http.ResponseWriter, req *http.Request) { - if req.URL.Path == "/api" { - APIHandler(r.lib).ServeHTTP(w, req) + if req.URL.Path == "/api/records" { + RecordsAPIHandler(r.rcol).ServeHTTP(w, req) + return + } + if req.URL.Path == "/api/books" { + BooksAPIHandler(r.lib).ServeHTTP(w, req) return } StaticHandler(r.static).ServeHTTP(w, req) } -func APIHandler(l Library) http.Handler { +func BooksAPIHandler(l Library) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { books, err := l.GetAllBooks(r.Context()) if err != nil { @@ -50,11 +71,25 @@ func APIHandler(l Library) http.Handler { if err != nil { http.Error(w, err.Error(), http.StatusInternalServerError) return + } - w.Header().Set("Content-Type", "application/json") - w.WriteHeader(http.StatusOK) - w.Write(b) - w.Write([]byte("\n")) + writeJSON(w, b, http.StatusOK) + }) +} + +func RecordsAPIHandler(l RecordCollection) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + books, err := l.GetAllRecords(r.Context()) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + b, err := json.Marshal(books) + if err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + writeJSON(w, b, http.StatusOK) }) } @@ -73,8 +108,11 @@ func main() { log.Fatalln(err) } if c.DBUser == "" || c.DBPass == "" || c.DBHost == "" || c.DBPort == "" || c.DBName == "" { - if c.DBPass != "" { // obscure password - c.DBPass = c.DBPass[0:max(3, len(c.DBPass))] + strings.Repeat("*", max(0, len(c.DBPass)-3)) + if c.DBPass != "" { + c.DBPass = obscureStr(c.DBPass, 3) + } + if c.Discogs_Token != "" { + c.Discogs_Token = obscureStr(c.Discogs_Token, 3) } log.Fatalf("vars: %+v", c) } @@ -91,10 +129,16 @@ func main() { log.Fatalln(err) } log.Printf("latest migration: %d; migrations run: %d", latest, run) + discogsCache, err := database.NewDiscogsCache(c.Discogs_Token, time.Hour*24, "delta.mu.alpha") + if err != nil { + log.Fatalln(err) + } + go discogsCache.FlushCache(context.Background()) r := &Router{ static: f, lib: lib, + rcol: discogsCache, } - log.Println("Listening on http://localhost:8080/") + log.Println("Listening on http://0.0.0.0:8080/") log.Fatalln(http.ListenAndServe(":8080", r)) } diff --git a/config/config.go b/config/config.go index 0813444..325f95d 100644 --- a/config/config.go +++ b/config/config.go @@ -1,10 +1,11 @@ package config type Config struct { - DBUser string - DBPass string - DBHost string - DBPort string - DBName string - Debug bool + DBUser string + DBPass string + DBHost string + DBPort string + DBName string + Discogs_Token string + Debug bool } diff --git a/database/mysql.go b/database/mysql.go index 8b769e4..0bd2a40 100644 --- a/database/mysql.go +++ b/database/mysql.go @@ -49,15 +49,15 @@ func (m *MySQL) PrepareDatabase(ctx context.Context) error { return fmt.Errorf("uninitialized mysql client") } - tablecheck := `SELECT count(*) AS count + tablecheck := fmt.Sprintf(`SELECT count(*) AS count FROM information_schema.TABLES - WHERE TABLE_NAME = '` + m.versionTable + `' - AND TABLE_SCHEMA in (SELECT DATABASE());` - tableschema := `CREATE TABLE ` + m.versionTable + `( + WHERE TABLE_NAME = '%s' + AND TABLE_SCHEMA in (SELECT DATABASE());`, m.versionTable) + tableschema := fmt.Sprintf(`CREATE TABLE %s ( id INT NOT NULL, name VARCHAR(100) NOT NULL, datetime DATE, - PRIMARY KEY (id))` + PRIMARY KEY (id))`, m.versionTable) var versionTableExists int m.connection.QueryRowContext(ctx, tablecheck).Scan(&versionTableExists) @@ -73,8 +73,9 @@ func (m *MySQL) GetLatestMigration(ctx context.Context) (int, error) { return 0, fmt.Errorf("uninitialized mysql client") } + migrationCheck := fmt.Sprintf("SELECT COALESCE(MAX(id), 0) FROM %s", m.versionTable) var latestMigration int - err := m.connection.QueryRowContext(ctx, "SELECT COALESCE(MAX(id), 0) FROM "+m.versionTable).Scan(&latestMigration) + err := m.connection.QueryRowContext(ctx, migrationCheck).Scan(&latestMigration) return latestMigration, err } @@ -97,6 +98,9 @@ func (m *MySQL) RunMigrations(ctx context.Context) (int, int, error) { } mig.id, mig.name = id, name mig.content, err = fs.ReadFile(migrationsFS, m.migrationsDirectory+"/"+dir[f].Name()) + if err != nil { + return 0, 0, fmt.Errorf("failure loading migration: %w", err) + } migrations[mig.id] = mig } } @@ -116,6 +120,7 @@ func (m *MySQL) RunMigrations(ctx context.Context) (int, int, error) { if err != nil { return latestMigrationRan, 0, err } + migrationLogSql := fmt.Sprintf("INSERT INTO %s (id, name, datetime) VALUES (?, ?, ?)", m.versionTable) migrationsRun := 0 for migrationsToRun := true; migrationsToRun; _, migrationsToRun = migrations[latestMigrationRan+1] { mig := migrations[latestMigrationRan+1] @@ -127,7 +132,7 @@ func (m *MySQL) RunMigrations(ctx context.Context) (int, int, error) { } return latestMigrationRan, migrationsRun, err } - _, err = tx.ExecContext(ctx, "INSERT INTO "+m.versionTable+" (id, name, datetime) VALUES (?, ?, ?)", mig.id, mig.name, time.Now()) + _, err = tx.ExecContext(ctx, migrationLogSql, mig.id, mig.name, time.Now()) if err != nil { nestederr := tx.Rollback() if nestederr != nil { @@ -147,26 +152,14 @@ func (m *MySQL) GetAllBooks(ctx context.Context) ([]media.Book, error) { return nil, fmt.Errorf("uninitialized mysql client") } + allBooksQuery := fmt.Sprintf(`SELECT + id, title, authors, sortauthor, isbn10, isbn13, format, + genre, publisher, series, volume, year, signed, + description, notes, onloan, coverurl + FROM %s`, m.tableName) + books := []media.Book{} - rows, err := m.connection.QueryContext(ctx, ` - SELECT id, - title, - authors, - sortauthor, - isbn10, - isbn13, - format, - genre, - publisher, - series, - volume, - year, - signed, - description, - notes, - onloan, - coverurl - FROM `+m.tableName) + rows, err := m.connection.QueryContext(ctx, allBooksQuery) if err != nil { return nil, err } diff --git a/database/records.go b/database/records.go new file mode 100644 index 0000000..0195c98 --- /dev/null +++ b/database/records.go @@ -0,0 +1,111 @@ +package database + +import ( + "context" + "fmt" + "log" + "strconv" + "sync" + "time" + + "git.yetaga.in/alazyreader/library/media" + "github.com/irlndts/go-discogs" +) + +type DiscogsCache struct { + authToken string + m sync.Mutex + cache []media.Record + maxCacheAge time.Duration + lastRefresh time.Time + client discogs.Discogs + username string +} + +func NewDiscogsCache(token string, maxCacheAge time.Duration, username string) (*DiscogsCache, error) { + client, err := discogs.New(&discogs.Options{ + UserAgent: "library.yetaga.in personal collection cache", + Token: token, + }) + if err != nil { + return nil, err + } + return &DiscogsCache{ + authToken: token, + client: client, + maxCacheAge: maxCacheAge, + username: username, + }, nil +} + +func (c *DiscogsCache) FlushCache(ctx context.Context) error { + c.m.Lock() + defer c.m.Unlock() + records, err := c.fetchRecords(ctx, nil) + c.cache = records + c.lastRefresh = time.Now() + return err +} + +func (c *DiscogsCache) GetAllRecords(ctx context.Context) ([]media.Record, error) { + c.m.Lock() + defer c.m.Unlock() + if time.Now().After(c.lastRefresh.Add(c.maxCacheAge)) { + records, err := c.fetchRecords(ctx, nil) + c.cache = records + c.lastRefresh = time.Now() + return c.cache, err + } + return c.cache, nil +} + +func (c *DiscogsCache) fetchRecords(ctx context.Context, pagination *discogs.Pagination) ([]media.Record, error) { + records := []media.Record{} + if pagination == nil { + pagination = getPagination(0) + } + log.Printf("calling discogs API, page %v", pagination.Page) + coll, err := c.client.CollectionItemsByFolder(c.username, 0, pagination) + if err != nil { + return records, fmt.Errorf("error loading collection: %w", err) + } + for i := range coll.Items { + records = append(records, collectionItemToRecord(&coll.Items[i])) + } + // recurse down the list + if coll.Pagination.URLs.Next != "" { + coll, err := c.fetchRecords(ctx, getPagination(pagination.Page+1)) + if err != nil { + return records, err + } + records = append(records, coll...) + } + return records, nil +} + +func getPagination(page int) *discogs.Pagination { + return &discogs.Pagination{Page: page, Sort: "added", SortOrder: "asc", PerPage: 100} +} + +func collectionItemToRecord(item *discogs.CollectionItemSource) media.Record { + artists := []string{} + for _, artist := range item.BasicInformation.Artists { + artists = append(artists, artist.Name) + } + year := strconv.Itoa(item.BasicInformation.Year) + if year == "0" { + year = "" + } + return media.Record{ + ID: item.ID, + AlbumName: item.BasicInformation.Title, + Artists: artists, + Identifier: item.BasicInformation.Labels[0].Catno, + Format: item.BasicInformation.Formats[0].Name, + Genre: item.BasicInformation.Genres[0], + Label: item.BasicInformation.Labels[0].Name, + Year: year, + CoverURL: item.BasicInformation.CoverImage, + DiscogsURL: fmt.Sprintf("https://www.discogs.com/release/%v", item.ID), + } +} diff --git a/frontend/files/app.js b/frontend/files/app.js index c43f19a..f218913 100644 --- a/frontend/files/app.js +++ b/frontend/files/app.js @@ -4,7 +4,7 @@ var sortState = { }; function init() { - fetch("/api") + fetch("/api/books") .then((response) => response.json()) .then((books) => { // prepare response diff --git a/go.mod b/go.mod index a6be73c..7d43457 100644 --- a/go.mod +++ b/go.mod @@ -2,8 +2,11 @@ module git.yetaga.in/alazyreader/library go 1.16 +replace github.com/irlndts/go-discogs v0.3.5 => git.yetaga.in/alazyreader/go-discogs v0.3.6 + require ( github.com/gdamore/tcell v1.4.0 github.com/go-sql-driver/mysql v1.6.0 + github.com/irlndts/go-discogs v0.3.5 github.com/kelseyhightower/envconfig v1.4.0 ) diff --git a/go.sum b/go.sum index 4780059..3ed2284 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,15 @@ +git.yetaga.in/alazyreader/go-discogs v0.3.5 h1:XcwcFJP0p1eQQ6OQhScYiNM8+vymeEO4V+M9H9x58os= +git.yetaga.in/alazyreader/go-discogs v0.3.5/go.mod h1:UVQ05FdCzH4P/usnSxQDh77QYE37HvmPnSCgogioljo= +git.yetaga.in/alazyreader/go-discogs v0.3.6 h1:VhV8/uhnWsxae6PvIVtXOfO4eWWqShX6DkiN2hFsZ8U= +git.yetaga.in/alazyreader/go-discogs v0.3.6/go.mod h1:UVQ05FdCzH4P/usnSxQDh77QYE37HvmPnSCgogioljo= github.com/gdamore/encoding v1.0.0 h1:+7OoQ1Bc6eTm5niUzBa0Ctsh6JbMW6Ra+YNuAtDBdko= github.com/gdamore/encoding v1.0.0/go.mod h1:alR0ol34c49FCSBLjhosxzcPHQbf2trDkoo5dl+VrEg= github.com/gdamore/tcell v1.4.0 h1:vUnHwJRvcPQa3tzi+0QI4U9JINXYJlOz9yiaiPQ2wMU= github.com/gdamore/tcell v1.4.0/go.mod h1:vxEiSDZdW3L+Uhjii9c3375IlDmR05bzxY404ZVSMo0= github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= +github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8= github.com/kelseyhightower/envconfig v1.4.0/go.mod h1:cccZRl6mQpaq41TPp5QxidR+Sa3axMbJDNb//FQX6Gg= github.com/lucasb-eyer/go-colorful v1.0.3 h1:QIbQXiugsb+q10B+MI+7DI1oQLdmnep86tWFlaaUAac= @@ -14,3 +20,4 @@ golang.org/x/sys v0.0.0-20190626150813-e07cf5db2756 h1:9nuHUbU8dRnRRfj9KjWUVrJeo golang.org/x/sys v0.0.0-20190626150813-e07cf5db2756/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/media/media.go b/media/media.go index 13ae99e..1b09442 100644 --- a/media/media.go +++ b/media/media.go @@ -21,4 +21,16 @@ type Book struct { } type Record struct { + ID int `json:"-"` + AlbumName string `json:"name"` + Artists []string `json:"artists"` + SortArtist string `json:"sortArtist"` + Identifier string `json:"identifier"` + Format string `json:"format"` + Genre string `json:"genre"` + Label string `json:"label"` + Year string `json:"year"` + Description string `json:"description"` + CoverURL string `json:"coverURL"` + DiscogsURL string `json:"discogsURL"` } -- 2.45.2 From 474ea9b57cc3fd66b9954779af4d9dbc00e82b0f Mon Sep 17 00:00:00 2001 From: David Ashby Date: Sat, 2 Apr 2022 13:10:16 -0400 Subject: [PATCH 2/5] struct name idiomatic --- cmd/serve/main.go | 6 +++--- config/config.go | 14 +++++++------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/cmd/serve/main.go b/cmd/serve/main.go index 447a77a..2a15152 100644 --- a/cmd/serve/main.go +++ b/cmd/serve/main.go @@ -111,8 +111,8 @@ func main() { if c.DBPass != "" { c.DBPass = obscureStr(c.DBPass, 3) } - if c.Discogs_Token != "" { - c.Discogs_Token = obscureStr(c.Discogs_Token, 3) + if c.DiscogsToken != "" { + c.DiscogsToken = obscureStr(c.DiscogsToken, 3) } log.Fatalf("vars: %+v", c) } @@ -129,7 +129,7 @@ func main() { log.Fatalln(err) } log.Printf("latest migration: %d; migrations run: %d", latest, run) - discogsCache, err := database.NewDiscogsCache(c.Discogs_Token, time.Hour*24, "delta.mu.alpha") + discogsCache, err := database.NewDiscogsCache(c.DiscogsToken, time.Hour*24, "delta.mu.alpha") if err != nil { log.Fatalln(err) } diff --git a/config/config.go b/config/config.go index 325f95d..3da0fee 100644 --- a/config/config.go +++ b/config/config.go @@ -1,11 +1,11 @@ package config type Config struct { - DBUser string - DBPass string - DBHost string - DBPort string - DBName string - Discogs_Token string - Debug bool + DBUser string + DBPass string + DBHost string + DBPort string + DBName string + DiscogsToken string + Debug bool } -- 2.45.2 From 832e2025a01a64a62104e2417dd1e23b774132dd Mon Sep 17 00:00:00 2001 From: David Ashby Date: Sat, 2 Apr 2022 13:38:14 -0400 Subject: [PATCH 3/5] initial UI, pagination starts at 1 for some reason --- database/records.go | 3 +- frontend/files/index.html | 1 + frontend/files/records/app.js | 168 +++++++++++++++++ frontend/files/records/favicon.ico | Bin 0 -> 4414 bytes frontend/files/records/favicon.png | Bin 0 -> 70262 bytes frontend/files/records/index.html | 46 +++++ frontend/files/records/style.css | 284 +++++++++++++++++++++++++++++ 7 files changed, 501 insertions(+), 1 deletion(-) create mode 100644 frontend/files/records/app.js create mode 100644 frontend/files/records/favicon.ico create mode 100644 frontend/files/records/favicon.png create mode 100644 frontend/files/records/index.html create mode 100644 frontend/files/records/style.css diff --git a/database/records.go b/database/records.go index 0195c98..46aac98 100644 --- a/database/records.go +++ b/database/records.go @@ -62,13 +62,14 @@ func (c *DiscogsCache) GetAllRecords(ctx context.Context) ([]media.Record, error func (c *DiscogsCache) fetchRecords(ctx context.Context, pagination *discogs.Pagination) ([]media.Record, error) { records := []media.Record{} if pagination == nil { - pagination = getPagination(0) + pagination = getPagination(1) } log.Printf("calling discogs API, page %v", pagination.Page) coll, err := c.client.CollectionItemsByFolder(c.username, 0, pagination) if err != nil { return records, fmt.Errorf("error loading collection: %w", err) } + log.Printf("length: %v, first item in list: %s", len(coll.Items), coll.Items[0].BasicInformation.Title) for i := range coll.Items { records = append(records, collectionItemToRecord(&coll.Items[i])) } diff --git a/frontend/files/index.html b/frontend/files/index.html index 5e0b3b1..d674e39 100644 --- a/frontend/files/index.html +++ b/frontend/files/index.html @@ -22,6 +22,7 @@