Adding support for PostgreSQL as database

This adds support for a second database backend: PostgreSQL (in addition to sqlite3). This allows externailzing the database used by gonic.

Author: 02strich

cmd/gonic/gonic.go

@@ -3,6 +3,7 @@ package main
 
 import (
 	"context"
+	"encoding/base64"
 	"errors"
 	"expvar"
 	"flag"
@@ -26,6 +27,7 @@ import (
 
 	"github.com/google/shlex"
 	"github.com/gorilla/securecookie"
+	_ "github.com/jinzhu/gorm/dialects/postgres"
 	_ "github.com/jinzhu/gorm/dialects/sqlite"
 	"github.com/sentriz/gormstore"
 	"golang.org/x/sync/errgroup"
@@ -65,8 +67,7 @@ func main() {
 	flag.Var(&confMusicPaths, "music-path", "path to music")
 
 	confPlaylistsPath := flag.String("playlists-path", "", "path to your list of new or existing m3u playlists that gonic can manage")
-
-	confDBPath := flag.String("db-path", "gonic.db", "path to database (optional)")
+	confDBURI := flag.String("db-uri", "", "db URI")
 
 	confScanIntervalMins := flag.Uint("scan-interval", 0, "interval (in minutes) to automatically scan music (optional)")
 	confScanAtStart := flag.Bool("scan-at-start-enabled", false, "whether to perform an initial scan at startup (optional)")
@@ -92,6 +93,7 @@ func main() {
 	confExpvar := flag.Bool("expvar", false, "enable the /debug/vars endpoint (optional)")
 
 	deprecatedConfGenreSplit := flag.String("genre-split", "", "(deprecated, see multi-value settings)")
+	deprecatedConfDBPath := flag.String("db-path", "gonic.db", "(deprecated, see db-uri)")
 
 	flag.Parse()
 	flagconf.ParseEnv()
@@ -136,15 +138,18 @@ func main() {
 		log.Fatalf("couldn't create covers cache path: %v\n", err)
 	}
 
-	dbc, err := db.New(*confDBPath, db.DefaultOptions())
+	if *confDBURI == "" {
+		*confDBURI = "sqlite3://" + *deprecatedConfDBPath
+	}
+
+	dbc, err := db.New(*confDBURI)
 	if err != nil {
 		log.Fatalf("error opening database: %v\n", err)
 	}
 	defer dbc.Close()
 
 	err = dbc.Migrate(db.MigrationContext{
 		Production:        true,
-		DBPath:            *confDBPath,
 		OriginalMusicPath: confMusicPaths[0].path,
 		PlaylistsPath:     *confPlaylistsPath,
 		PodcastsPath:      *confPodcastPath,
@@ -225,17 +230,18 @@ func main() {
 		jukebx = jukebox.New()
 	}
 
-	sessKey, err := dbc.GetSetting("session_key")
+	encSessKey, err := dbc.GetSetting("session_key")
 	if err != nil {
 		log.Panicf("error getting session key: %v\n", err)
 	}
-	if sessKey == "" {
-		sessKey = string(securecookie.GenerateRandomKey(32))
-		if err := dbc.SetSetting("session_key", sessKey); err != nil {
+	sessKey, err := base64.StdEncoding.DecodeString(encSessKey)
+	if err != nil || len(sessKey) == 0 {
+		sessKey = securecookie.GenerateRandomKey(32)
+		if err := dbc.SetSetting("session_key", base64.StdEncoding.EncodeToString(sessKey)); err != nil {
 			log.Panicf("error setting session key: %v\n", err)
 		}
 	}
-	sessDB := gormstore.New(dbc.DB, []byte(sessKey))
+	sessDB := gormstore.New(dbc.DB, []byte(encSessKey))
 	sessDB.SessionOpts.HttpOnly = true
 	sessDB.SessionOpts.SameSite = http.SameSiteLaxMode
 

db/db.go

@@ -1,7 +1,6 @@
 package db
 
 import (
-	"context"
 	"errors"
 	"fmt"
 	"log"
@@ -13,55 +12,55 @@ import (
 	"time"
 
 	"github.com/jinzhu/gorm"
-	"github.com/mattn/go-sqlite3"
+	_ "github.com/jinzhu/gorm/dialects/postgres"
+	_ "github.com/jinzhu/gorm/dialects/sqlite"
 
 	// TODO: remove this dep
 	"go.senan.xyz/gonic/server/ctrlsubsonic/specid"
 )
 
-func DefaultOptions() url.Values {
-	return url.Values{
-		// with this, multiple connections share a single data and schema cache.
-		// see https://www.sqlite.org/sharedcache.html
-		"cache": {"shared"},
-		// with this, the db sleeps for a little while when locked. can prevent
-		// a SQLITE_BUSY. see https://www.sqlite.org/c3ref/busy_timeout.html
-		"_busy_timeout": {"30000"},
-		"_journal_mode": {"WAL"},
-		"_foreign_keys": {"true"},
-	}
+type DB struct {
+	*gorm.DB
 }
 
-func mockOptions() url.Values {
-	return url.Values{
-		"_foreign_keys": {"true"},
+func New(uri string) (*DB, error) {
+	if uri == "" {
+		return nil, fmt.Errorf("empty db uri")
 	}
-}
 
-type DB struct {
-	*gorm.DB
-}
+	url, err := url.Parse(uri)
+	if err != nil {
+		return nil, fmt.Errorf("parse uri: %w", err)
+	}
 
-func New(path string, options url.Values) (*DB, error) {
-	// https://github.com/mattn/go-sqlite3#connection-string
-	url := url.URL{
-		Scheme: "file",
-		Opaque: path,
+	gormUrl := strings.TrimPrefix(url.String(), url.Scheme+"://")
+
+	//nolint:goconst
+	switch url.Scheme {
+	case "sqlite3":
+		q := url.Query()
+		q.Set("cache", "shared")
+		q.Set("_busy_timeout", "30000")
+		q.Set("_journal_mode", "WAL")
+		q.Set("_foreign_keys", "true")
+		url.RawQuery = q.Encode()
+	case "postgres":
+		// the postgres driver expects the schema prefix to be on the URL
+		gormUrl = url.String()
+	default:
+		return nil, fmt.Errorf("unknown db scheme")
 	}
-	url.RawQuery = options.Encode()
-	db, err := gorm.Open("sqlite3", url.String())
+
+	db, err := gorm.Open(url.Scheme, gormUrl)
 	if err != nil {
 		return nil, fmt.Errorf("with gorm: %w", err)
 	}
+
 	db.SetLogger(log.New(os.Stdout, "gorm ", 0))
 	db.DB().SetMaxOpenConns(1)
 	return &DB{DB: db}, nil
 }
 
-func NewMock() (*DB, error) {
-	return New(":memory:", mockOptions())
-}
-
 func (db *DB) InsertBulkLeftMany(table string, head []string, left int, col []int) error {
 	if len(col) == 0 {
 		return nil
@@ -72,10 +71,11 @@ func (db *DB) InsertBulkLeftMany(table string, head []string, left int, col []in
 		rows = append(rows, "(?, ?)")
 		values = append(values, left, c)
 	}
-	q := fmt.Sprintf("INSERT OR IGNORE INTO %q (%s) VALUES %s",
+	q := fmt.Sprintf("INSERT INTO %q (%s) VALUES %s ON CONFLICT (%s) DO NOTHING",
 		table,
 		strings.Join(head, ", "),
 		strings.Join(rows, ", "),
+		strings.Join(head, ", "),
 	)
 	return db.Exec(q, values...).Error
 }
@@ -611,45 +611,3 @@ func join[T fmt.Stringer](in []T, sep string) string {
 	}
 	return strings.Join(strs, sep)
 }
-
-func Dump(ctx context.Context, db *gorm.DB, to string) error {
-	dest, err := New(to, url.Values{})
-	if err != nil {
-		return fmt.Errorf("create dest db: %w", err)
-	}
-	defer dest.Close()
-
-	connSrc, err := db.DB().Conn(ctx)
-	if err != nil {
-		return fmt.Errorf("getting src raw conn: %w", err)
-	}
-	defer connSrc.Close()
-
-	connDest, err := dest.DB.DB().Conn(ctx)
-	if err != nil {
-		return fmt.Errorf("getting dest raw conn: %w", err)
-	}
-	defer connDest.Close()
-
-	err = connDest.Raw(func(connDest interface{}) error {
-		return connSrc.Raw(func(connSrc interface{}) error {
-			connDestq := connDest.(*sqlite3.SQLiteConn)
-			connSrcq := connSrc.(*sqlite3.SQLiteConn)
-			bk, err := connDestq.Backup("main", connSrcq, "main")
-			if err != nil {
-				return fmt.Errorf("create backup db: %w", err)
-			}
-			for done, _ := bk.Step(-1); !done; { //nolint: revive
-			}
-			if err := bk.Finish(); err != nil {
-				return fmt.Errorf("finishing dump: %w", err)
-			}
-			return nil
-		})
-	})
-	if err != nil {
-		return fmt.Errorf("backing up: %w", err)
-	}
-
-	return nil
-}

db/db_test.go

@@ -22,7 +22,7 @@ func TestGetSetting(t *testing.T) {
 	key := SettingKey(randKey())
 	value := "howdy"
 
-	testDB, err := NewMock()
+	testDB, err := New("sqlite3://:memory:")
 	if err != nil {
 		t.Fatalf("error creating db: %v", err)
 	}

db/migrations.go

@@ -2,7 +2,6 @@
 package db
 
 import (
-	"context"
 	"errors"
 	"fmt"
 	"log"
@@ -20,7 +19,6 @@ import (
 
 type MigrationContext struct {
 	Production        bool
-	DBPath            string
 	OriginalMusicPath string
 	PlaylistsPath     string
 	PodcastsPath      string
@@ -59,7 +57,6 @@ func (db *DB) Migrate(ctx MigrationContext) error {
 		construct(ctx, "202206101425", migrateUser),
 		construct(ctx, "202207251148", migrateStarRating),
 		construct(ctx, "202211111057", migratePlaylistsQueuesToFullID),
-		constructNoTx(ctx, "202212272312", backupDBPre016),
 		construct(ctx, "202304221528", migratePlaylistsToM3U),
 		construct(ctx, "202305301718", migratePlayCountToLength),
 		construct(ctx, "202307281628", migrateAlbumArtistsMany2Many),
@@ -179,12 +176,18 @@ func migrateAddGenre(tx *gorm.DB, _ MigrationContext) error {
 
 func migrateUpdateTranscodePrefIDX(tx *gorm.DB, _ MigrationContext) error {
 	var hasIDX int
-	tx.
-		Select("1").
-		Table("sqlite_master").
-		Where("type = ?", "index").
-		Where("name = ?", "idx_user_id_client").
-		Count(&hasIDX)
+	if tx.Dialect().GetName() == "sqlite3" {
+		tx.Select("1").
+			Table("sqlite_master").
+			Where("type = ?", "index").
+			Where("name = ?", "idx_user_id_client").
+			Count(&hasIDX)
+	} else if tx.Dialect().GetName() == "postgres" {
+		tx.Select("1").
+			Table("pg_indexes").
+			Where("indexname = ?", "idx_user_id_client").
+			Count(&hasIDX)
+	}
 	if hasIDX == 1 {
 		// index already exists
 		return nil
@@ -461,9 +464,15 @@ func migratePlaylistsQueuesToFullID(tx *gorm.DB, _ MigrationContext) error {
 	if err := step.Error; err != nil {
 		return fmt.Errorf("step migrate play_queues to full id: %w", err)
 	}
-	step = tx.Exec(`
+	if tx.Dialect().GetName() == "postgres" {
+		step = tx.Exec(`
+		UPDATE play_queues SET newcurrent=('tr-' || current)::varchar[200];
+	`)
+	} else {
+		step = tx.Exec(`
 		UPDATE play_queues SET newcurrent=('tr-' || CAST(current AS varchar(10)));
 	`)
+	}
 	if err := step.Error; err != nil {
 		return fmt.Errorf("step migrate play_queues to full id: %w", err)
 	}
@@ -590,7 +599,7 @@ func migrateAlbumArtistsMany2Many(tx *gorm.DB, _ MigrationContext) error {
 			return fmt.Errorf("step insert from albums: %w", err)
 		}
 
-		step = tx.Exec(`DROP INDEX idx_albums_tag_artist_id`)
+		step = tx.Exec(`DROP INDEX IF EXISTS idx_albums_tag_artist_id`)
 		if err := step.Error; err != nil {
 			return fmt.Errorf("step drop index: %w", err)
 		}
@@ -729,13 +738,6 @@ func migratePlaylistsPaths(tx *gorm.DB, ctx MigrationContext) error {
 	return nil
 }
 
-func backupDBPre016(tx *gorm.DB, ctx MigrationContext) error {
-	if !ctx.Production {
-		return nil
-	}
-	return Dump(context.Background(), tx, fmt.Sprintf("%s.%d.bak", ctx.DBPath, time.Now().Unix()))
-}
-
 func migrateAlbumTagArtistString(tx *gorm.DB, _ MigrationContext) error {
 	return tx.AutoMigrate(Album{}).Error
 }
@@ -770,12 +772,22 @@ func migrateArtistAppearances(tx *gorm.DB, _ MigrationContext) error {
 		return fmt.Errorf("step transfer album artists: %w", err)
 	}
 
-	step = tx.Exec(`
+	if tx.Dialect().GetName() == "sqlite3" {
+		step = tx.Exec(`
 		INSERT OR IGNORE INTO artist_appearances (artist_id, album_id)
 		SELECT track_artists.artist_id, tracks.album_id
 		FROM track_artists
 		JOIN tracks ON tracks.id=track_artists.track_id
 	`)
+	} else {
+		step = tx.Exec(`
+		INSERT INTO artist_appearances (artist_id, album_id)
+		SELECT track_artists.artist_id, tracks.album_id
+		FROM track_artists
+		JOIN tracks ON tracks.id=track_artists.track_id
+		ON CONFLICT DO NOTHING
+	`)
+	}
 	if err := step.Error; err != nil {
 		return fmt.Errorf("step transfer album artists: %w", err)
 	}
@@ -795,7 +807,7 @@ func migrateTemporaryDisplayAlbumArtist(tx *gorm.DB, _ MigrationContext) error {
 	return tx.Exec(`
 		UPDATE albums
 		SET tag_album_artist=(
-			SELECT group_concat(artists.name, ', ')
+			SELECT string_agg(artists.name, ', ')
 			FROM artists
 			JOIN album_artists ON album_artists.artist_id=artists.id AND album_artists.album_id=albums.id
 			GROUP BY album_artists.album_id