multi: introduce dev migrations

When the kvdb to sql migration is initially introduced, we will want to
ensure that it is only run under dev builds during the testing phase.

We therefore introduce the functionality to have separate dev
migrations, which are only included in a separate migration stream that
is used only in dev builds.

Note that these dev migrations are currently not included in the
`sqlc.yaml` file, to ensure that the main sqlc models doesn't include
the dev migrations.

db/migrations.go

@@ -13,6 +13,15 @@ const (
 	//
 	// NOTE: This MUST be updated when a new migration is added.
 	LatestMigrationVersion = 5
+
+	// LatestDevMigrationVersion is the latest dev migration version of the
+	// database. This is used to implement downgrade protection for the
+	// daemon. This represents the latest number used in the migrations_dev
+	// directory.
+	//
+	// NOTE: This MUST be updated when a migration is added or removed, from
+	// the migrations_dev directory.
+	LatestDevMigrationVersion = 1
 )
 
 // MakeTestMigrationStreams creates the migration streams for the unit test
@@ -40,5 +49,24 @@ func MakeTestMigrationStreams() []sqldb.MigrationStream {
 		},
 	}
 
-	return []sqldb.MigrationStream{migStream}
+	migStreamDev := sqldb.MigrationStream{
+		TrackingTableName: pgx.DefaultMigrationsTable + "_dev",
+		SQLFileDirectory:  "sqlc/migrations_dev",
+		SQLFiles:          SqlSchemas,
+
+		// LatestMigrationVersion is the latest migration version of the
+		// dev migrations database. This is used to implement downgrade
+		// protection for the daemon.
+		//
+		// NOTE: This MUST be updated when a new dev migration is added.
+		LatestMigrationVersion: LatestDevMigrationVersion,
+
+		MakeProgrammaticMigrations: func(db *sqldb.BaseDB) (
+			map[uint]migrate.ProgrammaticMigrEntry, error) {
+
+			return make(map[uint]migrate.ProgrammaticMigrEntry), nil
+		},
+	}
+
+	return []sqldb.MigrationStream{migStream, migStreamDev}
 }

db/migstreams/sql_migrations.go

@@ -1,3 +1,5 @@
+//go:build !dev
+
 package migstreams
 
 import (

db/migstreams/sql_migrations_dev.go

@@ -0,0 +1,53 @@
+//go:build dev
+
+package migstreams
+
+import (
+	"github.com/golang-migrate/migrate/v4"
+	"github.com/golang-migrate/migrate/v4/database/pgx/v5"
+	"github.com/lightninglabs/lightning-terminal/db"
+	"github.com/lightningnetwork/lnd/sqldb/v2"
+)
+
+var (
+	// Create the prod migration stream.
+	migStream = sqldb.MigrationStream{
+		TrackingTableName: pgx.DefaultMigrationsTable,
+		SQLFileDirectory:  "sqlc/migrations",
+		SQLFiles:          db.SqlSchemas,
+
+		// LatestMigrationVersion is the latest migration version of the
+		// database.  This is used to implement downgrade protection for
+		// the daemon.
+		//
+		// NOTE: This MUST be updated when a new migration is added.
+		LatestMigrationVersion: db.LatestMigrationVersion,
+
+		MakeProgrammaticMigrations: func(db *sqldb.BaseDB) (
+			map[uint]migrate.ProgrammaticMigrEntry, error) {
+
+			return make(map[uint]migrate.ProgrammaticMigrEntry), nil
+		},
+	}
+
+	// Create the dev migration stream.
+	migStreamDev = sqldb.MigrationStream{
+		TrackingTableName: pgx.DefaultMigrationsTable + "_dev",
+		SQLFileDirectory:  "sqlc/migrations_dev",
+		SQLFiles:          db.SqlSchemas,
+
+		// LatestMigrationVersion is the latest migration version of the
+		// dev migrations database. This is used to implement downgrade
+		// protection for the daemon.
+		//
+		// NOTE: This MUST be updated when a new dev migration is added.
+		LatestMigrationVersion: db.LatestDevMigrationVersion,
+
+		MakeProgrammaticMigrations: func(db *sqldb.BaseDB) (
+			map[uint]migrate.ProgrammaticMigrEntry, error) {
+
+			return make(map[uint]migrate.ProgrammaticMigrEntry), nil
+		},
+	}
+	LitdMigrationStreams = []sqldb.MigrationStream{migStream, migStreamDev}
+)

db/schemas.go

@@ -5,5 +5,5 @@ import (
 	_ "embed"
 )
 
-//go:embed sqlc/migrations/*.*.sql
+//go:embed sqlc/migration*/*.*.sql
 var SqlSchemas embed.FS

db/sqlc/migrations_dev/000001_dev_test_migration.down.sql

@@ -0,0 +1 @@
+-- Comment to ensure the file created and picked up in the migration stream.

db/sqlc/migrations_dev/000001_dev_test_migration.up.sql

@@ -0,0 +1 @@
+-- Comment to ensure the file created and picked up in the migration stream.

scripts/gen_sqlc_docker.sh

@@ -5,7 +5,7 @@ set -e
 # restore_files is a function to restore original schema files.
 restore_files() {
 	echo "Restoring SQLite bigint patch..."
-	for file in db/sqlc/migrations/*.up.sql.bak; do
+	for file in db/sqlc/{migrations,migrations_dev}/*.up.sql.bak; do
 		mv "$file" "${file%.bak}"
 	done
 }
@@ -30,7 +30,7 @@ GOMODCACHE=$(go env GOMODCACHE)
 # source schema SQL files to use "BIGINT PRIMARY KEY" instead of "INTEGER
 # PRIMARY KEY".
 echo "Applying SQLite bigint patch..."
-for file in db/sqlc/migrations/*.up.sql; do
+for file in db/sqlc/{migrations,migrations_dev}/*.up.sql; do
 	echo "Patching $file"
 	sed -i.bak -E 's/INTEGER PRIMARY KEY/BIGINT PRIMARY KEY/g' "$file"
 done