From 0c6b3c05394101066dd20dbdce91df897f80159d Mon Sep 17 00:00:00 2001 From: Viktor Torstensson Date: Fri, 11 Jul 2025 21:38:29 +0200 Subject: [PATCH] multi: introduce dev migrations When the kvdb to sql migration is initially introduced, we will want to ensure that it is only run under dev builds during the testing phase. We therefore introduce the functionality to have separate dev migrations, which are only included in a separate migration stream that is used only in dev builds. Note that these dev migrations are currently not included in the `sqlc.yaml` file, to ensure that the main sqlc models doesn't include the dev migrations. --- db/migrations.go | 30 ++++++++++- db/migstreams/sql_migrations.go | 2 + db/migstreams/sql_migrations_dev.go | 53 +++++++++++++++++++ db/schemas.go | 2 +- .../000001_dev_test_migration.down.sql | 1 + .../000001_dev_test_migration.up.sql | 1 + scripts/gen_sqlc_docker.sh | 4 +- 7 files changed, 89 insertions(+), 4 deletions(-) create mode 100644 db/migstreams/sql_migrations_dev.go create mode 100644 db/sqlc/migrations_dev/000001_dev_test_migration.down.sql create mode 100644 db/sqlc/migrations_dev/000001_dev_test_migration.up.sql diff --git a/db/migrations.go b/db/migrations.go index b7e366f1..70713252 100644 --- a/db/migrations.go +++ b/db/migrations.go @@ -13,6 +13,15 @@ const ( // // NOTE: This MUST be updated when a new migration is added. LatestMigrationVersion = 5 + + // LatestDevMigrationVersion is the latest dev migration version of the + // database. This is used to implement downgrade protection for the + // daemon. This represents the latest number used in the migrations_dev + // directory. + // + // NOTE: This MUST be updated when a migration is added or removed, from + // the migrations_dev directory. + LatestDevMigrationVersion = 1 ) // MakeTestMigrationStreams creates the migration streams for the unit test @@ -40,5 +49,24 @@ func MakeTestMigrationStreams() []sqldb.MigrationStream { }, } - return []sqldb.MigrationStream{migStream} + migStreamDev := sqldb.MigrationStream{ + TrackingTableName: pgx.DefaultMigrationsTable + "_dev", + SQLFileDirectory: "sqlc/migrations_dev", + SQLFiles: SqlSchemas, + + // LatestMigrationVersion is the latest migration version of the + // dev migrations database. This is used to implement downgrade + // protection for the daemon. + // + // NOTE: This MUST be updated when a new dev migration is added. + LatestMigrationVersion: LatestDevMigrationVersion, + + MakeProgrammaticMigrations: func(db *sqldb.BaseDB) ( + map[uint]migrate.ProgrammaticMigrEntry, error) { + + return make(map[uint]migrate.ProgrammaticMigrEntry), nil + }, + } + + return []sqldb.MigrationStream{migStream, migStreamDev} } diff --git a/db/migstreams/sql_migrations.go b/db/migstreams/sql_migrations.go index 256c3826..4364fb30 100644 --- a/db/migstreams/sql_migrations.go +++ b/db/migstreams/sql_migrations.go @@ -1,3 +1,5 @@ +//go:build !dev + package migstreams import ( diff --git a/db/migstreams/sql_migrations_dev.go b/db/migstreams/sql_migrations_dev.go new file mode 100644 index 00000000..516864e8 --- /dev/null +++ b/db/migstreams/sql_migrations_dev.go @@ -0,0 +1,53 @@ +//go:build dev + +package migstreams + +import ( + "github.com/golang-migrate/migrate/v4" + "github.com/golang-migrate/migrate/v4/database/pgx/v5" + "github.com/lightninglabs/lightning-terminal/db" + "github.com/lightningnetwork/lnd/sqldb/v2" +) + +var ( + // Create the prod migration stream. + migStream = sqldb.MigrationStream{ + TrackingTableName: pgx.DefaultMigrationsTable, + SQLFileDirectory: "sqlc/migrations", + SQLFiles: db.SqlSchemas, + + // LatestMigrationVersion is the latest migration version of the + // database. This is used to implement downgrade protection for + // the daemon. + // + // NOTE: This MUST be updated when a new migration is added. + LatestMigrationVersion: db.LatestMigrationVersion, + + MakeProgrammaticMigrations: func(db *sqldb.BaseDB) ( + map[uint]migrate.ProgrammaticMigrEntry, error) { + + return make(map[uint]migrate.ProgrammaticMigrEntry), nil + }, + } + + // Create the dev migration stream. + migStreamDev = sqldb.MigrationStream{ + TrackingTableName: pgx.DefaultMigrationsTable + "_dev", + SQLFileDirectory: "sqlc/migrations_dev", + SQLFiles: db.SqlSchemas, + + // LatestMigrationVersion is the latest migration version of the + // dev migrations database. This is used to implement downgrade + // protection for the daemon. + // + // NOTE: This MUST be updated when a new dev migration is added. + LatestMigrationVersion: db.LatestDevMigrationVersion, + + MakeProgrammaticMigrations: func(db *sqldb.BaseDB) ( + map[uint]migrate.ProgrammaticMigrEntry, error) { + + return make(map[uint]migrate.ProgrammaticMigrEntry), nil + }, + } + LitdMigrationStreams = []sqldb.MigrationStream{migStream, migStreamDev} +) diff --git a/db/schemas.go b/db/schemas.go index dce7fa84..565fb561 100644 --- a/db/schemas.go +++ b/db/schemas.go @@ -5,5 +5,5 @@ import ( _ "embed" ) -//go:embed sqlc/migrations/*.*.sql +//go:embed sqlc/migration*/*.*.sql var SqlSchemas embed.FS diff --git a/db/sqlc/migrations_dev/000001_dev_test_migration.down.sql b/db/sqlc/migrations_dev/000001_dev_test_migration.down.sql new file mode 100644 index 00000000..0d246b2d --- /dev/null +++ b/db/sqlc/migrations_dev/000001_dev_test_migration.down.sql @@ -0,0 +1 @@ +-- Comment to ensure the file created and picked up in the migration stream. \ No newline at end of file diff --git a/db/sqlc/migrations_dev/000001_dev_test_migration.up.sql b/db/sqlc/migrations_dev/000001_dev_test_migration.up.sql new file mode 100644 index 00000000..0d246b2d --- /dev/null +++ b/db/sqlc/migrations_dev/000001_dev_test_migration.up.sql @@ -0,0 +1 @@ +-- Comment to ensure the file created and picked up in the migration stream. \ No newline at end of file diff --git a/scripts/gen_sqlc_docker.sh b/scripts/gen_sqlc_docker.sh index 16db97f2..3d93f37f 100755 --- a/scripts/gen_sqlc_docker.sh +++ b/scripts/gen_sqlc_docker.sh @@ -5,7 +5,7 @@ set -e # restore_files is a function to restore original schema files. restore_files() { echo "Restoring SQLite bigint patch..." - for file in db/sqlc/migrations/*.up.sql.bak; do + for file in db/sqlc/{migrations,migrations_dev}/*.up.sql.bak; do mv "$file" "${file%.bak}" done } @@ -30,7 +30,7 @@ GOMODCACHE=$(go env GOMODCACHE) # source schema SQL files to use "BIGINT PRIMARY KEY" instead of "INTEGER # PRIMARY KEY". echo "Applying SQLite bigint patch..." -for file in db/sqlc/migrations/*.up.sql; do +for file in db/sqlc/{migrations,migrations_dev}/*.up.sql; do echo "Patching $file" sed -i.bak -E 's/INTEGER PRIMARY KEY/BIGINT PRIMARY KEY/g' "$file" done