diff --git a/README.md b/README.md index 1572f1dc..a80cdc04 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ The application can run in two modes: Ideally the app runs 24/7 (on a node, VPS or always-online desktop/laptop machine) so it can be connected to a lightning address and receive online payments. -## Run on Alby Cloud +## Learn more about Alby Hub Visit [albyhub.com](https://albyhub.com) to learn more and get started and get Alby Hub running in minutes. @@ -207,6 +207,12 @@ Migration of the database is currently experimental. Please make a backup before go run cmd/db_migrate/main.go -from .data/nwc.db -to postgresql://myuser:mypass@localhost:5432/nwc +#### Migration from Postgres to Sqlite + +No manual steps are needed: create a migration file from Settings -> Migrate Alby Hub. The contents of the Postgres database will automatically be copied into a Sqlite database which is included in the migration file. Alternatively, run the migration tool manually: + +go run cmd/db_migrate/main.go -from postgresql://myuser:mypass@localhost:5432/nwc -to .data/nwc.db + ## Node-specific backend parameters - `ENABLE_ADVANCED_SETUP`: set to `false` to force a specific backend type (combined with backend parameters below) diff --git a/api/api.go b/api/api.go index 55634a34..1cc227f3 100644 --- a/api/api.go +++ b/api/api.go @@ -1519,6 +1519,8 @@ func (api *api) GetInfo(ctx context.Context) (*InfoResponse, error) { info.LdkVssEnabled = ldkVssEnabled == "true" info.JitChannelsEnabled = jitChannelsEnabled != "false" info.VssSupported = backendType == config.LDKBackendType && api.cfg.GetEnv().LDKVssUrl != "" + info.LdkVssUrl = api.cfg.GetEnv().LDKVssUrl + info.DatabaseType = api.db.Dialector.Name() info.SupportsBolt12 = backendType == config.LDKBackendType || backendType == config.CLNBackendType info.AutoUnlockPasswordEnabled = autoUnlockPassword != "" info.AutoUnlockPasswordSupported = api.cfg.GetEnv().IsDefaultClientId() diff --git a/api/backup.go b/api/backup.go index 9d89ef7b..60db76b7 100644 --- a/api/backup.go +++ b/api/backup.go @@ -38,8 +38,9 @@ func (api *api) CreateBackup(unlockPassword string, w io.Writer) error { return errors.New("Please disable auto-unlock before using this feature") } - if api.db.Dialector.Name() != "sqlite" { - return errors.New("Migration with non-sqlite backend is currently not supported") + dbBackend := api.db.Dialector.Name() + if dbBackend != "sqlite" && dbBackend != "postgres" { + return fmt.Errorf("migration with %s backend is currently not supported", dbBackend) } workDir, err := filepath.Abs(api.cfg.GetEnv().Workdir) @@ -76,6 +77,50 @@ func (api *api) CreateBackup(unlockPassword string, w io.Writer) error { return errors.New("failed to remove oauth access token") } + // Locate the main database file. + dbFilePath := api.cfg.GetEnv().DatabaseUri + + if dbBackend == "postgres" { + // The migration file must contain a sqlite database, so copy the + // contents of the postgres database into a temporary sqlite database + // and add that to the archive instead. + dbFilePath = filepath.Join(workDir, "migration.db") + + removeConvertedDb := func() { + for _, path := range []string{dbFilePath, dbFilePath + "-wal", dbFilePath + "-shm"} { + if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) { + logger.Logger.WithError(err).WithField("path", path).Error("Failed to remove converted database file") + } + } + } + // Remove stale files from a previously failed migration attempt. + removeConvertedDb() + defer removeConvertedDb() + + logger.Logger.WithField("path", dbFilePath).Info("Copying postgres database to sqlite") + sqliteDb, err := db.NewDB(dbFilePath, api.cfg.GetEnv().LogDBQueries) + if err != nil { + logger.Logger.WithError(err).Error("Failed to create sqlite database for migration") + return fmt.Errorf("failed to create sqlite database for migration: %w", err) + } + + err = db.MigrateDB(api.db, sqliteDb) + if err != nil { + logger.Logger.WithError(err).Error("Failed to copy database contents to sqlite") + if stopErr := db.Stop(sqliteDb); stopErr != nil { + logger.Logger.WithError(stopErr).Error("Failed to stop sqlite database") + } + return fmt.Errorf("failed to copy database contents to sqlite: %w", err) + } + + // Close the sqlite database to checkpoint the WAL before archiving it. + err = db.Stop(sqliteDb) + if err != nil { + logger.Logger.WithError(err).Error("Failed to stop sqlite database") + return fmt.Errorf("failed to close sqlite database: %w", err) + } + } + // Closing the database leaves the service in an inconsistent state, // but that should not be a problem since the app is not expected // to be used after its data is exported. @@ -126,8 +171,6 @@ func (api *api) CreateBackup(unlockPassword string, w io.Writer) error { return err } - // Locate the main database file. - dbFilePath := api.cfg.GetEnv().DatabaseUri // Add the database file to the archive. logger.Logger.WithField("nwc.db", dbFilePath).Info("adding nwc db to zip") err = addFileToZip(dbFilePath, "nwc.db") @@ -245,7 +288,7 @@ func (api *api) RestoreBackup(unlockPassword string, r io.Reader) error { // ensure no -shm or -wal files exist as they will stop the restore for _, filename := range []string{"nwc.db", "nwc.db-shm", "nwc.db-wal"} { err = os.Remove(filepath.Join(workDir, filename)) - if err != nil { + if err != nil && !errors.Is(err, os.ErrNotExist) { logger.Logger.WithError(err).WithField("filename", filename).Error("failed to remove old nwc db file before restore") } } diff --git a/api/backup_test.go b/api/backup_test.go new file mode 100644 index 00000000..b41eea19 --- /dev/null +++ b/api/backup_test.go @@ -0,0 +1,106 @@ +package api + +import ( + "archive/zip" + "bytes" + "io" + "os" + "path/filepath" + "strconv" + "testing" + + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" + "gorm.io/datatypes" + + "github.com/getAlby/hub/config" + "github.com/getAlby/hub/db" + "github.com/getAlby/hub/logger" + test_db "github.com/getAlby/hub/tests/db" + "github.com/getAlby/hub/tests/mocks" +) + +// TestCreateBackup creates a backup from the test database (sqlite by +// default, postgres when TEST_DATABASE_URI is set) and verifies that the +// archive contains a valid sqlite database with the expected data. +func TestCreateBackup(t *testing.T) { + logger.Init(strconv.Itoa(int(logrus.DebugLevel))) + + workDir := t.TempDir() + + gormDB, err := test_db.NewDB(t) + require.NoError(t, err) + defer test_db.CloseDB(gormDB) + + appConfig := &config.AppConfig{ + Workdir: workDir, + DatabaseUri: test_db.GetTestDatabaseURI(), + } + cfg, err := config.NewConfig(appConfig, gormDB) + require.NoError(t, err) + + app := &db.App{ + Name: "test", + AppPubkey: "2b7dea2866958f17c568cf024e113db7a3baa9c253a9016889196b8d0b11c7ae", + Metadata: datatypes.JSON("{}"), + } + require.NoError(t, gormDB.Create(app).Error) + + lnClient := mocks.NewMockLNClient(t) + lnClient.On("GetStorageDir").Return("", nil) + lnClient.On("ResetRouter", "ALL").Return(nil) + + svc := mocks.NewMockService(t) + svc.On("GetLNClient").Return(lnClient) + svc.On("StopApp").Return() + + albyOAuthSvc := mocks.NewMockAlbyOAuthService(t) + albyOAuthSvc.On("RemoveOAuthAccessToken").Return(nil) + + theAPI := &api{ + db: gormDB, + cfg: cfg, + svc: svc, + albyOAuthSvc: albyOAuthSvc, + } + + unlockPassword := "" + + var buf bytes.Buffer + err = theAPI.CreateBackup(unlockPassword, &buf) + require.NoError(t, err) + + // The temporary database created when converting from postgres must + // not be left behind in the working directory. + entries, err := os.ReadDir(workDir) + require.NoError(t, err) + require.Empty(t, entries) + + cr, err := decryptingReader(&buf, unlockPassword) + require.NoError(t, err) + decrypted, err := io.ReadAll(cr) + require.NoError(t, err) + + zr, err := zip.NewReader(bytes.NewReader(decrypted), int64(len(decrypted))) + require.NoError(t, err) + + dbFile, err := zr.Open("nwc.db") + require.NoError(t, err) + dbContents, err := io.ReadAll(dbFile) + require.NoError(t, err) + require.NoError(t, dbFile.Close()) + + restoredPath := filepath.Join(workDir, "restored.db") + require.NoError(t, os.WriteFile(restoredPath, dbContents, 0600)) + + restoredDB, err := db.NewDB(restoredPath, false) + require.NoError(t, err) + defer func() { + require.NoError(t, db.Stop(restoredDB)) + }() + + var restoredApp db.App + require.NoError(t, restoredDB.First(&restoredApp).Error) + require.Equal(t, app.Name, restoredApp.Name) + require.Equal(t, app.AppPubkey, restoredApp.AppPubkey) +} diff --git a/api/models.go b/api/models.go index eda20676..aadfd0ed 100644 --- a/api/models.go +++ b/api/models.go @@ -317,7 +317,9 @@ type InfoResponse struct { Network string `json:"network"` EnableAdvancedSetup bool `json:"enableAdvancedSetup"` LdkVssEnabled bool `json:"ldkVssEnabled"` + LdkVssUrl string `json:"ldkVssUrl"` VssSupported bool `json:"vssSupported"` + DatabaseType string `json:"databaseType"` StartupState string `json:"startupState"` StartupError string `json:"startupError"` StartupErrorTime time.Time `json:"startupErrorTime"` diff --git a/cmd/db_migrate/main.go b/cmd/db_migrate/main.go index c10f4a3e..4207a4c6 100644 --- a/cmd/db_migrate/main.go +++ b/cmd/db_migrate/main.go @@ -2,9 +2,7 @@ package main import ( "flag" - "fmt" "os" - "slices" "strconv" "github.com/sirupsen/logrus" @@ -14,18 +12,6 @@ import ( "github.com/getAlby/hub/logger" ) -var expectedTables = []string{ - "apps", - "app_permissions", - "request_events", - "response_events", - "transactions", - "swaps", - "user_configs", - "migrations", - "forwards", -} - func main() { var fromDSN, toDSN string @@ -64,54 +50,29 @@ func main() { } defer stopDB(toDB) - // Migrations are applied to both the source and the target DB, so - // schemas should be equal at this point. - err = checkSchema(fromDB) - if err != nil { - logger.Logger.WithError(err).Error("database schema check failed; the migration tool may be outdated") - os.Exit(1) - } - - // Check if VSS is enabled in the source database - var vssConfig db.UserConfig - result := fromDB.Where("key = ?", "LdkVssEnabled").First(&vssConfig) - if result.Error != nil { - if result.Error == gorm.ErrRecordNotFound { - logger.Logger.Error("LdkVssEnabled config not found in source DB. Migration will not proceed.") - } else { - logger.Logger.WithError(result.Error).Error("failed to query LdkVssEnabled config from source DB") + // When migrating to Postgres (e.g. a cloud deployment) the node data must + // be stored in VSS, since only the database is migrated by this tool. + if toDB.Dialector.Name() == "postgres" { + var vssConfig db.UserConfig + result := fromDB.Where("key = ?", "LdkVssEnabled").First(&vssConfig) + if result.Error != nil { + if result.Error == gorm.ErrRecordNotFound { + logger.Logger.Error("LdkVssEnabled config not found in source DB. Migration will not proceed.") + } else { + logger.Logger.WithError(result.Error).Error("failed to query LdkVssEnabled config from source DB") + } + os.Exit(1) } - os.Exit(1) - } - if vssConfig.Value != "true" { - logger.Logger.Error("VSS is not enabled in the source DB (LdkVssEnabled is not 'true'). Migration will not proceed.") - os.Exit(1) - } - logger.Logger.Info("LdkVssEnabled check passed.") - - // NOTE: we assume that excess request events have already been cleaned up due to the background task - // and only a maximum of ~1000 remain. - logger.Logger.Info("Deleting orphaned request events.") - err = fromDB.Exec("DELETE FROM request_events WHERE app_id NOT IN (SELECT id FROM apps);").Error - - if err != nil { - logger.Logger.WithError(err).Error("failed to delete orphaned request events") - os.Exit(1) - } - - // NOTE: we assume that excess response events have already been cleaned up due to the background task - // and only a maximum of ~1000 remain. - logger.Logger.Info("Deleting orphaned response events.") - err = fromDB.Exec("DELETE FROM response_events WHERE request_id NOT IN (SELECT id FROM request_events);").Error - - if err != nil { - logger.Logger.WithError(err).Error("failed to delete orphaned response events") - os.Exit(1) + if vssConfig.Value != "true" { + logger.Logger.Error("VSS is not enabled in the source DB (LdkVssEnabled is not 'true'). Migration will not proceed.") + os.Exit(1) + } + logger.Logger.Info("LdkVssEnabled check passed.") } logger.Logger.Info("migrating...") - err = migrateDB(fromDB, toDB) + err = db.MigrateDB(fromDB, toDB) if err != nil { logger.Logger.WithError(err).Error("failed to migrate database") os.Exit(1) @@ -119,175 +80,3 @@ func main() { logger.Logger.Info("migration complete") } - -func migrateDB(from, to *gorm.DB) error { - tx := to.Begin() - defer tx.Rollback() - - if err := tx.Error; err != nil { - return fmt.Errorf("failed to start transaction: %w", err) - } - - // Table migration order matters: referenced tables must be migrated - // before referencing tables. - - logger.Logger.Info("migrating apps...") - if err := migrateTable[db.App](from, tx); err != nil { - return fmt.Errorf("failed to migrate apps: %w", err) - } - - logger.Logger.Info("migrating app_permissions...") - if err := migrateTable[db.AppPermission](from, tx); err != nil { - return fmt.Errorf("failed to migrate app_permissions: %w", err) - } - - logger.Logger.Info("migrating request_events...") - if err := migrateTable[db.RequestEvent](from, tx); err != nil { - return fmt.Errorf("failed to migrate request_events: %w", err) - } - - logger.Logger.Info("migrating response_events...") - if err := migrateTable[db.ResponseEvent](from, tx); err != nil { - return fmt.Errorf("failed to migrate response_events: %w", err) - } - - logger.Logger.Info("migrating transactions...") - if err := migrateTable[db.Transaction](from, tx); err != nil { - return fmt.Errorf("failed to migrate transactions: %w", err) - } - - logger.Logger.Info("migrating user_configs...") - if err := migrateTable[db.UserConfig](from, tx); err != nil { - return fmt.Errorf("failed to migrate user_configs: %w", err) - } - - if to.Dialector.Name() == "postgres" { - logger.Logger.Info("resetting sequences...") - if err := resetSequences(tx); err != nil { - return fmt.Errorf("failed to reset sequences: %w", err) - } - } - - tx.Commit() - if err := tx.Error; err != nil { - return fmt.Errorf("failed to commit transaction: %w", err) - } - - return nil -} - -func migrateTable[T any](from, to *gorm.DB) error { - var data []T - if err := from.Find(&data).Error; err != nil { - return fmt.Errorf("failed to fetch data: %w", err) - } - - if len(data) == 0 { - return nil - } - - // to avoid "failed to migrate transactions: failed to insert data: extended protocol limited to 65535 parameters" - // see https://stackoverflow.com/questions/77372430/extended-protocol-limited-to-65535-parameters-golang-gorm - // max statements is 65535 - // but it's the number of records * columns - // to be safe, using a lower value of 1000. - // this will fail if any table has more than 65 columns, which I doubt we will have - max := 1000 - for i := 0; i < len(data); i += max { - j := min(i+max, len(data)) - - if err := to.Create(data[i:j]).Error; err != nil { - return fmt.Errorf("failed to insert data: %w", err) - } - } - - return nil -} - -func checkSchema(db *gorm.DB) error { - tables, err := listTables(db) - if err != nil { - return fmt.Errorf("failed to list database tables: %w", err) - } - - for _, table := range expectedTables { - if !slices.Contains(tables, table) { - return fmt.Errorf("table missing from the database: %q", table) - } - } - - for _, table := range tables { - if !slices.Contains(expectedTables, table) { - return fmt.Errorf("unexpected table found in the database: %q", table) - } - } - - return nil -} - -func listTables(db *gorm.DB) ([]string, error) { - var query string - - switch db.Dialector.Name() { - case "sqlite": - query = "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%';" - case "postgres": - query = "SELECT tablename FROM pg_tables WHERE schemaname = 'public';" - default: - return nil, fmt.Errorf("unsupported database: %q", db.Dialector.Name()) - } - - rows, err := db.Raw(query).Rows() - if err != nil { - return nil, fmt.Errorf("failed to query table names: %w", err) - } - defer func() { - if err := rows.Close(); err != nil { - logger.Logger.WithError(err).Error("failed to close rows") - } - }() - - var tables []string - for rows.Next() { - var table string - if err := rows.Scan(&table); err != nil { - return nil, fmt.Errorf("failed to scan table name: %w", err) - } - tables = append(tables, table) - } - - return tables, nil -} - -func resetSequences(db *gorm.DB) error { - type resetReq struct { - table string - seq string - } - - resetReqs := []resetReq{ - {"apps", "apps_2_id_seq"}, - {"app_permissions", "app_permissions_2_id_seq"}, - {"request_events", "request_events_id_seq"}, - {"response_events", "response_events_id_seq"}, - {"transactions", "transactions_id_seq"}, - {"user_configs", "user_configs_id_seq"}, - } - - for _, req := range resetReqs { - if err := resetPostgresSequence(db, req.table, req.seq); err != nil { - return fmt.Errorf("failed to reset sequence %q for %q: %w", req.seq, req.table, err) - } - } - - return nil -} - -func resetPostgresSequence(db *gorm.DB, table string, seq string) error { - query := fmt.Sprintf("SELECT setval('%s', (SELECT MAX(id) FROM %s));", seq, table) - if err := db.Exec(query).Error; err != nil { - return fmt.Errorf("failed to execute setval(): %w", err) - } - - return nil -} diff --git a/cmd/db_migrate/migrate_test.go b/cmd/db_migrate/migrate_test.go index ecc6a78d..12ae54e1 100644 --- a/cmd/db_migrate/migrate_test.go +++ b/cmd/db_migrate/migrate_test.go @@ -30,40 +30,6 @@ func (e *testEnvironment) cleanup(t *testing.T) { require.NoError(t, err) } -func TestSchemaCheck(t *testing.T) { - type testCase struct { - name string - uri string - } - - tc := []testCase{ - { - name: "schema check sqlite", - uri: getTestSqliteURI(0), - }, - } - - if pgUri := getTestPostgresURI(); pgUri != "" { - tc = append(tc, testCase{ - name: "schema check postgres", - uri: pgUri, - }) - } - - logger.Init(strconv.Itoa(int(logrus.DebugLevel))) - - for _, tt := range tc { - t.Run(tt.name, func(t *testing.T) { - dbConn, err := test_db.NewDBWithURI(t, tt.uri) - require.NoError(t, err) - defer db.Stop(dbConn) - - err = checkSchema(dbConn) - require.NoError(t, err) - }) - } -} - func TestMigrate(t *testing.T) { type testCase struct { name string @@ -104,8 +70,17 @@ func TestMigrate(t *testing.T) { require.NoError(t, err) defer env.cleanup(t) - err = migrateDB(env.source, env.dest) + err = db.MigrateDB(env.source, env.dest) require.NoError(t, err) + + requireCount[db.App](t, env.dest, 2) + requireCount[db.AppPermission](t, env.dest, 2) + requireCount[db.RequestEvent](t, env.dest, 1) + requireCount[db.ResponseEvent](t, env.dest, 1) + requireCount[db.Transaction](t, env.dest, 1) + requireCount[db.Swap](t, env.dest, 1) + requireCount[db.Forward](t, env.dest, 1) + requireCount[db.UserConfig](t, env.dest, 1) }) } } @@ -200,6 +175,83 @@ func insertMockData(t *testing.T, tx *gorm.DB) { UpdatedAt: baseTime, } create(t, tx, app2Perm) + + requestEvent1 := &db.RequestEvent{ + AppId: &app1.ID, + NostrId: "a35a1ca6d1a06e08a509f2c8fe3edb2ba10811d030e2f6f3239e9f21203ac954", + ContentData: "{}", + Method: "pay_invoice", + State: "executed", + CreatedAt: baseTime, + UpdatedAt: baseTime, + } + create(t, tx, requestEvent1) + + responseEvent1 := &db.ResponseEvent{ + NostrId: "e30d55d0e4f0d5391a1a1379f1d8b7d38ad02b3554b06ca993aa8790a3153f61", + RequestId: requestEvent1.ID, + State: "confirmed", + RepliedAt: baseTime, + CreatedAt: baseTime, + UpdatedAt: baseTime, + } + create(t, tx, responseEvent1) + + transaction1 := &db.Transaction{ + AppId: &app1.ID, + RequestEventId: &requestEvent1.ID, + Type: "outgoing", + State: "settled", + AmountMsat: 21000, + FeeMsat: 1000, + PaymentRequest: "lnbc210n1invoice", + PaymentHash: "13d9764a54269fa4d5f4e7c410f4ffdbc839bbeaa2fcbb96343ca502f0c86e34", + Description: "test transaction", + Preimage: ptr("2c1ee1b464b1a1a147debe0ac0c8ce4b615f9bfa64d12a25c1c4d10ea45a5b02"), + CreatedAt: baseTime, + UpdatedAt: baseTime, + SettledAt: &baseTime, + Metadata: datatypes.JSON("{}"), + Boostagram: datatypes.JSON("{}"), + } + create(t, tx, transaction1) + + swap1 := &db.Swap{ + SwapId: "swap1", + Type: "out", + State: "success", + Invoice: "lnbc210n1swapinvoice", + SendAmountSat: 21000, + ReceiveAmountSat: 20000, + Preimage: "35a3f1a7a06a41b9ba3a1b1a8ff852e5085b3b593f8ba4677a35a1ca6d1a06e0", + PaymentHash: "e6b1a1379f1d8b7d38ad02b3554b06ca993aa8790a3153f61e30d55d0e4f0d53", + DestinationAddress: "bc1qtest", + LockupAddress: "bc1qlockup", + LockupTxId: "lockuptx", + ClaimTxId: "claimtx", + AutoSwap: false, + TimeoutBlockHeight: 900000, + BoltzPubkey: "02d1a06e08a509f2c8fe3edb2ba10811d030e2f6f3239e9f21203ac954a35a1c", + SwapTree: datatypes.JSON("{}"), + CreatedAt: baseTime, + UpdatedAt: baseTime, + } + create(t, tx, swap1) + + forward1 := &db.Forward{ + OutboundAmountForwardedMsat: 1000000, + TotalFeeEarnedMsat: 1000, + CreatedAt: baseTime, + UpdatedAt: baseTime, + } + create(t, tx, forward1) +} + +func requireCount[T any](t *testing.T, tx *gorm.DB, expected int64) { + var count int64 + var model T + require.NoError(t, tx.Model(&model).Count(&count).Error) + require.Equal(t, expected, count) } func create[T any](t *testing.T, tx *gorm.DB, v T) *gorm.DB { diff --git a/db/db_migrate.go b/db/db_migrate.go new file mode 100644 index 00000000..666e77c3 --- /dev/null +++ b/db/db_migrate.go @@ -0,0 +1,235 @@ +package db + +import ( + "fmt" + "slices" + + "gorm.io/gorm" + + "github.com/getAlby/hub/logger" +) + +var expectedTables = []string{ + "apps", + "app_permissions", + "request_events", + "response_events", + "transactions", + "swaps", + "user_configs", + "migrations", + "forwards", +} + +// MigrateDB copies all rows from one database to another. Both databases +// must have an up-to-date schema (they are checked against expectedTables). +// Orphaned request and response events are deleted from the source database +// before copying, as they would violate foreign key constraints in the +// destination database. +func MigrateDB(from, to *gorm.DB) error { + if err := checkSchema(from); err != nil { + return fmt.Errorf("source database schema check failed: %w", err) + } + + if err := checkSchema(to); err != nil { + return fmt.Errorf("destination database schema check failed: %w", err) + } + + // NOTE: we assume that excess request events have already been cleaned up due to the background task + // and only a maximum of ~1000 remain. + logger.Logger.Info("Deleting orphaned request events.") + err := from.Exec("DELETE FROM request_events WHERE app_id NOT IN (SELECT id FROM apps);").Error + if err != nil { + return fmt.Errorf("failed to delete orphaned request events: %w", err) + } + + // NOTE: we assume that excess response events have already been cleaned up due to the background task + // and only a maximum of ~1000 remain. + logger.Logger.Info("Deleting orphaned response events.") + err = from.Exec("DELETE FROM response_events WHERE request_id NOT IN (SELECT id FROM request_events);").Error + if err != nil { + return fmt.Errorf("failed to delete orphaned response events: %w", err) + } + + tx := to.Begin() + defer tx.Rollback() + + if err := tx.Error; err != nil { + return fmt.Errorf("failed to start transaction: %w", err) + } + + // Table migration order matters: referenced tables must be migrated + // before referencing tables. + + logger.Logger.Info("migrating apps...") + if err := migrateTable[App](from, tx); err != nil { + return fmt.Errorf("failed to migrate apps: %w", err) + } + + logger.Logger.Info("migrating app_permissions...") + if err := migrateTable[AppPermission](from, tx); err != nil { + return fmt.Errorf("failed to migrate app_permissions: %w", err) + } + + logger.Logger.Info("migrating request_events...") + if err := migrateTable[RequestEvent](from, tx); err != nil { + return fmt.Errorf("failed to migrate request_events: %w", err) + } + + logger.Logger.Info("migrating response_events...") + if err := migrateTable[ResponseEvent](from, tx); err != nil { + return fmt.Errorf("failed to migrate response_events: %w", err) + } + + logger.Logger.Info("migrating transactions...") + if err := migrateTable[Transaction](from, tx); err != nil { + return fmt.Errorf("failed to migrate transactions: %w", err) + } + + logger.Logger.Info("migrating swaps...") + if err := migrateTable[Swap](from, tx); err != nil { + return fmt.Errorf("failed to migrate swaps: %w", err) + } + + logger.Logger.Info("migrating forwards...") + if err := migrateTable[Forward](from, tx); err != nil { + return fmt.Errorf("failed to migrate forwards: %w", err) + } + + logger.Logger.Info("migrating user_configs...") + if err := migrateTable[UserConfig](from, tx); err != nil { + return fmt.Errorf("failed to migrate user_configs: %w", err) + } + + if to.Dialector.Name() == "postgres" { + logger.Logger.Info("resetting sequences...") + if err := resetSequences(tx); err != nil { + return fmt.Errorf("failed to reset sequences: %w", err) + } + } + + tx.Commit() + if err := tx.Error; err != nil { + return fmt.Errorf("failed to commit transaction: %w", err) + } + + return nil +} + +func migrateTable[T any](from, to *gorm.DB) error { + var data []T + if err := from.Find(&data).Error; err != nil { + return fmt.Errorf("failed to fetch data: %w", err) + } + + if len(data) == 0 { + return nil + } + + // to avoid "failed to migrate transactions: failed to insert data: extended protocol limited to 65535 parameters" + // see https://stackoverflow.com/questions/77372430/extended-protocol-limited-to-65535-parameters-golang-gorm + // max statements is 65535 + // but it's the number of records * columns + // to be safe, using a lower value of 1000. + // this will fail if any table has more than 65 columns, which I doubt we will have + max := 1000 + for i := 0; i < len(data); i += max { + j := min(i+max, len(data)) + + if err := to.Create(data[i:j]).Error; err != nil { + return fmt.Errorf("failed to insert data: %w", err) + } + } + + return nil +} + +func checkSchema(db *gorm.DB) error { + tables, err := listTables(db) + if err != nil { + return fmt.Errorf("failed to list database tables: %w", err) + } + + for _, table := range expectedTables { + if !slices.Contains(tables, table) { + return fmt.Errorf("table missing from the database: %q", table) + } + } + + for _, table := range tables { + if !slices.Contains(expectedTables, table) { + return fmt.Errorf("unexpected table found in the database: %q", table) + } + } + + return nil +} + +func listTables(db *gorm.DB) ([]string, error) { + var query string + + switch db.Dialector.Name() { + case "sqlite": + query = "SELECT name FROM sqlite_master WHERE type='table' AND name NOT LIKE 'sqlite_%';" + case "postgres": + query = "SELECT tablename FROM pg_tables WHERE schemaname = 'public';" + default: + return nil, fmt.Errorf("unsupported database: %q", db.Dialector.Name()) + } + + rows, err := db.Raw(query).Rows() + if err != nil { + return nil, fmt.Errorf("failed to query table names: %w", err) + } + defer func() { + if err := rows.Close(); err != nil { + logger.Logger.WithError(err).Error("failed to close rows") + } + }() + + var tables []string + for rows.Next() { + var table string + if err := rows.Scan(&table); err != nil { + return nil, fmt.Errorf("failed to scan table name: %w", err) + } + tables = append(tables, table) + } + + return tables, nil +} + +func resetSequences(db *gorm.DB) error { + type resetReq struct { + table string + seq string + } + + resetReqs := []resetReq{ + {"apps", "apps_2_id_seq"}, + {"app_permissions", "app_permissions_2_id_seq"}, + {"request_events", "request_events_id_seq"}, + {"response_events", "response_events_id_seq"}, + {"transactions", "transactions_id_seq"}, + {"swaps", "swaps_id_seq"}, + {"forwards", "forwards_id_seq"}, + {"user_configs", "user_configs_id_seq"}, + } + + for _, req := range resetReqs { + if err := resetPostgresSequence(db, req.table, req.seq); err != nil { + return fmt.Errorf("failed to reset sequence %q for %q: %w", req.seq, req.table, err) + } + } + + return nil +} + +func resetPostgresSequence(db *gorm.DB, table string, seq string) error { + query := fmt.Sprintf("SELECT setval('%s', (SELECT MAX(id) FROM %s));", seq, table) + if err := db.Exec(query).Error; err != nil { + return fmt.Errorf("failed to execute setval(): %w", err) + } + + return nil +} diff --git a/frontend/src/screens/MigrateNode.tsx b/frontend/src/screens/MigrateNode.tsx index 762114b8..0c97b578 100644 --- a/frontend/src/screens/MigrateNode.tsx +++ b/frontend/src/screens/MigrateNode.tsx @@ -1,4 +1,4 @@ -import { InfoIcon, TriangleAlertIcon } from "lucide-react"; +import { DatabaseIcon, InfoIcon, TriangleAlertIcon } from "lucide-react"; import React, { useState } from "react"; import { useNavigate } from "react-router"; import PasswordInput from "src/components/password/PasswordInput"; @@ -8,6 +8,13 @@ import { Button } from "src/components/ui/button"; import { LinkButton } from "src/components/ui/custom/link-button"; import { LoadingButton } from "src/components/ui/custom/loading-button"; import { Label } from "src/components/ui/label"; +import { + Tooltip, + TooltipContent, + TooltipProvider, + TooltipTrigger, +} from "src/components/ui/tooltip"; +import { useInfo } from "src/hooks/useInfo"; import { handleRequestError } from "src/utils/handleRequestError"; import { isHttpMode } from "src/utils/isHttpMode"; @@ -15,6 +22,7 @@ import { request } from "src/utils/request"; export function MigrateNode() { const navigate = useNavigate(); + const { data: info } = useInfo(); const [unlockPassword, setUnlockPassword] = React.useState(""); const [showPasswordScreen, setShowPasswordScreen] = useState(false); @@ -113,6 +121,24 @@ export function MigrateNode() { another device or server.

+ {info?.databaseType === "postgres" && ( +
+ +

Your database will be migrated

+ + + + + + + The contents of your PostgreSQL database will be copied into a + local SQLite database while the migration file is being + created. Your new Alby Hub will use this SQLite database. + + + +
+ )} {showPasswordScreen ? ( diff --git a/frontend/src/screens/settings/About.tsx b/frontend/src/screens/settings/About.tsx index e82f784b..b2e254ba 100644 --- a/frontend/src/screens/settings/About.tsx +++ b/frontend/src/screens/settings/About.tsx @@ -66,6 +66,26 @@ export function About() {

{backendTypeConfigs[info.backendType].title}

+ {info.databaseType && ( +
+

Database Storage

+

+ {info.databaseType === "postgres" + ? "PostgreSQL" + : info.databaseType === "sqlite" + ? "SQLite" + : info.databaseType} +

+
+ )} + {info.backendType === "LDK" && ( +
+

VSS

+

+ {info.ldkVssEnabled ? `Enabled (${info.ldkVssUrl})` : "Disabled"} +

+
+ )} {info.chainDataSourceType && (

Chain Data Source

diff --git a/frontend/src/screens/setup/RestoreNode.tsx b/frontend/src/screens/setup/RestoreNode.tsx index e581ac2e..e364eca4 100644 --- a/frontend/src/screens/setup/RestoreNode.tsx +++ b/frontend/src/screens/setup/RestoreNode.tsx @@ -52,9 +52,9 @@ export function RestoreNode() { />

- If you're running in the cloud, your Alby Hub will restart - automatically. Otherwise, please manually restart your Alby Hub to - finish the restore process. + If you're running in a cloud VM or linux service, your Alby Hub will + restart automatically. Otherwise, please manually restart your Alby + Hub to finish the restore process.

Waiting for restart...

@@ -158,9 +158,9 @@ export function RestoreNode() { down.

- If you're running in the cloud, your Alby Hub will restart - automatically. Otherwise, please manually restart your Alby - Hub to finish the restore process. + If you're running in a cloud VM or linux service, your Alby + Hub will restart automatically. Otherwise, please manually + restart your Alby Hub to finish the restore process.

diff --git a/frontend/src/types.ts b/frontend/src/types.ts index 2e67d98f..f828f2ca 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -158,7 +158,9 @@ export interface InfoResponse { oauthRedirect: boolean; albyAccountConnected: boolean; ldkVssEnabled: boolean; + ldkVssUrl: string; vssSupported: boolean; + databaseType: string; running: boolean; albyAuthUrl: string; nextBackupReminder: string;