diff --git a/cmd/db_migrate/migrate_test.go b/cmd/db_migrate/migrate_test.go index 98e99929..240751cb 100644 --- a/cmd/db_migrate/migrate_test.go +++ b/cmd/db_migrate/migrate_test.go @@ -115,7 +115,7 @@ func getTestSqliteURI(dbIndex int) string { return uri } - return fmt.Sprintf("file:testmemdb%d?mode=memory&cache=shared&_txlock=immediate", dbIndex) + return fmt.Sprintf("file:testmemdb%d?mode=memory&cache=shared&_txlock=immediate&_foreign_keys=1", dbIndex) } func getTestPostgresURI() string { diff --git a/db/db.go b/db/db.go index f1934be5..e088c38a 100644 --- a/db/db.go +++ b/db/db.go @@ -10,6 +10,7 @@ import ( gorm_logger "gorm.io/gorm/logger" "github.com/getAlby/hub/db/migrations" + sqlite_wrapper "github.com/getAlby/hub/db/sqlite-wrapper" "github.com/getAlby/hub/logger" ) @@ -49,14 +50,29 @@ func NewDBWithConfig(cfg *Config) (*gorm.DB, error) { } } else { sqliteURI := cfg.URI - // avoid SQLITE_BUSY errors with _txlock=IMMEDIATE - if !strings.Contains(sqliteURI, "_txlock=") { - sqliteURI = sqliteURI + "?_txlock=IMMEDIATE" + + // apply pragma if we're not running the tests + if !strings.Contains(sqliteURI, "?mode=memory") { + // see https://github.com/mattn/go-sqlite3?tab=readme-ov-file#connection-string + // _txlock: avoid SQLITE_BUSY errors with _txlock=immediate + // _auto_vacuum: properly cleanup disk when deleting records with auto_vacuum=1 + // _busy_timeout: avoid SQLITE_BUSY errors with 5 second lock timeout + // _journal_mode: enables write-ahead log so that your reads do not block writes and vice-versa. + // _synchronous: sqlite will sync less frequently and be more performant, still safe to use because of the enabled WAL mode + // _cache_size: 20MB memory cache + sqliteURI = sqliteURI + "?_txlock=immediate&_foreign_keys=1&_auto_vacuum=1&_busy_timeout=5000&_journal_mode=WAL&_synchronous=NORMAL&_cache_size=-20000" } + + driverName := sqlite_wrapper.Sqlite3WrapperDriverName + if cfg.DriverName != "" { + driverName = cfg.DriverName + } + sqliteConfig := sqlite.Config{ - DriverName: cfg.DriverName, + DriverName: driverName, DSN: sqliteURI, } + var err error ret, err = newSqliteDB(sqliteConfig, gormConfig) if err != nil { @@ -80,46 +96,6 @@ func newSqliteDB(sqliteConfig sqlite.Config, gormConfig *gorm.Config) (*gorm.DB, if err != nil { return nil, err } - err = gormDB.Exec("PRAGMA foreign_keys = ON", nil).Error - if err != nil { - return nil, err - } - - // properly cleanup disk when deleting records - err = gormDB.Exec("PRAGMA auto_vacuum = FULL", nil).Error - if err != nil { - return nil, err - } - - // avoid SQLITE_BUSY errors with 5 second lock timeout - err = gormDB.Exec("PRAGMA busy_timeout = 5000", nil).Error - if err != nil { - return nil, err - } - - // enables write-ahead log so that your reads do not block writes and vice-versa. - err = gormDB.Exec("PRAGMA journal_mode = WAL", nil).Error - if err != nil { - return nil, err - } - - // sqlite will sync less frequently and be more performant, still safe to use because of the enabled WAL mode - err = gormDB.Exec("PRAGMA synchronous = NORMAL", nil).Error - if err != nil { - return nil, err - } - - // 20MB memory cache - err = gormDB.Exec("PRAGMA cache_size = -20000", nil).Error - if err != nil { - return nil, err - } - - // moves temporary tables from disk into RAM, speeds up performance a lot - err = gormDB.Exec("PRAGMA temp_store = memory", nil).Error - if err != nil { - return nil, err - } return gormDB, nil } diff --git a/db/migrations/202508041712_delete_non_cascade_deleted_records.go b/db/migrations/202508041712_delete_non_cascade_deleted_records.go new file mode 100644 index 00000000..e47f3628 --- /dev/null +++ b/db/migrations/202508041712_delete_non_cascade_deleted_records.go @@ -0,0 +1,48 @@ +package migrations + +import ( + _ "embed" + + "github.com/go-gormigrate/gormigrate/v2" + "gorm.io/gorm" +) + +var _202508041712_delete_non_cascade_deleted_records = &gormigrate.Migration{ + ID: "202508041712_delete_non_cascade_deleted_records", + Migrate: func(db *gorm.DB) error { + + // the following tables have ON DELETE CASCADE + // which was not being applied due to PRAGMA foreign_keys = ON; + // not applying to all DB connections: + // - app_permissions -> apps + // - request_events -> apps + // - response_events -> request_events + + if err := db.Transaction(func(tx *gorm.DB) error { + + if err := tx.Exec(`DELETE FROM app_permissions +WHERE app_id NOT IN (SELECT id FROM apps);`).Error; err != nil { + return err + } + + if err := tx.Exec(`DELETE FROM request_events +WHERE app_id NOT IN (SELECT id FROM apps);`).Error; err != nil { + return err + } + + if err := tx.Exec(`DELETE FROM response_events +WHERE request_id NOT IN (SELECT id FROM request_events);`).Error; err != nil { + return err + } + + return nil + }); err != nil { + return err + } + + return nil + }, + Rollback: func(tx *gorm.DB) error { + return nil + }, +} diff --git a/db/migrations/202508041737_postgres_amount_bigint.go b/db/migrations/202508041737_postgres_amount_bigint.go new file mode 100644 index 00000000..9262e850 --- /dev/null +++ b/db/migrations/202508041737_postgres_amount_bigint.go @@ -0,0 +1,30 @@ +package migrations + +import ( + _ "embed" + + "github.com/go-gormigrate/gormigrate/v2" + "gorm.io/gorm" +) + +var _202508041737_postgres_amount_bigint = &gormigrate.Migration{ + ID: "202508041737_postgres_amount_bigint", + Migrate: func(db *gorm.DB) error { + + // sqlite works fine but postgres integers are only 4 bytes + // amounts are in msats (= max ~2.1M sats) + if db.Dialector.Name() != "postgres" { + return nil + } + + if err := db.Exec(`ALTER TABLE transactions +ALTER COLUMN amount_msat TYPE bigint;`).Error; err != nil { + return err + } + + return nil + }, + Rollback: func(tx *gorm.DB) error { + return nil + }, +} diff --git a/db/migrations/migrate.go b/db/migrations/migrate.go index d7ad380c..7bee84ec 100644 --- a/db/migrations/migrate.go +++ b/db/migrations/migrate.go @@ -30,6 +30,8 @@ func Migrate(gormDB *gorm.DB) error { _202412212345_fix_types, _202504231037_add_indexes, _202505091314_hold_invoices, + _202508041712_delete_non_cascade_deleted_records, + _202508041737_postgres_amount_bigint, }) return m.Migrate() diff --git a/db/sqlite-wrapper/driver.go b/db/sqlite-wrapper/driver.go new file mode 100644 index 00000000..a26d1cd6 --- /dev/null +++ b/db/sqlite-wrapper/driver.go @@ -0,0 +1,22 @@ +package sqlite_wrapper + +import ( + "database/sql" + + "github.com/mattn/go-sqlite3" +) + +const Sqlite3WrapperDriverName = "sqlite3_wrapper" + +func init() { + // We need to set the temp_store setting on every connection, including + // those that are implicitly opened by Go's database/sql package. + // Unfortunately, this setting cannot be provided in the DSN; therefore + // we execute the PRAGMA statement in the sqlite3's connection hook. + sql.Register(Sqlite3WrapperDriverName, &sqlite3.SQLiteDriver{ + ConnectHook: func(conn *sqlite3.SQLiteConn) error { + _, err := conn.Exec("PRAGMA temp_store = MEMORY", nil) + return err + }, + }) +} diff --git a/db/test/db_test.go b/db/test/db_test.go new file mode 100644 index 00000000..4d352bef --- /dev/null +++ b/db/test/db_test.go @@ -0,0 +1,33 @@ +package test + +import ( + "strconv" + "testing" + + "github.com/sirupsen/logrus" + + "github.com/getAlby/hub/logger" + "github.com/getAlby/hub/tests/db" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestTempStorePragmaIsApplied(t *testing.T) { + logger.Init(strconv.Itoa(int(logrus.DebugLevel))) + gormDb, err := db.NewDB(t) + require.NoError(t, err) + defer db.CloseDB(gormDb) + + if gormDb.Dialector.Name() != "sqlite" { + t.Skip("Skipping non-sqlite dialector") + } + + var result string + err = gormDb.Raw("PRAGMA temp_store").Scan(&result).Error + require.NoError(t, err) + + // PRAGMA temp_store = MEMORY + // MEMORY = 2 + assert.Equal(t, "2", result) +} diff --git a/nip47/controllers/create_connection_controller_test.go b/nip47/controllers/create_connection_controller_test.go index dd6d881a..612979b5 100644 --- a/nip47/controllers/create_connection_controller_test.go +++ b/nip47/controllers/create_connection_controller_test.go @@ -21,9 +21,9 @@ import ( func TestHandleCreateConnectionEvent(t *testing.T) { ctx := context.TODO() svc, err := tests.CreateTestService(t) - svc.Cfg.SetUpdate("LNBackendType", config.LDKBackendType, "") require.NoError(t, err) defer svc.Remove() + svc.Cfg.SetUpdate("LNBackendType", config.LDKBackendType, "") pairingSecretKey := nostr.GeneratePrivateKey() pairingPublicKey, err := nostr.GetPublicKey(pairingSecretKey) diff --git a/nip47/controllers/multi_pay_invoice_controller_test.go b/nip47/controllers/multi_pay_invoice_controller_test.go index 7133a0a2..d582ac94 100644 --- a/nip47/controllers/multi_pay_invoice_controller_test.go +++ b/nip47/controllers/multi_pay_invoice_controller_test.go @@ -126,6 +126,10 @@ func TestHandleMultiPayInvoiceEvent_Success(t *testing.T) { assert.Equal(t, 2, len(responses)) + for i := 0; i < len(responses); i++ { + require.Nil(t, responses[i].Error) + } + // we can't guarantee which request was processed first // so swap them if they are back to front if dTags[0].GetFirst([]string{"d"}).Value() != paymentHashes[0] { @@ -141,7 +145,6 @@ func TestHandleMultiPayInvoiceEvent_Success(t *testing.T) { for i := 0; i < len(responses); i++ { assert.Equal(t, preimages[i], responses[i].Result.(payResponse).Preimage) assert.Equal(t, paymentHashes[i], dTags[i].GetFirst([]string{"d"}).Value()) - assert.Nil(t, responses[i].Error) } } diff --git a/tests/db/test_db.go b/tests/db/test_db.go index 5239b8bc..35b83624 100644 --- a/tests/db/test_db.go +++ b/tests/db/test_db.go @@ -11,6 +11,7 @@ import ( "gorm.io/gorm" "github.com/getAlby/hub/db" + "github.com/getAlby/hub/logger" ) const defaultTestDB = "test.db" @@ -26,7 +27,16 @@ func GetTestDatabaseURI() string { } func NewDB(t *testing.T) (*gorm.DB, error) { - return NewDBWithURI(t, GetTestDatabaseURI()) + dbUri := GetTestDatabaseURI() + + if dbUri == defaultTestDB { + //in case the file was not removed in the last run, remove it before starting the test + logger.Logger.WithField("uri", defaultTestDB).Info("removing test db") + os.Remove(defaultTestDB) + } + + logger.Logger.WithField("uri", dbUri).Info("Creating new test DB with URI") + return NewDBWithURI(t, dbUri) } func NewDBWithURI(t *testing.T, uri string) (*gorm.DB, error) { @@ -68,6 +78,7 @@ func CloseDB(d *gorm.DB) { } if GetTestDatabaseURI() == defaultTestDB { + logger.Logger.WithField("uri", defaultTestDB).Info("removing test db") os.Remove(defaultTestDB) } } diff --git a/transactions/receive_keysend_test.go b/transactions/receive_keysend_test.go index 242d3bfa..21ccd23d 100644 --- a/transactions/receive_keysend_test.go +++ b/transactions/receive_keysend_test.go @@ -56,7 +56,7 @@ func TestReceiveKeysendWithCustomKey(t *testing.T) { transactionsService.ConsumeEvent(ctx, &event, map[string]interface{}{}) transaction, err := transactionsService.LookupTransaction(ctx, tx.PaymentHash, nil, svc.LNClient, nil) - assert.NoError(t, err) + require.NoError(t, err) assert.Equal(t, app.ID, *transaction.AppId) assert.Equal(t, uint(1), app.ID) } diff --git a/transactions/transactions_service.go b/transactions/transactions_service.go index 7a35b2a6..7817ae05 100644 --- a/transactions/transactions_service.go +++ b/transactions/transactions_service.go @@ -498,7 +498,7 @@ func (svc *transactionsService) SendKeysend(ctx context.Context, amount uint64, if selfPayment { // for keysend self-payments we need to create an incoming payment at the time of the payment - recipientAppId := svc.getAppIdFromCustomRecords(customRecords) + recipientAppId := svc.getAppIdFromCustomRecords(customRecords, svc.db) dbTransaction := db.Transaction{ AppId: recipientAppId, RequestEventId: nil, // it is related to this request but for a different app @@ -790,7 +790,7 @@ func (svc *transactionsService) ConsumeEvent(ctx context.Context, event *events. description = extractedDescription } // find app by custom key/value records - appId = svc.getAppIdFromCustomRecords(customRecords) + appId = svc.getAppIdFromCustomRecords(customRecords, tx) } var expiresAt *time.Time if lnClientTransaction.ExpiresAt != nil { @@ -1165,7 +1165,7 @@ func (svc *transactionsService) getDescriptionFromCustomRecords(customRecords [] return description } -func (svc *transactionsService) getAppIdFromCustomRecords(customRecords []lnclient.TLVRecord) *uint { +func (svc *transactionsService) getAppIdFromCustomRecords(customRecords []lnclient.TLVRecord, tx *gorm.DB) *uint { app := db.App{} for _, record := range customRecords { if record.Type == CustomKeyTlvType { @@ -1179,7 +1179,7 @@ func (svc *transactionsService) getAppIdFromCustomRecords(customRecords []lnclie logger.Logger.WithError(err).Error("Failed to parse custom key TLV record as number") continue } - err = svc.db.Take(&app, &db.App{ + err = tx.Take(&app, &db.App{ ID: uint(customValue), }).Error if err != nil { @@ -1362,15 +1362,15 @@ func (svc *transactionsService) markTransactionSettled(tx *gorm.DB, dbTransactio }) if dbTransaction.Type == constants.TRANSACTION_TYPE_OUTGOING && dbTransaction.AppId != nil { - svc.checkBudgetUsage(dbTransaction) + svc.checkBudgetUsage(dbTransaction, tx) } return dbTransaction, nil } -func (svc *transactionsService) checkBudgetUsage(dbTransaction *db.Transaction) { +func (svc *transactionsService) checkBudgetUsage(dbTransaction *db.Transaction, gormTransaction *gorm.DB) { var app db.App - result := svc.db.Limit(1).Find(&app, &db.App{ + result := gormTransaction.Limit(1).Find(&app, &db.App{ ID: *dbTransaction.AppId, }) if result.RowsAffected == 0 { @@ -1382,7 +1382,7 @@ func (svc *transactionsService) checkBudgetUsage(dbTransaction *db.Transaction) } var appPermission db.AppPermission - result = svc.db.Limit(1).Find(&appPermission, &db.AppPermission{ + result = gormTransaction.Limit(1).Find(&appPermission, &db.AppPermission{ AppId: app.ID, Scope: constants.PAY_INVOICE_SCOPE, }) @@ -1391,7 +1391,7 @@ func (svc *transactionsService) checkBudgetUsage(dbTransaction *db.Transaction) return } - budgetUsage := queries.GetBudgetUsageSat(svc.db, &appPermission) + budgetUsage := queries.GetBudgetUsageSat(gormTransaction, &appPermission) warningUsage := uint64(math.Floor(float64(appPermission.MaxAmountSat) * 0.8)) if budgetUsage >= warningUsage && budgetUsage-dbTransaction.AmountMsat/1000 < warningUsage { svc.eventPublisher.Publish(&events.Event{