Fix: sqlite connection parameters, add migrations to delete orphaned records and change postgres transaction amount to bigint (#1462)

* fix: sqlite pragma statement not applying to all sqlite connections

* fix: add migration to delete orphaned records

* fix: migrate postgres transaction amount column to bigint

* chore: simplify migration, update comment

* fix: sqlite uri when running tests

* fix: check that breaks tests

* fix: db locking within transaction

* fix: sqlite3 driver wrapper to execute PRAGMAs on each new connection (#1464)

* fix: sqlite3 driver wrapper to execute PRAGMAs on each new connection

* chore: add test to check temp_store value

---------

Co-authored-by: Roland Bewick <roland.bewick@gmail.com>

---------

Co-authored-by: Roman D <roman@dmitrienko.com>

cmd/db_migrate/migrate_test.go

@@ -115,7 +115,7 @@ func getTestSqliteURI(dbIndex int) string {
 		return uri
 	}
 
-	return fmt.Sprintf("file:testmemdb%d?mode=memory&cache=shared&_txlock=immediate", dbIndex)
+	return fmt.Sprintf("file:testmemdb%d?mode=memory&cache=shared&_txlock=immediate&_foreign_keys=1", dbIndex)
 }
 
 func getTestPostgresURI() string {

db/db.go

@@ -10,6 +10,7 @@ import (
 	gorm_logger "gorm.io/gorm/logger"
 
 	"github.com/getAlby/hub/db/migrations"
+	sqlite_wrapper "github.com/getAlby/hub/db/sqlite-wrapper"
 	"github.com/getAlby/hub/logger"
 )
 
@@ -49,14 +50,29 @@ func NewDBWithConfig(cfg *Config) (*gorm.DB, error) {
 		}
 	} else {
 		sqliteURI := cfg.URI
-		// avoid SQLITE_BUSY errors with _txlock=IMMEDIATE
-		if !strings.Contains(sqliteURI, "_txlock=") {
-			sqliteURI = sqliteURI + "?_txlock=IMMEDIATE"
+
+		// apply pragma if we're not running the tests
+		if !strings.Contains(sqliteURI, "?mode=memory") {
+			// see https://github.com/mattn/go-sqlite3?tab=readme-ov-file#connection-string
+			// _txlock: avoid SQLITE_BUSY errors with _txlock=immediate
+			// _auto_vacuum: properly cleanup disk when deleting records with auto_vacuum=1
+			// _busy_timeout: avoid SQLITE_BUSY errors with 5 second lock timeout
+			// _journal_mode: enables write-ahead log so that your reads do not block writes and vice-versa.
+			// _synchronous: sqlite will sync less frequently and be more performant, still safe to use because of the enabled WAL mode
+			// _cache_size: 20MB memory cache
+			sqliteURI = sqliteURI + "?_txlock=immediate&_foreign_keys=1&_auto_vacuum=1&_busy_timeout=5000&_journal_mode=WAL&_synchronous=NORMAL&_cache_size=-20000"
 		}
+
+		driverName := sqlite_wrapper.Sqlite3WrapperDriverName
+		if cfg.DriverName != "" {
+			driverName = cfg.DriverName
+		}
+
 		sqliteConfig := sqlite.Config{
-			DriverName: cfg.DriverName,
+			DriverName: driverName,
 			DSN:        sqliteURI,
 		}
+
 		var err error
 		ret, err = newSqliteDB(sqliteConfig, gormConfig)
 		if err != nil {
@@ -80,46 +96,6 @@ func newSqliteDB(sqliteConfig sqlite.Config, gormConfig *gorm.Config) (*gorm.DB,
 	if err != nil {
 		return nil, err
 	}
-	err = gormDB.Exec("PRAGMA foreign_keys = ON", nil).Error
-	if err != nil {
-		return nil, err
-	}
-
-	// properly cleanup disk when deleting records
-	err = gormDB.Exec("PRAGMA auto_vacuum = FULL", nil).Error
-	if err != nil {
-		return nil, err
-	}
-
-	// avoid SQLITE_BUSY errors with 5 second lock timeout
-	err = gormDB.Exec("PRAGMA busy_timeout = 5000", nil).Error
-	if err != nil {
-		return nil, err
-	}
-
-	// enables write-ahead log so that your reads do not block writes and vice-versa.
-	err = gormDB.Exec("PRAGMA journal_mode = WAL", nil).Error
-	if err != nil {
-		return nil, err
-	}
-
-	// sqlite will sync less frequently and be more performant, still safe to use because of the enabled WAL mode
-	err = gormDB.Exec("PRAGMA synchronous = NORMAL", nil).Error
-	if err != nil {
-		return nil, err
-	}
-
-	// 20MB memory cache
-	err = gormDB.Exec("PRAGMA cache_size = -20000", nil).Error
-	if err != nil {
-		return nil, err
-	}
-
-	// moves temporary tables from disk into RAM, speeds up performance a lot
-	err = gormDB.Exec("PRAGMA temp_store = memory", nil).Error
-	if err != nil {
-		return nil, err
-	}
 
 	return gormDB, nil
 }

db/migrations/202508041712_delete_non_cascade_deleted_records.go

@@ -0,0 +1,48 @@
+package migrations
+
+import (
+	_ "embed"
+
+	"github.com/go-gormigrate/gormigrate/v2"
+	"gorm.io/gorm"
+)
+
+var _202508041712_delete_non_cascade_deleted_records = &gormigrate.Migration{
+	ID: "202508041712_delete_non_cascade_deleted_records",
+	Migrate: func(db *gorm.DB) error {
+
+		// the following tables have ON DELETE CASCADE
+		// which was not being applied due to PRAGMA foreign_keys = ON;
+		// not applying to all DB connections:
+		// - app_permissions -> apps
+		// - request_events -> apps
+		// - response_events -> request_events
+
+		if err := db.Transaction(func(tx *gorm.DB) error {
+
+			if err := tx.Exec(`DELETE FROM app_permissions
+WHERE app_id NOT IN (SELECT id FROM apps);`).Error; err != nil {
+				return err
+			}
+
+			if err := tx.Exec(`DELETE FROM request_events
+WHERE app_id NOT IN (SELECT id FROM apps);`).Error; err != nil {
+				return err
+			}
+
+			if err := tx.Exec(`DELETE FROM response_events
+WHERE request_id NOT IN (SELECT id FROM request_events);`).Error; err != nil {
+				return err
+			}
+
+			return nil
+		}); err != nil {
+			return err
+		}
+
+		return nil
+	},
+	Rollback: func(tx *gorm.DB) error {
+		return nil
+	},
+}

db/migrations/202508041737_postgres_amount_bigint.go

@@ -0,0 +1,30 @@
+package migrations
+
+import (
+	_ "embed"
+
+	"github.com/go-gormigrate/gormigrate/v2"
+	"gorm.io/gorm"
+)
+
+var _202508041737_postgres_amount_bigint = &gormigrate.Migration{
+	ID: "202508041737_postgres_amount_bigint",
+	Migrate: func(db *gorm.DB) error {
+
+		// sqlite works fine but postgres integers are only 4 bytes
+		// amounts are in msats (= max ~2.1M sats)
+		if db.Dialector.Name() != "postgres" {
+			return nil
+		}
+
+		if err := db.Exec(`ALTER TABLE transactions
+ALTER COLUMN amount_msat TYPE bigint;`).Error; err != nil {
+			return err
+		}
+
+		return nil
+	},
+	Rollback: func(tx *gorm.DB) error {
+		return nil
+	},
+}

db/migrations/migrate.go

@@ -30,6 +30,8 @@ func Migrate(gormDB *gorm.DB) error {
 		_202412212345_fix_types,
 		_202504231037_add_indexes,
 		_202505091314_hold_invoices,
+		_202508041712_delete_non_cascade_deleted_records,
+		_202508041737_postgres_amount_bigint,
 	})
 
 	return m.Migrate()

db/sqlite-wrapper/driver.go

@@ -0,0 +1,22 @@
+package sqlite_wrapper
+
+import (
+	"database/sql"
+
+	"github.com/mattn/go-sqlite3"
+)
+
+const Sqlite3WrapperDriverName = "sqlite3_wrapper"
+
+func init() {
+	// We need to set the temp_store setting on every connection, including
+	// those that are implicitly opened by Go's database/sql package.
+	// Unfortunately, this setting cannot be provided in the DSN; therefore
+	// we execute the PRAGMA statement in the sqlite3's connection hook.
+	sql.Register(Sqlite3WrapperDriverName, &sqlite3.SQLiteDriver{
+		ConnectHook: func(conn *sqlite3.SQLiteConn) error {
+			_, err := conn.Exec("PRAGMA temp_store = MEMORY", nil)
+			return err
+		},
+	})
+}

db/test/db_test.go

@@ -0,0 +1,33 @@
+package test
+
+import (
+	"strconv"
+	"testing"
+
+	"github.com/sirupsen/logrus"
+
+	"github.com/getAlby/hub/logger"
+	"github.com/getAlby/hub/tests/db"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestTempStorePragmaIsApplied(t *testing.T) {
+	logger.Init(strconv.Itoa(int(logrus.DebugLevel)))
+	gormDb, err := db.NewDB(t)
+	require.NoError(t, err)
+	defer db.CloseDB(gormDb)
+
+	if gormDb.Dialector.Name() != "sqlite" {
+		t.Skip("Skipping non-sqlite dialector")
+	}
+
+	var result string
+	err = gormDb.Raw("PRAGMA temp_store").Scan(&result).Error
+	require.NoError(t, err)
+
+	// PRAGMA temp_store = MEMORY
+	// MEMORY = 2
+	assert.Equal(t, "2", result)
+}

nip47/controllers/create_connection_controller_test.go

@@ -21,9 +21,9 @@ import (
 func TestHandleCreateConnectionEvent(t *testing.T) {
 	ctx := context.TODO()
 	svc, err := tests.CreateTestService(t)
-	svc.Cfg.SetUpdate("LNBackendType", config.LDKBackendType, "")
 	require.NoError(t, err)
 	defer svc.Remove()
+	svc.Cfg.SetUpdate("LNBackendType", config.LDKBackendType, "")
 
 	pairingSecretKey := nostr.GeneratePrivateKey()
 	pairingPublicKey, err := nostr.GetPublicKey(pairingSecretKey)

nip47/controllers/multi_pay_invoice_controller_test.go

@@ -126,6 +126,10 @@ func TestHandleMultiPayInvoiceEvent_Success(t *testing.T) {
 
 	assert.Equal(t, 2, len(responses))
 
+	for i := 0; i < len(responses); i++ {
+		require.Nil(t, responses[i].Error)
+	}
+
 	// we can't guarantee which request was processed first
 	// so swap them if they are back to front
 	if dTags[0].GetFirst([]string{"d"}).Value() != paymentHashes[0] {
@@ -141,7 +145,6 @@ func TestHandleMultiPayInvoiceEvent_Success(t *testing.T) {
 	for i := 0; i < len(responses); i++ {
 		assert.Equal(t, preimages[i], responses[i].Result.(payResponse).Preimage)
 		assert.Equal(t, paymentHashes[i], dTags[i].GetFirst([]string{"d"}).Value())
-		assert.Nil(t, responses[i].Error)
 	}
 }
 

tests/db/test_db.go

@@ -11,6 +11,7 @@ import (
 	"gorm.io/gorm"
 
 	"github.com/getAlby/hub/db"
+	"github.com/getAlby/hub/logger"
 )
 
 const defaultTestDB = "test.db"
@@ -26,7 +27,16 @@ func GetTestDatabaseURI() string {
 }
 
 func NewDB(t *testing.T) (*gorm.DB, error) {
-	return NewDBWithURI(t, GetTestDatabaseURI())
+	dbUri := GetTestDatabaseURI()
+
+	if dbUri == defaultTestDB {
+		//in case the file was not removed in the last run, remove it before starting the test
+		logger.Logger.WithField("uri", defaultTestDB).Info("removing test db")
+		os.Remove(defaultTestDB)
+	}
+
+	logger.Logger.WithField("uri", dbUri).Info("Creating new test DB with URI")
+	return NewDBWithURI(t, dbUri)
 }
 
 func NewDBWithURI(t *testing.T, uri string) (*gorm.DB, error) {
@@ -68,6 +78,7 @@ func CloseDB(d *gorm.DB) {
 	}
 
 	if GetTestDatabaseURI() == defaultTestDB {
+		logger.Logger.WithField("uri", defaultTestDB).Info("removing test db")
 		os.Remove(defaultTestDB)
 	}
 }

transactions/receive_keysend_test.go

@@ -56,7 +56,7 @@ func TestReceiveKeysendWithCustomKey(t *testing.T) {
 	transactionsService.ConsumeEvent(ctx, &event, map[string]interface{}{})
 
 	transaction, err := transactionsService.LookupTransaction(ctx, tx.PaymentHash, nil, svc.LNClient, nil)
-	assert.NoError(t, err)
+	require.NoError(t, err)
 	assert.Equal(t, app.ID, *transaction.AppId)
 	assert.Equal(t, uint(1), app.ID)
 }

transactions/transactions_service.go

@@ -498,7 +498,7 @@ func (svc *transactionsService) SendKeysend(ctx context.Context, amount uint64,
 
 	if selfPayment {
 		// for keysend self-payments we need to create an incoming payment at the time of the payment
-		recipientAppId := svc.getAppIdFromCustomRecords(customRecords)
+		recipientAppId := svc.getAppIdFromCustomRecords(customRecords, svc.db)
 		dbTransaction := db.Transaction{
 			AppId:          recipientAppId,
 			RequestEventId: nil, // it is related to this request but for a different app
@@ -790,7 +790,7 @@ func (svc *transactionsService) ConsumeEvent(ctx context.Context, event *events.
 						description = extractedDescription
 					}
 					// find app by custom key/value records
-					appId = svc.getAppIdFromCustomRecords(customRecords)
+					appId = svc.getAppIdFromCustomRecords(customRecords, tx)
 				}
 				var expiresAt *time.Time
 				if lnClientTransaction.ExpiresAt != nil {
@@ -1165,7 +1165,7 @@ func (svc *transactionsService) getDescriptionFromCustomRecords(customRecords []
 	return description
 }
 
-func (svc *transactionsService) getAppIdFromCustomRecords(customRecords []lnclient.TLVRecord) *uint {
+func (svc *transactionsService) getAppIdFromCustomRecords(customRecords []lnclient.TLVRecord, tx *gorm.DB) *uint {
 	app := db.App{}
 	for _, record := range customRecords {
 		if record.Type == CustomKeyTlvType {
@@ -1179,7 +1179,7 @@ func (svc *transactionsService) getAppIdFromCustomRecords(customRecords []lnclie
 				logger.Logger.WithError(err).Error("Failed to parse custom key TLV record as number")
 				continue
 			}
-			err = svc.db.Take(&app, &db.App{
+			err = tx.Take(&app, &db.App{
 				ID: uint(customValue),
 			}).Error
 			if err != nil {
@@ -1362,15 +1362,15 @@ func (svc *transactionsService) markTransactionSettled(tx *gorm.DB, dbTransactio
 	})
 
 	if dbTransaction.Type == constants.TRANSACTION_TYPE_OUTGOING && dbTransaction.AppId != nil {
-		svc.checkBudgetUsage(dbTransaction)
+		svc.checkBudgetUsage(dbTransaction, tx)
 	}
 
 	return dbTransaction, nil
 }
 
-func (svc *transactionsService) checkBudgetUsage(dbTransaction *db.Transaction) {
+func (svc *transactionsService) checkBudgetUsage(dbTransaction *db.Transaction, gormTransaction *gorm.DB) {
 	var app db.App
-	result := svc.db.Limit(1).Find(&app, &db.App{
+	result := gormTransaction.Limit(1).Find(&app, &db.App{
 		ID: *dbTransaction.AppId,
 	})
 	if result.RowsAffected == 0 {
@@ -1382,7 +1382,7 @@ func (svc *transactionsService) checkBudgetUsage(dbTransaction *db.Transaction)
 	}
 
 	var appPermission db.AppPermission
-	result = svc.db.Limit(1).Find(&appPermission, &db.AppPermission{
+	result = gormTransaction.Limit(1).Find(&appPermission, &db.AppPermission{
 		AppId: app.ID,
 		Scope: constants.PAY_INVOICE_SCOPE,
 	})
@@ -1391,7 +1391,7 @@ func (svc *transactionsService) checkBudgetUsage(dbTransaction *db.Transaction)
 		return
 	}
 
-	budgetUsage := queries.GetBudgetUsageSat(svc.db, &appPermission)
+	budgetUsage := queries.GetBudgetUsageSat(gormTransaction, &appPermission)
 	warningUsage := uint64(math.Floor(float64(appPermission.MaxAmountSat) * 0.8))
 	if budgetUsage >= warningUsage && budgetUsage-dbTransaction.AmountMsat/1000 < warningUsage {
 		svc.eventPublisher.Publish(&events.Event{