multi: add dev kvdb to sql code migration

Add the necessary code to trigger the kvdb to sql code migration in dev
builds.

config_dev.go

@@ -3,6 +3,7 @@
 package terminal
 
 import (
+	"context"
 	"fmt"
 	"path/filepath"
 
@@ -13,6 +14,7 @@ import (
 	"github.com/lightninglabs/lightning-terminal/firewalldb"
 	"github.com/lightninglabs/lightning-terminal/session"
 	"github.com/lightningnetwork/lnd/clock"
+	"github.com/lightningnetwork/lnd/lnrpc"
 	"github.com/lightningnetwork/lnd/sqldb/v2"
 )
 
@@ -87,7 +89,9 @@ func defaultDevConfig() *DevConfig {
 }
 
 // NewStores creates a new stores instance based on the chosen database backend.
-func NewStores(cfg *Config, clock clock.Clock) (*stores, error) {
+func NewStores(ctx context.Context, cfg *Config,
+	basicClient lnrpc.LightningClient, clock clock.Clock) (*stores, error) {
+
 	var (
 		networkDir = filepath.Join(cfg.LitDir, cfg.Network)
 		stores     = &stores{
@@ -114,7 +118,11 @@ func NewStores(cfg *Config, clock clock.Clock) (*stores, error) {
 
 		if !cfg.Sqlite.SkipMigrations {
 			err = sqldb.ApplyAllMigrations(
-				sqlStore, migstreams.LitdMigrationStreams,
+				sqlStore,
+				migstreams.MakeMigrationStreams(
+					ctx, basicClient, cfg.MacaroonPath,
+					clock,
+				),
 			)
 			if err != nil {
 				return stores, fmt.Errorf("error applying "+
@@ -156,7 +164,11 @@ func NewStores(cfg *Config, clock clock.Clock) (*stores, error) {
 
 		if !cfg.Postgres.SkipMigrations {
 			err = sqldb.ApplyAllMigrations(
-				sqlStore, migstreams.LitdMigrationStreams,
+				sqlStore,
+				migstreams.MakeMigrationStreams(
+					ctx, basicClient, cfg.MacaroonPath,
+					clock,
+				),
 			)
 			if err != nil {
 				return stores, fmt.Errorf("error applying "+

config_prod.go

@@ -3,6 +3,7 @@
 package terminal
 
 import (
+	"context"
 	"fmt"
 	"path/filepath"
 
@@ -10,6 +11,7 @@ import (
 	"github.com/lightninglabs/lightning-terminal/firewalldb"
 	"github.com/lightninglabs/lightning-terminal/session"
 	"github.com/lightningnetwork/lnd/clock"
+	"github.com/lightningnetwork/lnd/lnrpc"
 )
 
 // DevConfig is an empty shell struct that allows us to build without the dev
@@ -29,7 +31,9 @@ func (c *DevConfig) Validate(_, _ string) error {
 
 // NewStores creates a new instance of the stores struct using the default Bolt
 // backend since in production, this is currently the only backend supported.
-func NewStores(cfg *Config, clock clock.Clock) (*stores, error) {
+func NewStores(_ context.Context, cfg *Config,
+	_ lnrpc.LightningClient, clock clock.Clock) (*stores, error) {
+
 	networkDir := filepath.Join(cfg.LitDir, cfg.Network)
 
 	stores := &stores{

db/migrations.go

@@ -28,7 +28,8 @@ const (
 // environment.
 //
 // NOTE: This function is not located in the migstreams package to avoid
-// cyclic dependencies.
+// cyclic dependencies. This test migration stream does not run the kvdb to sql
+// migration, as we already have separate unit tests which tests the migration.
 func MakeTestMigrationStreams() []sqldb.MigrationStream {
 	migStream := sqldb.MigrationStream{
 		TrackingTableName: pgx.DefaultMigrationsTable,

db/migstreams/post_migration_callbacks_dev.go

@@ -0,0 +1,195 @@
+//go:build dev
+
+package migstreams
+
+import (
+	"context"
+	"database/sql"
+	"encoding/binary"
+	"fmt"
+	"path/filepath"
+	"time"
+
+	"github.com/golang-migrate/migrate/v4"
+	"github.com/golang-migrate/migrate/v4/database"
+	"github.com/lightninglabs/lightning-terminal/accounts"
+	"github.com/lightninglabs/lightning-terminal/db/sqlcmig6"
+	"github.com/lightninglabs/lightning-terminal/firewalldb"
+	"github.com/lightninglabs/lightning-terminal/session"
+	"github.com/lightningnetwork/lnd/clock"
+	"github.com/lightningnetwork/lnd/lnrpc"
+	"github.com/lightningnetwork/lnd/sqldb/v2"
+)
+
+// MakePostStepCallbacksMig6 turns the post migration checks into a map of post
+// step callbacks that can be used with the migrate package. The keys of the map
+// are the migration versions, and the values are the callbacks that will be
+// executed after the migration with the corresponding version is applied.
+func MakePostStepCallbacksMig6(ctx context.Context,
+	basicClient lnrpc.LightningClient, db *sqldb.BaseDB,
+	macPath string, clock clock.Clock,
+	migVersion uint) migrate.ProgrammaticMigrEntry {
+
+	mig6queries := sqlcmig6.NewForType(db, db.BackendType)
+	mig6executor := sqldb.NewTransactionExecutor(
+		db, func(tx *sql.Tx) *sqlcmig6.Queries {
+			return mig6queries.WithTx(tx)
+		},
+	)
+
+	pMigr := func(_ *migrate.Migration, _ database.Driver) error {
+		// We ignore the actual driver that's being returned here, since
+		// we use migrate.NewWithInstance() to create the migration
+		// instance from our already instantiated database backend that
+		// is also passed into this function.
+		return mig6executor.ExecTx(
+			ctx, sqldb.WriteTxOpt(),
+			func(q6 *sqlcmig6.Queries) error {
+				log.Infof("Running post migration callback "+
+					"for migration version %d", migVersion)
+
+				return kvdbToSqlMigrationCallback(
+					ctx, basicClient, macPath, db, clock,
+					q6,
+				)
+			}, sqldb.NoOpReset,
+		)
+	}
+
+	return migrate.ProgrammaticMigrEntry{
+		// We want the migration to rerun on next startup if it errors,
+		// and not set the user's db to a dirty state.
+		ResetVersionOnError: true,
+		ProgrammaticMigr:    pMigr,
+	}
+}
+
+func kvdbToSqlMigrationCallback(ctx context.Context,
+	basicClient lnrpc.LightningClient, macPath string, _ *sqldb.BaseDB,
+	clock clock.Clock, q *sqlcmig6.Queries) error {
+
+	start := time.Now()
+	log.Infof("Starting KVDB to SQL migration for all stores")
+
+	accountStore, err := accounts.NewBoltStore(
+		filepath.Dir(macPath), accounts.DBFilename, clock,
+	)
+	if err != nil {
+		return err
+	}
+
+	defer func() {
+		err := accountStore.Close()
+		if err != nil {
+			log.Errorf("Error closing bbolt account store during "+
+				"migration: %v", err)
+		}
+	}()
+
+	err = accounts.MigrateAccountStoreToSQL(ctx, accountStore.DB, q)
+	if err != nil {
+		return fmt.Errorf("error migrating account store to "+
+			"SQL: %w", err)
+	}
+
+	sessionStore, err := session.NewDB(
+		filepath.Dir(macPath), session.DBFilename,
+		clock, accountStore,
+	)
+	if err != nil {
+		return err
+	}
+
+	defer func() {
+		err := sessionStore.Close()
+		if err != nil {
+			log.Errorf("Error closing bbolt session store during "+
+				"migration: %v", err)
+		}
+	}()
+
+	err = session.MigrateSessionStoreToSQL(ctx, sessionStore.DB, q)
+	if err != nil {
+		return fmt.Errorf("error migrating session store to "+
+			"SQL: %w", err)
+	}
+
+	firewallStore, err := firewalldb.NewBoltDB(
+		filepath.Dir(macPath), firewalldb.DBFilename,
+		sessionStore, accountStore, clock,
+	)
+	if err != nil {
+		return err
+	}
+
+	defer func() {
+		err := firewallStore.Close()
+		if err != nil {
+			log.Errorf("Error closing bbolt rules store during "+
+				"migration: %v", err)
+		}
+	}()
+
+	// We'll fetch the macaroonIDList from `lnd` next. Note that since lnd's
+	// RPC servers may not have been fully started yet if the execution of
+	// accounts and session migration were really quick, we poll the request
+	// up to 10 times with a 0.5 second delay between the attempts. This
+	// should be a sufficient amount of time for the RPC servers to start.
+	const (
+		maxListMacaroonIDAttempts = 10
+		listMacaroonIDRetryDelay  = 500 * time.Millisecond
+	)
+
+	var macaroonIDList *lnrpc.ListMacaroonIDsResponse
+	for i := 1; i <= maxListMacaroonIDAttempts; i++ {
+		macaroonIDList, err = basicClient.ListMacaroonIDs(
+			ctx, &lnrpc.ListMacaroonIDsRequest{},
+		)
+		if err == nil {
+			break
+		}
+
+		if i == maxListMacaroonIDAttempts {
+			return fmt.Errorf("error listing macaroon IDs when "+
+				"migrating stores to SQL after %d attempts: %w",
+				maxListMacaroonIDAttempts, err)
+		}
+
+		log.Warnf("Failed to list macaroon IDs when migrating "+
+			"stores to SQL (attempt %d/%d), retrying in %v: %v",
+			i, maxListMacaroonIDAttempts, listMacaroonIDRetryDelay,
+			err)
+
+		select {
+		case <-ctx.Done():
+			return fmt.Errorf("context canceled while retrying "+
+				"to list macaroon IDs: %w", ctx.Err())
+		case <-time.After(listMacaroonIDRetryDelay):
+		}
+	}
+
+	log.Infof("Successfully listed macaroon IDs during store migration.")
+
+	var macRootKeyIDs [][]byte
+	if macaroonIDList != nil {
+		for _, rootKeyID := range macaroonIDList.RootKeyIds {
+			rootKeyBytes := make([]byte, 8)
+			binary.BigEndian.PutUint64(rootKeyBytes[:], rootKeyID)
+
+			macRootKeyIDs = append(macRootKeyIDs, rootKeyBytes)
+		}
+	}
+
+	err = firewalldb.MigrateFirewallDBToSQL(
+		ctx, firewallStore.DB, q, macRootKeyIDs,
+	)
+	if err != nil {
+		return fmt.Errorf("error migrating firewalldb store "+
+			"to SQL: %w", err)
+	}
+
+	log.Infof("Succesfully migrated all KVDB stores to SQL in: %v",
+		time.Since(start))
+
+	return nil
+}

db/migstreams/sql_migrations.go

@@ -3,16 +3,24 @@
 package migstreams
 
 import (
+	"context"
+
 	"github.com/golang-migrate/migrate/v4"
 	"github.com/golang-migrate/migrate/v4/database/pgx/v5"
 	"github.com/lightninglabs/lightning-terminal/db"
+	"github.com/lightningnetwork/lnd/clock"
+	"github.com/lightningnetwork/lnd/lnrpc"
 	"github.com/lightningnetwork/lnd/sqldb/v2"
 )
 
-var (
-	// LitdMigrationStream defines the SQL migration stream used to create
+// MakeMigrationStreams creates the migration streams for production
+// environments.
+func MakeMigrationStreams(_ context.Context, _ lnrpc.LightningClient, _ string,
+	_ clock.Clock) []sqldb.MigrationStream {
+
+	// migStream defines the SQL migration stream used to create
 	// and upgrade LiT's SQL schema.
-	LitdMigrationStream = sqldb.MigrationStream{
+	migStream := sqldb.MigrationStream{
 		TrackingTableName: pgx.DefaultMigrationsTable,
 		SQLFileDirectory:  "sqlc/migrations",
 		SQLFiles:          db.SqlSchemas,
@@ -30,5 +38,6 @@ var (
 			return make(map[uint]migrate.ProgrammaticMigrEntry), nil
 		},
 	}
-	LitdMigrationStreams = []sqldb.MigrationStream{LitdMigrationStream}
-)
+
+	return []sqldb.MigrationStream{migStream}
+}

db/migstreams/sql_migrations_dev.go

@@ -3,15 +3,33 @@
 package migstreams
 
 import (
+	"context"
+
 	"github.com/golang-migrate/migrate/v4"
 	"github.com/golang-migrate/migrate/v4/database/pgx/v5"
 	"github.com/lightninglabs/lightning-terminal/db"
+	"github.com/lightningnetwork/lnd/clock"
+	"github.com/lightningnetwork/lnd/lnrpc"
 	"github.com/lightningnetwork/lnd/sqldb/v2"
 )
 
-var (
+const (
+	// KVDBtoSQLMigVersion is the version of the migration that migrates the
+	// kvdb to the sql database.
+	//
+	// TODO: When this the kvdb to sql migration goes live into prod, this
+	// should be moved to non dev db/migrations.go file, and this constant
+	// value should be updated to reflect the real migration number.
+	KVDBtoSQLMigVersion = 1
+)
+
+// MakeMigrationStreams creates the migration streams for the dev environments.
+func MakeMigrationStreams(ctx context.Context,
+	basicClient lnrpc.LightningClient, macPath string,
+	clock clock.Clock) []sqldb.MigrationStream {
+
 	// Create the prod migration stream.
-	migStream = sqldb.MigrationStream{
+	migStream := sqldb.MigrationStream{
 		TrackingTableName: pgx.DefaultMigrationsTable,
 		SQLFileDirectory:  "sqlc/migrations",
 		SQLFiles:          db.SqlSchemas,
@@ -31,7 +49,7 @@ var (
 	}
 
 	// Create the dev migration stream.
-	migStreamDev = sqldb.MigrationStream{
+	migStreamDev := sqldb.MigrationStream{
 		TrackingTableName: pgx.DefaultMigrationsTable + "_dev",
 		SQLFileDirectory:  "sqlc/migrations_dev",
 		SQLFiles:          db.SqlSchemas,
@@ -46,8 +64,24 @@ var (
 		MakeProgrammaticMigrations: func(db *sqldb.BaseDB) (
 			map[uint]migrate.ProgrammaticMigrEntry, error) {
 
-			return make(map[uint]migrate.ProgrammaticMigrEntry), nil
+			// Any Callbacks added to this map will be executed when
+			// after the dev migration number for the uint key in
+			// the map has been applied. If no entry exists for a
+			// given uint, then no callback will be executed for
+			// that migration number. This is useful for adding a
+			// code migration step as a callback to be run
+			// after a specific migration of a given number has been
+			// applied.
+			res := make(map[uint]migrate.ProgrammaticMigrEntry)
+
+			res[KVDBtoSQLMigVersion] = MakePostStepCallbacksMig6(
+				ctx, basicClient, db, macPath, clock,
+				KVDBtoSQLMigVersion,
+			)
+
+			return res, nil
 		},
 	}
-	LitdMigrationStreams = []sqldb.MigrationStream{migStream, migStreamDev}
-)
+
+	return []sqldb.MigrationStream{migStream, migStreamDev}
+}

terminal.go

@@ -729,7 +729,9 @@ func (g *LightningTerminal) start(ctx context.Context) error {
 		return fmt.Errorf("could not start LND")
 	}
 
-	g.stores, err = NewStores(g.cfg, clock.NewDefaultClock())
+	g.stores, err = NewStores(
+		ctx, g.cfg, g.basicClient, clock.NewDefaultClock(),
+	)
 	if err != nil {
 		return fmt.Errorf("could not create stores: %v", err)
 	}