GlcoinCore/lightning-terminal
1
0
Fork 0
mirror of https://github.com/lightninglabs/lightning-terminal.git synced 2026-08-13 12:33:36 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Merge pull request #1301 from ViktorT-11/2026-05-improve-migration-progress-logging

Browse source 
[sql-69] multi: improve kvdb migration progress logging
This commit is contained in:
Viktor Torstensson 2026-05-19 11:54:05 +02:00 • committed by GitHub
commit 0f31fb594b
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 88 additions and 18 deletions
Download patch file Download diff file Expand all files Collapse all files

13
accounts/sql_migration.go
View file

@ -27,6 +27,8 @@ var (
"original account")
)
const migrationProgressLogInterval = 100
// MigrateAccountStoreToSQL runs the migration of all accounts and indices from
// the KV database to the SQL database. The migration is done in a single
// transaction to ensure that all accounts are migrated or none at all.
@ -63,7 +65,10 @@ func migrateAccountsToSQL(ctx context.Context, kvStore kvdb.Backend,
return err
}
for _, kvAccount := range kvAccounts {
log.Infof("Collected %d accounts for KV to SQL migration",
len(kvAccounts))
for i, kvAccount := range kvAccounts {
migratedAccountID, err := migrateSingleAccountToSQL(
ctx, tx, kvAccount,
)
@ -102,6 +107,12 @@ func migrateAccountsToSQL(ctx context.Context, kvStore kvdb.Backend,
return fmt.Errorf("%w: %v.\n%v", ErrMigrationMismatch,
kvAccount.ID, diffText)
}
migratedCount := i + 1
if migratedCount%migrationProgressLogInterval == 0 {
log.Infof("Migrated %d/%d accounts from KV to SQL",
migratedCount, len(kvAccounts))
}
}
log.Infof("All accounts migrated from KV to SQL. Total number of "+

69
firewalldb/sql_migration.go
View file

@ -81,6 +81,8 @@ func (e *kvEntry) namespacedKey() string {
// values.
type privacyPairs = map[int64]map[string]string
const migrationProgressLogInterval = 100
// MigrateFirewallDBToSQL runs the migration of the firwalldb stores from the
// bbolt database to a SQL database. The migration is done in a single
// transaction to ensure that all rows in the stores are migrated or none at
@ -145,10 +147,13 @@ func migrateKVStoresDBToSQL(ctx context.Context, kvStore *bbolt.DB,
return fmt.Errorf("collecting all kv pairs failed: %w", err)
}
log.Infof("Collected %d KV store rows for KV to SQL migration",
len(pairs))
var insertedPairs []*sqlKvEntry
// 2) Insert all collected key-value pairs into the SQL database.
for _, entry := range pairs {
for i, entry := range pairs {
insertedPair, err := insertPair(ctx, sqlTx, sessMap, entry)
if err != nil {
return fmt.Errorf("inserting kv pair %v failed: %w",
@ -156,6 +161,12 @@ func migrateKVStoresDBToSQL(ctx context.Context, kvStore *bbolt.DB,
}
insertedPairs = append(insertedPairs, insertedPair)
migratedCount := i + 1
if migratedCount%migrationProgressLogInterval == 0 {
log.Infof("Migrated %d/%d KV store rows from KV to SQL",
migratedCount, len(pairs))
}
}
// 3) Validate the migrated values against the original values.
@ -566,8 +577,16 @@ func migratePrivacyMapperDBToSQL(ctx context.Context, kvStore *bbolt.DB,
err)
}
totalPairs := 0
for _, groupPairs := range privPairs {
totalPairs += len(groupPairs)
}
log.Infof("Collected %d privacy mapper rows across %d session "+
"groups for KV to SQL migration", totalPairs, len(privPairs))
// 2) Insert all collected privacy pairs into the SQL database.
err = insertPrivacyPairs(ctx, sqlTx, privPairs)
err = insertPrivacyPairs(ctx, sqlTx, privPairs, totalPairs)
if err != nil {
return fmt.Errorf("insertion of privacy pairs failed: %w", err)
}
@ -583,7 +602,7 @@ func migratePrivacyMapperDBToSQL(ctx context.Context, kvStore *bbolt.DB,
}
log.Infof("Migration of the privacy mapper stores to SQL completed. "+
"Total number of rows migrated: %d", len(privPairs))
"Total number of rows migrated: %d", totalPairs)
return nil
}
@ -735,14 +754,27 @@ func collectPairs(pairsBucket *bbolt.Bucket) (map[string]string, error) {
// insertPrivacyPairs inserts the collected privacy pairs into the SQL database.
func insertPrivacyPairs(ctx context.Context, sqlTx *sqlcmig6.Queries,
pairs privacyPairs) error {
pairs privacyPairs, totalPairs int) error {
var (
migCount = 0
nextProgressLog = migrationProgressLogInterval
)
for groupId, groupPairs := range pairs {
err := insertGroupPairs(ctx, sqlTx, groupId, groupPairs)
count, err := insertGroupPairs(ctx, sqlTx, groupId, groupPairs)
if err != nil {
return fmt.Errorf("inserting group pairs for group "+
"id %d failed: %w", groupId, err)
}
migCount += count
for migCount >= nextProgressLog {
log.Infof("Migrated %d/%d privacy mapper rows from "+
"KV to SQL", nextProgressLog, totalPairs)
nextProgressLog += migrationProgressLogInterval
}
}
return nil
@ -754,7 +786,9 @@ func insertPrivacyPairs(ctx context.Context, sqlTx *sqlcmig6.Queries,
// to pseudo values, where the key is the real value and the value is the
// corresponding pseudo value.
func insertGroupPairs(ctx context.Context, sqlTx *sqlcmig6.Queries,
groupID int64, pairs map[string]string) error {
groupID int64, pairs map[string]string) (int, error) {
insertedCount := 0
for realVal, pseudoVal := range pairs {
err := sqlTx.InsertPrivacyPair(
@ -765,12 +799,15 @@ func insertGroupPairs(ctx context.Context, sqlTx *sqlcmig6.Queries,
},
)
if err != nil {
return fmt.Errorf("inserting privacy pair %s:%s "+
"failed: %w", realVal, pseudoVal, err)
return insertedCount, fmt.Errorf("inserting privacy "+
"pair %s:%s failed: %w", realVal, pseudoVal,
err)
}
insertedCount++
}
return nil
return insertedCount, nil
}
// validatePrivacyPairsMigration validates that the migrated privacy pairs
@ -878,6 +915,9 @@ func migrateActionsToSQL(ctx context.Context, kvStore *bbolt.DB,
err)
}
// migCount tracks the number of actions than have been migrated.
migCount := 0
// Iterate over and migrate all actions in the KVDB. Note that this
// function migrates each action while iterating over them, instead
// of first collecting all actions and storing them in memory before
@ -964,7 +1004,7 @@ func migrateActionsToSQL(ctx context.Context, kvStore *bbolt.DB,
"session %x: %w", macID, err)
}
log.Infof("Migrated Action: Macaroon ID: %x, "+
log.Tracef("Migrated Action: Macaroon ID: %x, "+
"ActionID: %x, Actor: %s, Feature: %s",
macID, actionID, action.ActorName,
action.FeatureName)
@ -981,6 +1021,12 @@ func migrateActionsToSQL(ctx context.Context, kvStore *bbolt.DB,
"to SQL failed: %w", err)
}
migCount++
if migCount%migrationProgressLogInterval == 0 {
log.Infof("Migrated %d actions from "+
"KV to SQL", migCount)
}
return nil
})
})
@ -989,8 +1035,7 @@ func migrateActionsToSQL(ctx context.Context, kvStore *bbolt.DB,
return fmt.Errorf("iterating over actions failed: %w", err)
}
log.Infof("Finished iterating actions in KV store " +
"(no persistence yet).")
log.Infof("Finished migration of %d actions from KV to SQL.", migCount)
return nil
}

24
session/sql_migration.go
View file

@ -31,6 +31,8 @@ var (
"original session")
)
const migrationProgressLogInterval = 100
// MigrateSessionStoreToSQL runs the migration of all sessions from the KV
// database to the SQL database. The migration is done in a single transaction
// to ensure that all sessions are migrated or none at all.
@ -48,21 +50,27 @@ func MigrateSessionStoreToSQL(ctx context.Context, kvStore *bbolt.DB,
}
initialGroupSessions, linkedSessions := filterSessions(kvSessions)
total := len(initialGroupSessions) + len(linkedSessions)
log.Infof("Collected %d sessions for KV to SQL migration", total)
// Migrate the non-linked sessions first.
err = migrateSessionsToSQLAndValidate(ctx, tx, initialGroupSessions)
err = migrateSessionsToSQLAndValidate(
ctx, tx, initialGroupSessions, 0, total,
)
if err != nil {
return fmt.Errorf("migration of non-linked session failed: %w",
err)
}
// Then migrate the linked sessions.
err = migrateSessionsToSQLAndValidate(ctx, tx, linkedSessions)
err = migrateSessionsToSQLAndValidate(
ctx, tx, linkedSessions, len(initialGroupSessions), total,
)
if err != nil {
return fmt.Errorf("migration of linked session failed: %w", err)
}
total := len(initialGroupSessions) + len(linkedSessions)
log.Infof("All sessions migrated from KV to SQL. Total number of "+
"sessions migrated: %d", total)
@ -177,9 +185,9 @@ func getBBoltSessions(db *bbolt.DB) ([]*Session, error) {
// from the KV database to the SQL database, and validates that the migrated
// sessions match the original sessions.
func migrateSessionsToSQLAndValidate(ctx context.Context, tx *s6.Queries,
kvSessions []*Session) error {
kvSessions []*Session, migratedOffset, totalCount int) error {
for _, kvSession := range kvSessions {
for i, kvSession := range kvSessions {
err := migrateSingleSessionToSQL(ctx, tx, kvSession)
if err != nil {
return fmt.Errorf("unable to migrate session(%v): %w",
@ -219,6 +227,12 @@ func migrateSessionsToSQLAndValidate(ctx context.Context, tx *s6.Queries,
return fmt.Errorf("%w: %v.\n%v", ErrMigrationMismatch,
kvSession.ID, diffText)
}
migratedCount := migratedOffset + i + 1
if migratedCount%migrationProgressLogInterval == 0 {
log.Infof("Migrated %d/%d sessions from KV to SQL",
migratedCount, totalCount)
}
}
return nil