firewalldb: add kvstores kvdb to SQL migration

This commit introduces the migration logic for transitioning the
kvstores store from kvdb to SQL.

Note that as of this commit, the migration is not yet triggered by any
production code, i.e. only tests execute the migration logic.

firewalldb/sql_migration.go

@@ -0,0 +1,492 @@
+package firewalldb
+
+import (
+	"bytes"
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+
+	"github.com/lightninglabs/lightning-terminal/db/sqlc"
+	"github.com/lightningnetwork/lnd/fn"
+	"github.com/lightningnetwork/lnd/sqldb"
+	"go.etcd.io/bbolt"
+)
+
+// kvEntry represents a single KV entry inserted into the BoltDB.
+type kvEntry struct {
+	perm     bool
+	ruleName string
+	key      string
+	value    []byte
+
+	// groupAlias is the legacy session group alias that the entry is
+	// associated with. For global entries, this will be fn.None[[]byte].
+	groupAlias fn.Option[[]byte]
+
+	// featureName is the name of the feature that the entry is associated
+	// with. If the entry is not feature specific, this will be
+	// fn.None[string].
+	featureName fn.Option[string]
+}
+
+// sqlKvEntry represents a single KV entry inserted into the SQL DB, containing
+// the same fields as the kvEntry, but with additional fields that represent the
+// SQL IDs of the rule, session group, and feature.
+type sqlKvEntry struct {
+	*kvEntry
+
+	ruleID int64
+
+	// groupID is the sql session group ID that the entry is associated
+	// with. For global entries, this will be Valid=false.
+	groupID sql.NullInt64
+
+	// featureID is the sql feature ID that the entry is associated with.
+	// This is only set if the entry is feature specific, and will be
+	// Valid=false for other types entries. If this is set, then groupID
+	// will also be set.
+	featureID sql.NullInt64
+}
+
+// namespacedKey returns a string representation of the kvEntry purely used for
+// logging purposes.
+func (e *kvEntry) namespacedKey() string {
+	ns := fmt.Sprintf("perm: %t, rule: %s", e.perm, e.ruleName)
+
+	e.groupAlias.WhenSome(func(alias []byte) {
+		ns += fmt.Sprintf(", group: %s", alias)
+	})
+
+	e.featureName.WhenSome(func(feature string) {
+		ns += fmt.Sprintf(", feature: %s", feature)
+	})
+
+	ns += fmt.Sprintf(", key: %s", e.key)
+
+	return ns
+}
+
+// MigrateFirewallDBToSQL runs the migration of the firwalldb stores from the
+// bbolt database to a SQL database. The migration is done in a single
+// transaction to ensure that all rows in the stores are migrated or none at
+// all.
+//
+// Note that this migration currently only migrates the kvstores, but will be
+// extended in the future to also migrate the privacy mapper and action stores.
+//
+// NOTE: As sessions may contain linked sessions and accounts, the sessions and
+// accounts sql migration MUST be run prior to this migration.
+func MigrateFirewallDBToSQL(ctx context.Context, kvStore *bbolt.DB,
+	sqlTx SQLQueries) error {
+
+	log.Infof("Starting migration of the rules DB to SQL")
+
+	err := migrateKVStoresDBToSQL(ctx, kvStore, sqlTx)
+	if err != nil {
+		return err
+	}
+
+	log.Infof("The rules DB has been migrated from KV to SQL.")
+
+	// TODO(viktor): Add migration for the privacy mapper and the action
+	// stores.
+
+	return nil
+}
+
+// migrateKVStoresDBToSQL runs the migration of all KV stores from the KV
+// database to the SQL database. The function also asserts that the
+// migrated values match the original values in the KV store.
+func migrateKVStoresDBToSQL(ctx context.Context, kvStore *bbolt.DB,
+	sqlTx SQLQueries) error {
+
+	log.Infof("Starting migration of the KV stores to SQL")
+
+	var pairs []*kvEntry
+
+	// 1) Collect all key-value pairs from the KV store.
+	err := kvStore.View(func(tx *bbolt.Tx) error {
+		var err error
+		pairs, err = collectAllPairs(tx)
+		return err
+	})
+	if err != nil {
+		return fmt.Errorf("collecting all kv pairs failed: %w", err)
+	}
+
+	var insertedPairs []*sqlKvEntry
+
+	// 2) Insert all collected key-value pairs into the SQL database.
+	for _, entry := range pairs {
+		insertedPair, err := insertPair(ctx, sqlTx, entry)
+		if err != nil {
+			return fmt.Errorf("inserting kv pair %v failed: %w",
+				entry.key, err)
+		}
+
+		insertedPairs = append(insertedPairs, insertedPair)
+	}
+
+	// 3) Validate the migrated values against the original values.
+	for _, insertedPair := range insertedPairs {
+		// Fetch the appropriate SQL entry's value.
+		migratedValue, err := getSQLValue(ctx, sqlTx, insertedPair)
+		if err != nil {
+			return fmt.Errorf("getting SQL value for key %s "+
+				"failed: %w", insertedPair.namespacedKey(), err)
+		}
+
+		// Compare the value of the migrated entry with the original
+		// value from the KV store.
+		// NOTE: if the insert a []byte{} value into the sqldb as the
+		// entry value, and then retrieve it, the value will be
+		// returned as nil. The bytes.Equal will pass in that case,
+		// and therefore such cases won't error out. The kvdb instance
+		// can store []byte{} values.
+		if !bytes.Equal(migratedValue, insertedPair.value) {
+			return fmt.Errorf("migrated value for key %s "+
+				"does not match original value: "+
+				"migrated %x, original %x",
+				insertedPair.namespacedKey(), migratedValue,
+				insertedPair.value)
+		}
+	}
+
+	log.Infof("Migration of the KV stores to SQL completed. Total number "+
+		"of rows migrated: %d", len(pairs))
+
+	return nil
+}
+
+// collectAllPairs collects all key-value pairs from the KV store, and returns
+// them as a slice of kvEntry structs. The function expects the KV store to be
+// stuctured as described in the comment in the firewalldb/kvstores_kvdb.go
+// file. Any other structure will result in an error.
+// Note that this function and the subsequent functions are intentionally
+// designed to iterate over all buckets and values that exist in the KV store.
+// That ensures that we find all stores and values that exist in the KV store,
+// and can be sure that the kv store actually follows the expected structure.
+func collectAllPairs(tx *bbolt.Tx) ([]*kvEntry, error) {
+	var entries []*kvEntry
+	for _, perm := range []bool{true, false} {
+		mainBucket, err := getMainBucket(tx, false, perm)
+		if err != nil {
+			return nil, err
+		}
+
+		if mainBucket == nil {
+			// If the mainBucket doesn't exist, there are no entries
+			// to migrate under that bucket, therefore we don't
+			// error, and just proceed to not migrate any entries
+			// under that bucket.
+			continue
+		}
+
+		// Loop over each rule-name bucket.
+		err = mainBucket.ForEach(func(rule, v []byte) error {
+			if v != nil {
+				return errors.New("expected only " +
+					"buckets under main bucket")
+			}
+
+			ruleBucket := mainBucket.Bucket(rule)
+			if ruleBucket == nil {
+				return fmt.Errorf("rule bucket %s not found",
+					rule)
+			}
+
+			pairs, err := collectRulePairs(
+				ruleBucket, perm, string(rule),
+			)
+			if err != nil {
+				return err
+			}
+
+			entries = append(entries, pairs...)
+
+			return nil
+		})
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return entries, nil
+}
+
+// collectRulePairs processes a single rule bucket, which should contain the
+// global and session-kv-store key buckets.
+func collectRulePairs(bkt *bbolt.Bucket, perm bool, rule string) ([]*kvEntry,
+	error) {
+
+	var params []*kvEntry
+
+	err := verifyBktKeys(
+		bkt, true, globalKVStoreBucketKey, sessKVStoreBucketKey,
+	)
+	if err != nil {
+		return params, fmt.Errorf("verifying rule bucket %s keys "+
+			"failed: %w", rule, err)
+	}
+
+	if globalBkt := bkt.Bucket(globalKVStoreBucketKey); globalBkt != nil {
+		p, err := collectKVPairs(
+			globalBkt, true, perm, rule,
+			fn.None[[]byte](), fn.None[string](),
+		)
+		if err != nil {
+			return nil, fmt.Errorf("collecting global kv pairs "+
+				"failed: %w", err)
+		}
+
+		params = append(params, p...)
+	}
+
+	if sessBkt := bkt.Bucket(sessKVStoreBucketKey); sessBkt != nil {
+		err := sessBkt.ForEach(func(groupAlias, v []byte) error {
+			if v != nil {
+				return fmt.Errorf("expected only buckets "+
+					"under %s bucket", sessKVStoreBucketKey)
+			}
+
+			groupBucket := sessBkt.Bucket(groupAlias)
+			if groupBucket == nil {
+				return fmt.Errorf("group bucket for group "+
+					"alias %s not found", groupAlias)
+			}
+
+			kvPairs, err := collectKVPairs(
+				groupBucket, false, perm, rule,
+				fn.Some(groupAlias), fn.None[string](),
+			)
+			if err != nil {
+				return fmt.Errorf("collecting group kv "+
+					"pairs failed: %w", err)
+			}
+
+			params = append(params, kvPairs...)
+
+			err = verifyBktKeys(
+				groupBucket, false, featureKVStoreBucketKey,
+			)
+			if err != nil {
+				return fmt.Errorf("verification of group "+
+					"bucket %s keys failed: %w", groupAlias,
+					err)
+			}
+
+			ftBkt := groupBucket.Bucket(featureKVStoreBucketKey)
+			if ftBkt == nil {
+				return nil
+			}
+
+			return ftBkt.ForEach(func(ftName, v []byte) error {
+				if v != nil {
+					return fmt.Errorf("expected only "+
+						"buckets under %s bucket",
+						featureKVStoreBucketKey)
+				}
+
+				// The feature name should exist, as per the
+				// verification above.
+				featureBucket := ftBkt.Bucket(ftName)
+				if featureBucket == nil {
+					return fmt.Errorf("feature bucket "+
+						"%s not found", ftName)
+				}
+
+				featurePairs, err := collectKVPairs(
+					featureBucket, true, perm, rule,
+					fn.Some(groupAlias),
+					fn.Some(string(ftName)),
+				)
+				if err != nil {
+					return fmt.Errorf("collecting "+
+						"feature kv pairs failed: %w",
+						err)
+				}
+
+				params = append(params, featurePairs...)
+
+				return nil
+			})
+		})
+		if err != nil {
+			return nil, fmt.Errorf("collecting session kv pairs "+
+				"failed: %w", err)
+		}
+	}
+
+	return params, nil
+}
+
+// collectKVPairs collects all key-value pairs from the given bucket, and
+// returns them as a slice of kvEntry structs. If the errorOnBuckets parameter
+// is set to true, then the function will return an error if the bucket
+// contains any sub-buckets. Note that when the errorOnBuckets parameter is
+// set to false, the function will not collect any key-value pairs from the
+// sub-buckets, and will just ignore them.
+func collectKVPairs(bkt *bbolt.Bucket, errorOnBuckets, perm bool,
+	ruleName string, groupAlias fn.Option[[]byte],
+	featureName fn.Option[string]) ([]*kvEntry, error) {
+
+	var params []*kvEntry
+
+	return params, bkt.ForEach(func(key, value []byte) error {
+		// If the value is nil, then this is a bucket, which we
+		// don't want to process here, as we only want to collect
+		// the key-value pairs, not the buckets. If we should
+		// error on buckets, then we return an error here.
+		if value == nil {
+			if errorOnBuckets {
+				return fmt.Errorf("unexpected bucket %s found "+
+					"in when collecting kv pairs", key)
+			}
+
+			return nil
+		}
+
+		params = append(params, &kvEntry{
+			perm:        perm,
+			ruleName:    ruleName,
+			key:         string(key),
+			featureName: featureName,
+			groupAlias:  groupAlias,
+			value:       value,
+		})
+
+		return nil
+	})
+}
+
+// insertPair inserts a single key-value pair into the SQL database.
+func insertPair(ctx context.Context, tx SQLQueries,
+	entry *kvEntry) (*sqlKvEntry, error) {
+
+	ruleID, err := tx.GetOrInsertRuleID(ctx, entry.ruleName)
+	if err != nil {
+		return nil, err
+	}
+
+	p := sqlc.InsertKVStoreRecordParams{
+		Perm:     entry.perm,
+		RuleID:   ruleID,
+		EntryKey: entry.key,
+		Value:    entry.value,
+	}
+
+	entry.groupAlias.WhenSome(func(alias []byte) {
+		var groupID int64
+		groupID, err = tx.GetSessionIDByAlias(ctx, alias)
+		if err != nil {
+			err = fmt.Errorf("getting group id by alias %x "+
+				"failed: %w", alias, err)
+			return
+		}
+
+		p.GroupID = sqldb.SQLInt64(groupID)
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	entry.featureName.WhenSome(func(feature string) {
+		var featureID int64
+		featureID, err = tx.GetOrInsertFeatureID(ctx, feature)
+		if err != nil {
+			err = fmt.Errorf("getting/inserting feature id for %s "+
+				"failed: %w", feature, err)
+			return
+		}
+
+		p.FeatureID = sqldb.SQLInt64(featureID)
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	err = tx.InsertKVStoreRecord(ctx, p)
+	if err != nil {
+		return nil, err
+	}
+
+	return &sqlKvEntry{
+		kvEntry:   entry,
+		ruleID:    p.RuleID,
+		groupID:   p.GroupID,
+		featureID: p.FeatureID,
+	}, nil
+}
+
+// getSQLValue retrieves the key value for the given kvEntry from the SQL
+// database.
+func getSQLValue(ctx context.Context, tx SQLQueries,
+	entry *sqlKvEntry) ([]byte, error) {
+
+	switch {
+	case entry.featureID.Valid && entry.groupID.Valid:
+		return tx.GetFeatureKVStoreRecord(
+			ctx, sqlc.GetFeatureKVStoreRecordParams{
+				Perm:      entry.perm,
+				RuleID:    entry.ruleID,
+				GroupID:   entry.groupID,
+				FeatureID: entry.featureID,
+				Key:       entry.key,
+			},
+		)
+	case entry.groupID.Valid:
+		return tx.GetGroupKVStoreRecord(
+			ctx, sqlc.GetGroupKVStoreRecordParams{
+				Perm:    entry.perm,
+				RuleID:  entry.ruleID,
+				GroupID: entry.groupID,
+				Key:     entry.key,
+			},
+		)
+	case !entry.featureID.Valid && !entry.groupID.Valid:
+		return tx.GetGlobalKVStoreRecord(
+			ctx, sqlc.GetGlobalKVStoreRecordParams{
+				Perm:   entry.perm,
+				RuleID: entry.ruleID,
+				Key:    entry.key,
+			},
+		)
+	default:
+		return nil, fmt.Errorf("invalid combination of feature and "+
+			"session ID: featureID valid: %v, groupID valid: %v",
+			entry.featureID.Valid, entry.groupID.Valid)
+	}
+}
+
+// verifyBktKeys checks that the given bucket only contains buckets with the
+// passed keys, and optionally also key-value pairs. If the errorOnKeyValues
+// parameter is set to true, the function will error if it finds key-value pairs
+// in the bucket.
+func verifyBktKeys(bkt *bbolt.Bucket, errorOnKeyValues bool,
+	keys ...[]byte) error {
+
+	return bkt.ForEach(func(key, v []byte) error {
+		if v != nil {
+			// If we allow key-values, then we can just continue
+			// to the next key. Else we need to error out, as we
+			// only expect buckets under the passed bucket.
+			if errorOnKeyValues {
+				return fmt.Errorf("unexpected key-value pair "+
+					"found: key=%s, value=%x", key, v)
+			}
+
+			return nil
+		}
+
+		for _, expectedKey := range keys {
+			if bytes.Equal(key, expectedKey) {
+				// If this is an expected key, we can continue
+				// to the next key.
+				return nil
+			}
+		}
+
+		return fmt.Errorf("unexpected key found: %s", key)
+	})
+}

firewalldb/sql_migration_test.go

@@ -0,0 +1,664 @@
+package firewalldb
+
+import (
+	"bytes"
+	"context"
+	"database/sql"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/lightninglabs/lightning-terminal/accounts"
+	"github.com/lightninglabs/lightning-terminal/db"
+	"github.com/lightninglabs/lightning-terminal/db/sqlc"
+	"github.com/lightninglabs/lightning-terminal/session"
+	"github.com/lightningnetwork/lnd/clock"
+	"github.com/lightningnetwork/lnd/fn"
+	"github.com/lightningnetwork/lnd/sqldb"
+	"github.com/stretchr/testify/require"
+	"golang.org/x/exp/rand"
+)
+
+const (
+	testRuleName     = "test-rule"
+	testRuleName2    = "test-rule-2"
+	testFeatureName  = "test-feature"
+	testFeatureName2 = "test-feature-2"
+	testEntryKey     = "test-entry-key"
+	testEntryKey2    = "test-entry-key-2"
+	testEntryKey3    = "test-entry-key-3"
+	testEntryKey4    = "test-entry-key-4"
+)
+
+var (
+	testEntryValue = []byte{1, 2, 3}
+)
+
+// TestFirewallDBMigration tests the migration of firewalldb from a bolt
+// backend to a SQL database. Note that this test does not attempt to be a
+// complete migration test.
+// This test only tests the migration of the KV stores currently, but will
+// be extended in the future to also test the migration of the privacy mapper
+// and the actions store in the future.
+func TestFirewallDBMigration(t *testing.T) {
+	t.Parallel()
+
+	ctx := context.Background()
+	clock := clock.NewTestClock(time.Now())
+
+	// When using build tags that creates a kvdb store for NewTestDB, we
+	// skip this test as it is only applicable for postgres and sqlite tags.
+	store := NewTestDB(t, clock)
+	if _, ok := store.(*BoltDB); ok {
+		t.Skipf("Skipping Firewall DB migration test for kvdb build")
+	}
+
+	makeSQLDB := func(t *testing.T, sessionsStore session.Store) (*SQLDB,
+		*db.TransactionExecutor[SQLQueries]) {
+
+		testDBStore := NewTestDBWithSessions(t, sessionsStore, clock)
+
+		store, ok := testDBStore.(*SQLDB)
+		require.True(t, ok)
+
+		baseDB := store.BaseDB
+
+		genericExecutor := db.NewTransactionExecutor(
+			baseDB, func(tx *sql.Tx) SQLQueries {
+				return baseDB.WithTx(tx)
+			},
+		)
+
+		return store, genericExecutor
+	}
+
+	// The assertMigrationResults function will currently assert that
+	// the migrated kv stores entries in the SQLDB match the original kv
+	// stores entries in the BoltDB.
+	assertMigrationResults := func(t *testing.T, sqlStore *SQLDB,
+		kvEntries []*kvEntry) {
+
+		var (
+			ruleIDs    = make(map[string]int64)
+			groupIDs   = make(map[string]int64)
+			featureIDs = make(map[string]int64)
+			err        error
+		)
+
+		getRuleID := func(ruleName string) int64 {
+			ruleID, ok := ruleIDs[ruleName]
+			if !ok {
+				ruleID, err = sqlStore.GetRuleID(ctx, ruleName)
+				require.NoError(t, err)
+
+				ruleIDs[ruleName] = ruleID
+			}
+
+			return ruleID
+		}
+
+		getGroupID := func(groupAlias []byte) int64 {
+			groupID, ok := groupIDs[string(groupAlias)]
+			if !ok {
+				groupID, err = sqlStore.GetSessionIDByAlias(
+					ctx, groupAlias,
+				)
+				require.NoError(t, err)
+
+				groupIDs[string(groupAlias)] = groupID
+			}
+
+			return groupID
+		}
+
+		getFeatureID := func(featureName string) int64 {
+			featureID, ok := featureIDs[featureName]
+			if !ok {
+				featureID, err = sqlStore.GetFeatureID(
+					ctx, featureName,
+				)
+				require.NoError(t, err)
+
+				featureIDs[featureName] = featureID
+			}
+
+			return featureID
+		}
+
+		// First we extract all migrated kv entries from the SQLDB,
+		// in order to be able to compare them to the original kv
+		// entries, to ensure that the migration was successful.
+		sqlKvEntries, err := sqlStore.ListAllKVStoresRecords(ctx)
+		require.NoError(t, err)
+		require.Equal(t, len(kvEntries), len(sqlKvEntries))
+
+		// We then iterate over the original kv entries that were
+		// migrated from the BoltDB to the SQLDB, and assert that they
+		// match the migrated SQL kv entries.
+		// NOTE: when fetching kv entries that were inserted into the
+		// sql store with the entry value []byte{}, a nil value is
+		// returned. Therefore, require.Equal would error on such cases,
+		// while bytes.Equal would not. Therefore, the comparison below
+		// uses bytes.Equal to compare the values.
+		for _, entry := range kvEntries {
+			ruleID := getRuleID(entry.ruleName)
+
+			if entry.groupAlias.IsNone() {
+				sqlVal, err := sqlStore.GetGlobalKVStoreRecord(
+					ctx,
+					sqlc.GetGlobalKVStoreRecordParams{
+						Key:    entry.key,
+						Perm:   entry.perm,
+						RuleID: ruleID,
+					},
+				)
+				require.NoError(t, err)
+				// See docs for the loop above on why
+				// bytes.Equal is used here.
+				require.True(
+					t, bytes.Equal(entry.value, sqlVal),
+				)
+			} else if entry.featureName.IsNone() {
+				groupAlias := entry.groupAlias.UnwrapOrFail(t)
+				groupID := getGroupID(groupAlias[:])
+
+				v, err := sqlStore.GetGroupKVStoreRecord(
+					ctx,
+					sqlc.GetGroupKVStoreRecordParams{
+						Key:    entry.key,
+						Perm:   entry.perm,
+						RuleID: ruleID,
+						GroupID: sql.NullInt64{
+							Int64: groupID,
+							Valid: true,
+						},
+					},
+				)
+				require.NoError(t, err)
+				// See docs for the loop above on why
+				// bytes.Equal is used here.
+				require.True(
+					t, bytes.Equal(entry.value, v),
+				)
+			} else {
+				groupAlias := entry.groupAlias.UnwrapOrFail(t)
+				groupID := getGroupID(groupAlias[:])
+				featureID := getFeatureID(
+					entry.featureName.UnwrapOrFail(t),
+				)
+
+				sqlVal, err := sqlStore.GetFeatureKVStoreRecord(
+					ctx,
+					sqlc.GetFeatureKVStoreRecordParams{
+						Key:    entry.key,
+						Perm:   entry.perm,
+						RuleID: ruleID,
+						GroupID: sql.NullInt64{
+							Int64: groupID,
+							Valid: true,
+						},
+						FeatureID: sql.NullInt64{
+							Int64: featureID,
+							Valid: true,
+						},
+					},
+				)
+				require.NoError(t, err)
+				// See docs for the loop above on why
+				// bytes.Equal is used here.
+				require.True(
+					t, bytes.Equal(entry.value, sqlVal),
+				)
+			}
+		}
+	}
+
+	// The tests slice contains all the tests that we will run for the
+	// migration of the firewalldb from a BoltDB to a SQLDB.
+	// Note that the tests currently only test the migration of the KV
+	// stores, but will be extended in the future to also test the migration
+	// of the privacy mapper and the actions store.
+	tests := []struct {
+		name       string
+		populateDB func(t *testing.T, ctx context.Context,
+			boltDB *BoltDB, sessionStore session.Store) []*kvEntry
+	}{
+		{
+			name: "empty",
+			populateDB: func(t *testing.T, ctx context.Context,
+				boltDB *BoltDB,
+				sessionStore session.Store) []*kvEntry {
+
+				// Don't populate the DB.
+				return make([]*kvEntry, 0)
+			},
+		},
+		{
+			name:       "global entries",
+			populateDB: globalEntries,
+		},
+		{
+			name:       "session specific entries",
+			populateDB: sessionSpecificEntries,
+		},
+		{
+			name:       "feature specific entries",
+			populateDB: featureSpecificEntries,
+		},
+		{
+			name:       "all entry combinations",
+			populateDB: allEntryCombinations,
+		},
+		{
+			name:       "random entries",
+			populateDB: randomKVEntries,
+		},
+	}
+
+	for _, test := range tests {
+		tc := test
+
+		t.Run(tc.name, func(t *testing.T) {
+			t.Parallel()
+
+			// First let's create a sessions store to link to in
+			// the kvstores DB. In order to create the sessions
+			// store though, we also need to create an accounts
+			// store, that we link to the sessions store.
+			// Note that both of these stores will be sql stores due
+			// to the build tags enabled when running this test,
+			// which means we can also pass the sessions store to
+			// the sql version of the kv stores that we'll create
+			// in test, without also needing to migrate it.
+			accountStore := accounts.NewTestDB(t, clock)
+			sessionsStore := session.NewTestDBWithAccounts(
+				t, clock, accountStore,
+			)
+
+			// Create a new firewall store to populate with test
+			// data.
+			firewallStore, err := NewBoltDB(
+				t.TempDir(), DBFilename, sessionsStore,
+				accountStore, clock,
+			)
+			require.NoError(t, err)
+			t.Cleanup(func() {
+				require.NoError(t, firewallStore.Close())
+			})
+
+			// Populate the kv store.
+			entries := test.populateDB(
+				t, ctx, firewallStore, sessionsStore,
+			)
+
+			// Create the SQL store that we will migrate the data
+			// to.
+			sqlStore, txEx := makeSQLDB(t, sessionsStore)
+
+			// Perform the migration.
+			var opts sqldb.MigrationTxOptions
+			err = txEx.ExecTx(ctx, &opts,
+				func(tx SQLQueries) error {
+					return MigrateFirewallDBToSQL(
+						ctx, firewallStore.DB, tx,
+					)
+				},
+			)
+			require.NoError(t, err)
+
+			// Assert migration results.
+			assertMigrationResults(t, sqlStore, entries)
+		})
+	}
+}
+
+// globalEntries populates the kv store with one global entry for the temp
+// store, and one for the perm store.
+func globalEntries(t *testing.T, ctx context.Context, boltDB *BoltDB,
+	_ session.Store) []*kvEntry {
+
+	return insertTempAndPermEntry(
+		t, ctx, boltDB, testRuleName, fn.None[[]byte](),
+		fn.None[string](), testEntryKey, testEntryValue,
+	)
+}
+
+// sessionSpecificEntries populates the kv store with one session specific
+// entry for the local temp store, and one session specific entry for the perm
+// local store.
+func sessionSpecificEntries(t *testing.T, ctx context.Context, boltDB *BoltDB,
+	sessionStore session.Store) []*kvEntry {
+
+	groupAlias := getNewSessionAlias(t, ctx, sessionStore)
+
+	return insertTempAndPermEntry(
+		t, ctx, boltDB, testRuleName, groupAlias, fn.None[string](),
+		testEntryKey, testEntryValue,
+	)
+}
+
+// featureSpecificEntries populates the kv store with one feature specific
+// entry for the local temp store, and one feature specific entry for the perm
+// local store.
+func featureSpecificEntries(t *testing.T, ctx context.Context, boltDB *BoltDB,
+	sessionStore session.Store) []*kvEntry {
+
+	groupAlias := getNewSessionAlias(t, ctx, sessionStore)
+
+	return insertTempAndPermEntry(
+		t, ctx, boltDB, testRuleName, groupAlias,
+		fn.Some(testFeatureName), testEntryKey, testEntryValue,
+	)
+}
+
+// allEntryCombinations adds all types of different entries at all possible
+// levels of the kvstores, including multple entries with the same
+// ruleName, groupAlias and featureName. The test aims to cover all possible
+// combinations of entries in the kvstores, including nil and empty entry
+// values. That therefore ensures that the migrations don't overwrite or miss
+// any entries when the entry set is more complex than just a single entry at
+// each level.
+func allEntryCombinations(t *testing.T, ctx context.Context, boltDB *BoltDB,
+	sessionStore session.Store) []*kvEntry {
+
+	var result []*kvEntry
+	add := func(entry []*kvEntry) {
+		result = append(result, entry...)
+	}
+
+	// First lets create standard entries at all levels, which represents
+	// the entries added by other tests.
+	add(globalEntries(t, ctx, boltDB, sessionStore))
+	add(sessionSpecificEntries(t, ctx, boltDB, sessionStore))
+	add(featureSpecificEntries(t, ctx, boltDB, sessionStore))
+
+	groupAlias := getNewSessionAlias(t, ctx, sessionStore)
+
+	// Now lets add a few more entries at with different rule names and
+	// features, just to ensure that we cover entries in different rule and
+	// feature tables.
+	add(insertTempAndPermEntry(
+		t, ctx, boltDB, testRuleName2, fn.None[[]byte](),
+		fn.None[string](), testEntryKey, testEntryValue,
+	))
+	add(insertTempAndPermEntry(
+		t, ctx, boltDB, testRuleName2, groupAlias,
+		fn.None[string](), testEntryKey, testEntryValue,
+	))
+	add(insertTempAndPermEntry(
+		t, ctx, boltDB, testRuleName2, groupAlias,
+		fn.Some(testFeatureName), testEntryKey, testEntryValue,
+	))
+	// Let's also create an entry with a different feature name that's still
+	// referencing the same group ID as the previous entry.
+	add(insertTempAndPermEntry(
+		t, ctx, boltDB, testRuleName2, groupAlias,
+		fn.Some(testFeatureName2), testEntryKey, testEntryValue,
+	))
+
+	// Finally, lets add a few entries with nil and empty values set for the
+	// actual key value, at all different levels, to ensure that tests don't
+	// break if the value is nil or empty.
+	var (
+		nilValue      []byte = nil
+		nilSliceValue        = []byte(nil)
+		emptyValue           = []byte{}
+	)
+
+	add(insertTempAndPermEntry(
+		t, ctx, boltDB, testRuleName2, fn.None[[]byte](),
+		fn.None[string](), testEntryKey2, nilValue,
+	))
+	add(insertTempAndPermEntry(
+		t, ctx, boltDB, testRuleName2, fn.None[[]byte](),
+		fn.None[string](), testEntryKey3, nilSliceValue,
+	))
+	add(insertTempAndPermEntry(
+		t, ctx, boltDB, testRuleName2, fn.None[[]byte](),
+		fn.None[string](), testEntryKey4, emptyValue,
+	))
+
+	add(insertTempAndPermEntry(
+		t, ctx, boltDB, testRuleName2, groupAlias,
+		fn.None[string](), testEntryKey2, nilValue,
+	))
+	add(insertTempAndPermEntry(
+		t, ctx, boltDB, testRuleName2, groupAlias,
+		fn.None[string](), testEntryKey3, nilSliceValue,
+	))
+	add(insertTempAndPermEntry(
+		t, ctx, boltDB, testRuleName2, groupAlias,
+		fn.None[string](), testEntryKey4, emptyValue,
+	))
+
+	add(insertTempAndPermEntry(
+		t, ctx, boltDB, testRuleName2, groupAlias,
+		fn.Some(testFeatureName), testEntryKey2, nilValue,
+	))
+	add(insertTempAndPermEntry(
+		t, ctx, boltDB, testRuleName2, groupAlias,
+		fn.Some(testFeatureName), testEntryKey3, nilSliceValue,
+	))
+	add(insertTempAndPermEntry(
+		t, ctx, boltDB, testRuleName2, groupAlias,
+		fn.Some(testFeatureName), testEntryKey4, emptyValue,
+	))
+
+	return result
+}
+
+func getNewSessionAlias(t *testing.T, ctx context.Context,
+	sessionStore session.Store) fn.Option[[]byte] {
+
+	sess, err := sessionStore.NewSession(
+		ctx, "test", session.TypeAutopilot,
+		time.Unix(1000, 0), "something",
+	)
+	require.NoError(t, err)
+
+	return fn.Some(sess.GroupID[:])
+}
+
+// insertTempAndPermEntry populates the kv store with one entry for the temp
+// store, and one entry for the perm store. Both of the entries will be inserted
+// with the same groupAlias, ruleName, entryKey and entryValue.
+func insertTempAndPermEntry(t *testing.T, ctx context.Context,
+	boltDB *BoltDB, ruleName string, groupAlias fn.Option[[]byte],
+	featureNameOpt fn.Option[string], entryKey string,
+	entryValue []byte) []*kvEntry {
+
+	tempKvEntry := &kvEntry{
+		ruleName:    ruleName,
+		groupAlias:  groupAlias,
+		featureName: featureNameOpt,
+		key:         entryKey,
+		value:       entryValue,
+		perm:        false,
+	}
+
+	insertKvEntry(t, ctx, boltDB, tempKvEntry)
+
+	permKvEntry := &kvEntry{
+		ruleName:    ruleName,
+		groupAlias:  groupAlias,
+		featureName: featureNameOpt,
+		key:         entryKey,
+		value:       entryValue,
+		perm:        true,
+	}
+
+	insertKvEntry(t, ctx, boltDB, permKvEntry)
+
+	return []*kvEntry{tempKvEntry, permKvEntry}
+}
+
+// insertKvEntry populates the kv store with passed entry, and asserts that the
+// entry is inserted correctly.
+func insertKvEntry(t *testing.T, ctx context.Context,
+	boltDB *BoltDB, entry *kvEntry) {
+
+	if entry.groupAlias.IsNone() && entry.featureName.IsSome() {
+		t.Fatalf("cannot set both global and feature specific at the " +
+			"same time")
+	}
+
+	// We get the kv stores that the entry will be inserted into. Note that
+	// we set an empty group ID if the entry is global, as the group ID
+	// will not be used when fetching the actual kv store that's used for
+	// global entries.
+	groupID := [4]byte{}
+	if entry.groupAlias.IsSome() {
+		copy(groupID[:], entry.groupAlias.UnwrapOrFail(t))
+	}
+
+	kvStores := boltDB.GetKVStores(
+		entry.ruleName, groupID, entry.featureName.UnwrapOr(""),
+	)
+
+	err := kvStores.Update(ctx, func(ctx context.Context,
+		tx KVStoreTx) error {
+
+		store := tx.Global()
+
+		switch {
+		case entry.groupAlias.IsNone() && !entry.perm:
+			store = tx.GlobalTemp()
+		case entry.groupAlias.IsSome() && !entry.perm:
+			store = tx.LocalTemp()
+		case entry.groupAlias.IsSome() && entry.perm:
+			store = tx.Local()
+		}
+
+		return store.Set(ctx, entry.key, entry.value)
+	})
+	require.NoError(t, err)
+}
+
+// randomKVEntries populates the kv store with random kv entries that span
+// across all possible combinations of different levels of entries in the kv
+// store. All values and different bucket names are randomly generated.
+func randomKVEntries(t *testing.T, ctx context.Context,
+	boltDB *BoltDB, sessionStore session.Store) []*kvEntry {
+
+	var (
+		// We set the number of entries to insert to 1000, as that
+		// should be enough to cover as many different
+		// combinations of entries as possible, while still being
+		// fast enough to run in a reasonable time.
+		numberOfEntries = 1000
+		insertedEntries = make([]*kvEntry, 0)
+		ruleName        = "initial-rule"
+		groupAlias      []byte
+		featureName     = "initial-feature"
+	)
+
+	// Create a random session that we can reference for the initial group
+	// ID.
+	sess, err := sessionStore.NewSession(
+		ctx, "initial-session", session.Type(1), time.Unix(1000, 0),
+		"serverAddr.test",
+	)
+	require.NoError(t, err)
+
+	groupAlias = sess.GroupID[:]
+
+	// Generate random entries. Note that many entries will use the same
+	// rule name, group ID and feature name, to simulate the real world
+	// usage of the kv stores as much as possible.
+	for i := 0; i < numberOfEntries; i++ {
+		// On average, we will generate a new rule which will be used
+		// for the kv store entry 10% of the time.
+		if rand.Intn(10) == 0 {
+			ruleName = fmt.Sprintf(
+				"rule-%s-%d", randomString(rand.Intn(30)+1), i,
+			)
+		}
+
+		// On average, we use the global store 25% of the time.
+		global := rand.Intn(4) == 0
+
+		// We'll use the perm store 50% of the time.
+		perm := rand.Intn(2) == 0
+
+		// For the non-global entries, we will generate a new group
+		// alias 25% of the time.
+		if !global && rand.Intn(4) == 0 {
+			newSess, err := sessionStore.NewSession(
+				ctx, fmt.Sprintf("session-%d", i),
+				session.Type(uint8(rand.Intn(5))),
+				time.Unix(1000, 0),
+				randomString(rand.Intn(10)+1),
+			)
+			require.NoError(t, err)
+
+			groupAlias = newSess.GroupID[:]
+		}
+
+		featureNameOpt := fn.None[string]()
+
+		// For 50% of the non-global entries, we insert a feature
+		// specific entry. The other 50% will be session specific
+		// entries.
+		if !global && rand.Intn(2) == 0 {
+			// 25% of the time, we will generate a new feature name.
+			if rand.Intn(4) == 0 {
+				featureName = fmt.Sprintf(
+					"feature-%s-%d",
+					randomString(rand.Intn(30)+1), i,
+				)
+			}
+
+			featureNameOpt = fn.Some(featureName)
+		}
+
+		groupAliasOpt := fn.None[[]byte]()
+		if !global {
+			// If the entry is not global, we set the group ID
+			// to the latest session's group ID.
+			groupAliasOpt = fn.Some(groupAlias[:])
+		}
+
+		entry := &kvEntry{
+			ruleName:    ruleName,
+			groupAlias:  groupAliasOpt,
+			featureName: featureNameOpt,
+			key:         fmt.Sprintf("key-%d", i),
+			perm:        perm,
+		}
+
+		// When setting a value for the entry, 25% of the time, we will
+		// set a nil or empty value.
+		if rand.Intn(4) == 0 {
+			// in 50% of these cases, we will set the value to nil,
+			// and in the other 50% we will set it to an empty
+			// value
+			if rand.Intn(2) == 0 {
+				entry.value = nil
+			} else {
+				entry.value = []byte{}
+			}
+		} else {
+			// Else generate a random value for all entries,
+			entry.value = []byte(randomString(rand.Intn(100) + 1))
+		}
+
+		// Insert the entry into the kv store.
+		insertKvEntry(t, ctx, boltDB, entry)
+
+		// Add the entry to the list of inserted entries.
+		insertedEntries = append(insertedEntries, entry)
+	}
+
+	return insertedEntries
+}
+
+// randomString generates a random string of the passed length n.
+func randomString(n int) string {
+	letterBytes := "abcdefghijklmnopqrstuvwxyz"
+
+	b := make([]byte, n)
+	for i := range b {
+		b[i] = letterBytes[rand.Intn(len(letterBytes))]
+	}
+	return string(b)
+}