Merge pull request #829 from bhandras/cost-migration-rpc-batchsize

loopd: allow setting RPC batch size when running cost migration

cost_migration.go

@@ -16,10 +16,6 @@ import (
 const (
 	// costMigrationID is the identifier for the cost migration.
 	costMigrationID = "cost_migration"
-
-	// paymentBatchSize is the maximum number of payments we'll fetch in
-	// one go.
-	paymentBatchSize = 1000
 )
 
 // CalculateLoopOutCost calculates the total cost of a loop out swap. It will
@@ -112,7 +108,7 @@ func CalculateLoopOutCost(params *chaincfg.Params, loopOutSwap *loopdb.LoopOut,
 // MigrateLoopOutCosts will calculate the correct cost for all loop out swaps
 // and override the cost values of the last update in the database.
 func MigrateLoopOutCosts(ctx context.Context, lnd lndclient.LndServices,
-	db loopdb.SwapStore) error {
+	paymentBatchSize int, db loopdb.SwapStore) error {
 
 	migrationDone, err := db.HasMigration(ctx, costMigrationID)
 	if err != nil {
@@ -145,7 +141,7 @@ func MigrateLoopOutCosts(ctx context.Context, lnd lndclient.LndServices,
 		payments, err := lnd.Client.ListPayments(
 			ctx, lndclient.ListPaymentsRequest{
 				Offset:      offset,
-				MaxPayments: paymentBatchSize,
+				MaxPayments: uint64(paymentBatchSize),
 			},
 		)
 		if err != nil {

cost_migration_test.go

@@ -158,7 +158,7 @@ func TestCostMigration(t *testing.T) {
 	}
 
 	// Now we can run the migration.
-	err = MigrateLoopOutCosts(context.Background(), lnd.LndServices, store)
+	err = MigrateLoopOutCosts(context.Background(), lnd.LndServices, 1, store)
 	require.NoError(t, err)
 
 	// Finally check that the swap cost has been updated correctly.
@@ -179,6 +179,6 @@ func TestCostMigration(t *testing.T) {
 	// Now run the migration again to make sure it doesn't fail. This also
 	// indicates that the migration did not run the second time as
 	// otherwise the store mocks SetMigration function would fail.
-	err = MigrateLoopOutCosts(context.Background(), lnd.LndServices, store)
+	err = MigrateLoopOutCosts(context.Background(), lnd.LndServices, 1, store)
 	require.NoError(t, err)
 }

loopd/config.go

@@ -48,6 +48,12 @@ var (
 	defaultTotalPaymentTimeout = time.Minute * 60
 	defaultMaxPaymentRetries   = 3
 
+	// defaultRPCBatchSize is the default batch size to use for RPC calls
+	// we make to LND during migrations. If operations on the LND side are
+	// too slow a too large batch size may prevent the migration from
+	// completing.
+	defaultRPCBatchSize = 1000
+
 	// DefaultTLSCertFilename is the default file name for the autogenerated
 	// TLS certificate.
 	DefaultTLSCertFilename = "tls.cert"
@@ -179,6 +185,8 @@ type Config struct {
 
 	EnableExperimental bool `long:"experimental" description:"Enable experimental features: reservations"`
 
+	MigrationRPCBatchSize int `long:"migrationrpcbatchsize" description:"The RPC batch size to use during migrations."`
+
 	Lnd *lndConfig `group:"lnd" namespace:"lnd"`
 
 	Server *loopServerConfig `group:"server" namespace:"server"`
@@ -207,20 +215,21 @@ func DefaultConfig() Config {
 		Sqlite: &loopdb.SqliteConfig{
 			DatabaseFileName: defaultSqliteDatabasePath,
 		},
-		LogDir:              defaultLogDir,
-		MaxLogFiles:         defaultMaxLogFiles,
-		MaxLogFileSize:      defaultMaxLogFileSize,
-		DebugLevel:          defaultLogLevel,
-		TLSCertPath:         DefaultTLSCertPath,
-		TLSKeyPath:          DefaultTLSKeyPath,
-		TLSValidity:         DefaultAutogenValidity,
-		MacaroonPath:        DefaultMacaroonPath,
-		MaxL402Cost:         l402.DefaultMaxCostSats,
-		MaxL402Fee:          l402.DefaultMaxRoutingFeeSats,
-		LoopOutMaxParts:     defaultLoopOutMaxParts,
-		TotalPaymentTimeout: defaultTotalPaymentTimeout,
-		MaxPaymentRetries:   defaultMaxPaymentRetries,
-		EnableExperimental:  false,
+		LogDir:                defaultLogDir,
+		MaxLogFiles:           defaultMaxLogFiles,
+		MaxLogFileSize:        defaultMaxLogFileSize,
+		DebugLevel:            defaultLogLevel,
+		TLSCertPath:           DefaultTLSCertPath,
+		TLSKeyPath:            DefaultTLSKeyPath,
+		TLSValidity:           DefaultAutogenValidity,
+		MacaroonPath:          DefaultMacaroonPath,
+		MaxL402Cost:           l402.DefaultMaxCostSats,
+		MaxL402Fee:            l402.DefaultMaxRoutingFeeSats,
+		LoopOutMaxParts:       defaultLoopOutMaxParts,
+		TotalPaymentTimeout:   defaultTotalPaymentTimeout,
+		MaxPaymentRetries:     defaultMaxPaymentRetries,
+		EnableExperimental:    false,
+		MigrationRPCBatchSize: defaultRPCBatchSize,
 		Lnd: &lndConfig{
 			Host:         "localhost:10009",
 			MacaroonPath: DefaultLndMacaroonPath,
@@ -367,6 +376,10 @@ func Validate(cfg *Config) error {
 		return fmt.Errorf("TLS certificate minimum validity period is 24h")
 	}
 
+	if cfg.MigrationRPCBatchSize <= 0 {
+		return fmt.Errorf("migrationrpcbatchsize must be greater than 0")
+	}
+
 	return nil
 }
 

loopd/daemon.go

@@ -410,7 +410,10 @@ func (d *Daemon) initialize(withMacaroonService bool) error {
 	}
 
 	// Run the costs migration.
-	err = loop.MigrateLoopOutCosts(d.mainCtx, d.lnd.LndServices, swapDb)
+	err = loop.MigrateLoopOutCosts(
+		d.mainCtx, d.lnd.LndServices, d.cfg.MigrationRPCBatchSize,
+		swapDb,
+	)
 	if err != nil {
 		log.Errorf("Cost migration failed: %v", err)