sweepbatcher: allow swap_hash to be non-unique

SwapHash used to be a key in the sweeps table. Now the key is outpoint which
replaces columns outpoint_txid and outpoint_index. In-memory structures and unit
tests were also updated to use outpoint as key. Outpoint is truly unique.

loopdb/postgres.go

@@ -64,6 +64,21 @@ type PostgresStore struct {
 	*BaseDB
 }
 
+// In migration of sweeps table from outpoint_txid and outpoint_index to
+// outpoint we need to reverse the order of bytes in outpoint_txid and to
+// convert it to hex. This is done differently in sqlite and postgres.
+//
+// Changes from sqlite to postgres:
+//   - substr(blob, ...) -> get_byte(blob, index)
+//   - group_concat -> string_agg
+//   - 1-based indexing (32+1-i) -> 0-based (32 - i)
+//   - to_hex() + lpad(..., 2, '0') ensures each byte is two-digit hex
+const (
+	txidSqlite   = "group_concat(hex(substr(outpoint_txid,32+1-i,1)),'')"
+	txidPostgres = "string_agg(lpad(to_hex(get_byte(outpoint_txid, " +
+		"32 - i)), 2, '0'), '')"
+)
+
 // NewPostgresStore creates a new store that is backed by a Postgres database
 // backend.
 func NewPostgresStore(cfg *PostgresConfig,
@@ -93,6 +108,7 @@ func NewPostgresStore(cfg *PostgresConfig,
 		postgresFS := newReplacerFS(sqlSchemas, map[string]string{
 			"BLOB":                "BYTEA",
 			"INTEGER PRIMARY KEY": "SERIAL PRIMARY KEY",
+			txidSqlite:            txidPostgres,
 		})
 
 		err = applyMigrations(

loopdb/sqlc/batch.sql.go

@@ -35,7 +35,7 @@ func (q *Queries) DropBatch(ctx context.Context, id int32) error {
 
 const getBatchSweeps = `-- name: GetBatchSweeps :many
 SELECT
-        id, swap_hash, batch_id, outpoint_txid, outpoint_index, amt, completed
+        id, swap_hash, batch_id, outpoint, amt, completed
 FROM
         sweeps
 WHERE
@@ -57,8 +57,7 @@ func (q *Queries) GetBatchSweeps(ctx context.Context, batchID int32) ([]Sweep, e
 			&i.ID,
 			&i.SwapHash,
 			&i.BatchID,
-			&i.OutpointTxid,
-			&i.OutpointIndex,
+			&i.Outpoint,
 			&i.Amt,
 			&i.Completed,
 		); err != nil {
@@ -101,11 +100,11 @@ FROM
 JOIN
         sweeps ON sweep_batches.id = sweeps.batch_id
 WHERE
-        sweeps.swap_hash = $1
+        sweeps.outpoint = $1
 `
 
-func (q *Queries) GetParentBatch(ctx context.Context, swapHash []byte) (SweepBatch, error) {
-	row := q.db.QueryRowContext(ctx, getParentBatch, swapHash)
+func (q *Queries) GetParentBatch(ctx context.Context, outpoint string) (SweepBatch, error) {
+	row := q.db.QueryRowContext(ctx, getParentBatch, outpoint)
 	var i SweepBatch
 	err := row.Scan(
 		&i.ID,
@@ -125,11 +124,11 @@ SELECT
 FROM
     (SELECT false AS false_value) AS f
 LEFT JOIN
-    sweeps s ON s.swap_hash = $1
+    sweeps s ON s.outpoint = $1
 `
 
-func (q *Queries) GetSweepStatus(ctx context.Context, swapHash []byte) (bool, error) {
-	row := q.db.QueryRowContext(ctx, getSweepStatus, swapHash)
+func (q *Queries) GetSweepStatus(ctx context.Context, outpoint string) (bool, error) {
+	row := q.db.QueryRowContext(ctx, getSweepStatus, outpoint)
 	var completed bool
 	err := row.Scan(&completed)
 	return completed, err
@@ -251,8 +250,7 @@ const upsertSweep = `-- name: UpsertSweep :exec
 INSERT INTO sweeps (
         swap_hash,
         batch_id,
-        outpoint_txid,
-        outpoint_index,
+        outpoint,
         amt,
         completed
 ) VALUES (
@@ -260,31 +258,25 @@ INSERT INTO sweeps (
         $2,
         $3,
         $4,
-        $5,
-        $6
-) ON CONFLICT (swap_hash) DO UPDATE SET
+        $5
+) ON CONFLICT (outpoint) DO UPDATE SET
         batch_id = $2,
-        outpoint_txid = $3,
-        outpoint_index = $4,
-        amt = $5,
-        completed = $6
+        completed = $5
 `
 
 type UpsertSweepParams struct {
-	SwapHash      []byte
-	BatchID       int32
-	OutpointTxid  []byte
-	OutpointIndex int32
-	Amt           int64
-	Completed     bool
+	SwapHash  []byte
+	BatchID   int32
+	Outpoint  string
+	Amt       int64
+	Completed bool
 }
 
 func (q *Queries) UpsertSweep(ctx context.Context, arg UpsertSweepParams) error {
 	_, err := q.db.ExecContext(ctx, upsertSweep,
 		arg.SwapHash,
 		arg.BatchID,
-		arg.OutpointTxid,
-		arg.OutpointIndex,
+		arg.Outpoint,
 		arg.Amt,
 		arg.Completed,
 	)

loopdb/sqlc/migrations/000013_batcher_key_outpoint.down.sql

@@ -0,0 +1,3 @@
+-- We kept old table as sweeps_old. Use it.
+ALTER TABLE sweeps RENAME TO sweeps_new;
+ALTER TABLE sweeps_old RENAME TO sweeps;

loopdb/sqlc/migrations/000013_batcher_key_outpoint.up.sql

@@ -0,0 +1,67 @@
+-- We want to make column swap_hash non-unique and to use the outpoint as a key.
+-- We can't make a column non-unique or remove it in sqlite, so work around.
+-- See https://stackoverflow.com/a/42013422
+
+-- We also made outpoint a single point replacing columns outpoint_txid and
+-- outpoint_index.
+
+-- sweeps stores the individual sweeps that are part of a batch.
+CREATE TABLE sweeps2 (
+        -- id is the autoincrementing primary key.
+        id INTEGER PRIMARY KEY,
+
+        -- swap_hash is the hash of the swap that is being swept.
+        swap_hash BLOB NOT NULL,
+
+        -- batch_id is the id of the batch this swap is part of.
+        batch_id INTEGER NOT NULL,
+
+        -- outpoint is the UTXO id of the output being swept ("txid:index").
+        outpoint TEXT NOT NULL UNIQUE,
+
+        -- amt is the amount of the output being swept.
+        amt BIGINT NOT NULL,
+
+        -- completed indicates whether the sweep has been completed.
+        completed BOOLEAN NOT NULL DEFAULT FALSE,
+
+        -- Foreign key constraint to ensure that we reference an existing batch
+        -- id.
+        FOREIGN KEY (batch_id) REFERENCES sweep_batches(id),
+
+        -- Foreign key constraint to ensure that swap_hash references an
+        -- existing swap.
+        FOREIGN KEY (swap_hash) REFERENCES swaps(swap_hash)
+);
+
+-- Copy all the data from sweeps to sweeps2.
+-- Explanation:
+-- - seq(i) goes from 1 to 32
+-- - substr(outpoint_txid, 32+1-i, 1) indexes BLOB bytes in reverse order
+--   (SQLite uses 1-based indexing)
+-- - hex(...) gives uppercase by default, so wrapped in lower(...)
+-- - group_concat(..., '') combines all hex digits
+-- - concatenated with ':' || CAST(outpoint_index AS TEXT) for full outpoint.
+WITH RECURSIVE seq(i) AS (
+  SELECT 1
+  UNION ALL
+  SELECT i + 1 FROM seq WHERE i < 32
+)
+INSERT INTO sweeps2 (
+    id, swap_hash, batch_id, outpoint, amt, completed
+)
+SELECT
+    id,
+    swap_hash,
+    batch_id,
+    (
+      SELECT lower(group_concat(hex(substr(outpoint_txid,32+1-i,1)),''))
+      FROM seq
+    ) || ':' || CAST(outpoint_index AS TEXT),
+    amt,
+    completed
+FROM sweeps;
+
+-- Rename tables.
+ALTER TABLE sweeps RENAME TO sweeps_old;
+ALTER TABLE sweeps2 RENAME TO sweeps;

loopdb/sqlc/models.go

@@ -180,13 +180,12 @@ type SwapUpdate struct {
 }
 
 type Sweep struct {
-	ID            int32
-	SwapHash      []byte
-	BatchID       int32
-	OutpointTxid  []byte
-	OutpointIndex int32
-	Amt           int64
-	Completed     bool
+	ID        int32
+	SwapHash  []byte
+	BatchID   int32
+	Outpoint  string
+	Amt       int64
+	Completed bool
 }
 
 type SweepBatch struct {
@@ -198,3 +197,13 @@ type SweepBatch struct {
 	LastRbfSatPerKw    sql.NullInt32
 	MaxTimeoutDistance int32
 }
+
+type SweepsOld struct {
+	ID            int32
+	SwapHash      []byte
+	BatchID       int32
+	OutpointTxid  []byte
+	OutpointIndex int32
+	Amt           int64
+	Completed     bool
+}

loopdb/sqlc/querier.go

@@ -32,7 +32,7 @@ type Querier interface {
 	GetLoopOutSwap(ctx context.Context, swapHash []byte) (GetLoopOutSwapRow, error)
 	GetLoopOutSwaps(ctx context.Context) ([]GetLoopOutSwapsRow, error)
 	GetMigration(ctx context.Context, migrationID string) (MigrationTracker, error)
-	GetParentBatch(ctx context.Context, swapHash []byte) (SweepBatch, error)
+	GetParentBatch(ctx context.Context, outpoint string) (SweepBatch, error)
 	GetReservation(ctx context.Context, reservationID []byte) (Reservation, error)
 	GetReservationUpdates(ctx context.Context, reservationID []byte) ([]ReservationUpdate, error)
 	GetReservations(ctx context.Context) ([]Reservation, error)
@@ -40,7 +40,7 @@ type Querier interface {
 	GetStaticAddressLoopInSwap(ctx context.Context, swapHash []byte) (GetStaticAddressLoopInSwapRow, error)
 	GetStaticAddressLoopInSwapsByStates(ctx context.Context, dollar_1 sql.NullString) ([]GetStaticAddressLoopInSwapsByStatesRow, error)
 	GetSwapUpdates(ctx context.Context, swapHash []byte) ([]SwapUpdate, error)
-	GetSweepStatus(ctx context.Context, swapHash []byte) (bool, error)
+	GetSweepStatus(ctx context.Context, outpoint string) (bool, error)
 	GetUnconfirmedBatches(ctx context.Context) ([]SweepBatch, error)
 	InsertBatch(ctx context.Context, arg InsertBatchParams) (int32, error)
 	InsertDepositUpdate(ctx context.Context, arg InsertDepositUpdateParams) error

loopdb/sqlc/queries/batch.sql

@@ -47,8 +47,7 @@ WHERE
 INSERT INTO sweeps (
         swap_hash,
         batch_id,
-        outpoint_txid,
-        outpoint_index,
+        outpoint,
         amt,
         completed
 ) VALUES (
@@ -56,14 +55,10 @@ INSERT INTO sweeps (
         $2,
         $3,
         $4,
-        $5,
-        $6
-) ON CONFLICT (swap_hash) DO UPDATE SET
+        $5
+) ON CONFLICT (outpoint) DO UPDATE SET
         batch_id = $2,
-        outpoint_txid = $3,
-        outpoint_index = $4,
-        amt = $5,
-        completed = $6;
+        completed = $5;
 
 -- name: GetParentBatch :one
 SELECT
@@ -73,7 +68,7 @@ FROM
 JOIN
         sweeps ON sweep_batches.id = sweeps.batch_id
 WHERE
-        sweeps.swap_hash = $1;
+        sweeps.outpoint = $1;
 
 -- name: GetBatchSweptAmount :one
 SELECT
@@ -101,4 +96,4 @@ SELECT
 FROM
     (SELECT false AS false_value) AS f
 LEFT JOIN
-    sweeps s ON s.swap_hash = $1;
+    sweeps s ON s.outpoint = $1;