From f258d0c8d2b17e9c6e54e68aa0585e1db34fdd78 Mon Sep 17 00:00:00 2001
From: Calvin Kim <calvin@kcalvinalvin.info>
Date: Tue, 24 Jan 2023 15:53:26 +0900
Subject: [PATCH 01/14] database/ffldb: Change scanBlockFiles behavior

This change is part of the effort to add pruning support to btcd.

scanBlockFiles nows supports scanning files from an arbitrary block
number.  When blocks are pruned, the file number may not start from 0.
To account for this, scanBlockFiles now scans and retreives the start
and the end block file numbers and scans those files.
---
 database/ffldb/blockio.go | 72 +++++++++++++++++++++++++++------------
 database/ffldb/db.go      |  5 ++-
 2 files changed, 54 insertions(+), 23 deletions(-)

diff --git a/database/ffldb/blockio.go b/database/ffldb/blockio.go
index 94d2665e..2b415a17 100644
--- a/database/ffldb/blockio.go
+++ b/database/ffldb/blockio.go
@@ -15,6 +15,9 @@ import (
 	"io"
 	"os"
 	"path/filepath"
+	"sort"
+	"strconv"
+	"strings"
 	"sync"
 
 	"github.com/btcsuite/btcd/chaincfg/chainhash"
@@ -23,6 +26,10 @@ import (
 )
 
 const (
+	// blockFileExtension is the extension that's used to store the block
+	// files on the disk.
+	blockFileExtension = ".fdb"
+
 	// The Bitcoin protocol encodes block height as int32, so max number of
 	// blocks is 2^31.  Max block size per the protocol is 32MiB per block.
 	// So the theoretical max at the time this comment was written is 64PiB
@@ -32,7 +39,7 @@ const (
 	// 512MiB each for a total of ~476.84PiB (roughly 7.4 times the current
 	// theoretical max), so there is room for the max block size to grow in
 	// the future.
-	blockFilenameTemplate = "%09d.fdb"
+	blockFilenameTemplate = "%09d" + blockFileExtension
 
 	// maxOpenFiles is the max number of open files to maintain in the
 	// open blocks cache.  Note that this does not include the current
@@ -713,36 +720,57 @@ func (s *blockStore) handleRollback(oldBlockFileNum, oldBlockOffset uint32) {
 }
 
 // scanBlockFiles searches the database directory for all flat block files to
-// find the end of the most recent file.  This position is considered the
-// current write cursor which is also stored in the metadata.  Thus, it is used
-// to detect unexpected shutdowns in the middle of writes so the block files
-// can be reconciled.
-func scanBlockFiles(dbPath string) (int, uint32) {
-	lastFile := -1
-	fileLen := uint32(0)
-	for i := 0; ; i++ {
-		filePath := blockFilePath(dbPath, uint32(i))
-		st, err := os.Stat(filePath)
-		if err != nil {
-			break
-		}
-		lastFile = i
+// find the first file, last file, and the end of the most recent file.  The
+// position at the last file is considered the current write cursor which is
+// also stored in the metadata.  Thus, it is used to detect unexpected shutdowns
+// in the middle of writes so the block files can be reconciled.
+func scanBlockFiles(dbPath string) (int, int, uint32, error) {
+	firstFile, lastFile, lastFileLen, err := int(-1), int(-1), uint32(0), error(nil)
 
-		fileLen = uint32(st.Size())
+	files, err := filepath.Glob(filepath.Join(dbPath, "*"+blockFileExtension))
+	if err != nil {
+		return 0, 0, 0, err
+	}
+	sort.Strings(files)
+
+	// Return early if there's no block files.
+	if len(files) == 0 {
+		return firstFile, lastFile, lastFileLen, nil
 	}
 
-	log.Tracef("Scan found latest block file #%d with length %d", lastFile,
-		fileLen)
-	return lastFile, fileLen
+	// Grab the first and last file's number.
+	firstFile, err = strconv.Atoi(strings.TrimSuffix(filepath.Base(files[0]), blockFileExtension))
+	if err != nil {
+		return 0, 0, 0, fmt.Errorf("scanBlockFiles error: %v", err)
+	}
+	lastFile, err = strconv.Atoi(strings.TrimSuffix(filepath.Base(files[len(files)-1]), blockFileExtension))
+	if err != nil {
+		return 0, 0, 0, fmt.Errorf("scanBlockFiles error: %v", err)
+	}
+
+	// Get the last file's length.
+	filePath := blockFilePath(dbPath, uint32(lastFile))
+	st, err := os.Stat(filePath)
+	if err != nil {
+		return 0, 0, 0, err
+	}
+	lastFileLen = uint32(st.Size())
+
+	log.Tracef("Scan found latest block file #%d with length %d", lastFile, lastFileLen)
+
+	return firstFile, lastFile, lastFileLen, err
 }
 
 // newBlockStore returns a new block store with the current block file number
 // and offset set and all fields initialized.
-func newBlockStore(basePath string, network wire.BitcoinNet) *blockStore {
+func newBlockStore(basePath string, network wire.BitcoinNet) (*blockStore, error) {
 	// Look for the end of the latest block to file to determine what the
 	// write cursor position is from the viewpoing of the block files on
 	// disk.
-	fileNum, fileOff := scanBlockFiles(basePath)
+	_, fileNum, fileOff, err := scanBlockFiles(basePath)
+	if err != nil {
+		return nil, err
+	}
 	if fileNum == -1 {
 		fileNum = 0
 		fileOff = 0
@@ -765,5 +793,5 @@ func newBlockStore(basePath string, network wire.BitcoinNet) *blockStore {
 	store.openFileFunc = store.openFile
 	store.openWriteFileFunc = store.openWriteFile
 	store.deleteFileFunc = store.deleteFile
-	return store
+	return store, nil
 }
diff --git a/database/ffldb/db.go b/database/ffldb/db.go
index b796c3ae..e8b1ce9f 100644
--- a/database/ffldb/db.go
+++ b/database/ffldb/db.go
@@ -2016,7 +2016,10 @@ func openDB(dbPath string, network wire.BitcoinNet, create bool) (database.DB, e
 	// according to the data that is actually on disk.  Also create the
 	// database cache which wraps the underlying leveldb database to provide
 	// write caching.
-	store := newBlockStore(dbPath, network)
+	store, err := newBlockStore(dbPath, network)
+	if err != nil {
+		return nil, convertErr(err.Error(), err)
+	}
 	cache := newDbCache(ldb, store, defaultCacheSize, defaultFlushSecs)
 	pdb := &db{store: store, cache: cache}
 

From 5c1dd21e793c869cdf81e8fb0dd82f0349931d93 Mon Sep 17 00:00:00 2001
From: Calvin Kim <calvin@kcalvinalvin.info>
Date: Tue, 24 Jan 2023 16:52:06 +0900
Subject: [PATCH 02/14] database/ffldb: Add PruneBlocks to db interface

This change is part of the effort to add pruning support to btcd.

PruneBlocks will prune the earliest block files until it reaches the
given target size.  The returned hashes are the hashes of the blocks
that were pruned.
---
 database/ffldb/db.go          |  95 ++++++++++++++++++++++++
 database/ffldb/driver_test.go | 131 ++++++++++++++++++++++++++++++++++
 database/ffldb/export_test.go |   4 +-
 database/interface.go         |  15 ++++
 4 files changed, 244 insertions(+), 1 deletion(-)

diff --git a/database/ffldb/db.go b/database/ffldb/db.go
index e8b1ce9f..f8ac1e07 100644
--- a/database/ffldb/db.go
+++ b/database/ffldb/db.go
@@ -1669,6 +1669,101 @@ func (tx *transaction) writePendingAndCommit() error {
 	return tx.db.cache.commitTx(tx)
 }
 
+// PruneBlocks deletes the block files until it reaches the target size
+// (specified in bytes).  Throws an error if the target size is below
+// the maximum size of a single block file.
+//
+// This function is part of the database.Tx interface implementation.
+func (tx *transaction) PruneBlocks(targetSize uint64) ([]chainhash.Hash, error) {
+	// Ensure transaction state is valid.
+	if err := tx.checkClosed(); err != nil {
+		return nil, err
+	}
+
+	// Ensure the transaction is writable.
+	if !tx.writable {
+		str := "prune blocks requires a writable database transaction"
+		return nil, makeDbErr(database.ErrTxNotWritable, str, nil)
+	}
+
+	// Make a local alias for the maxBlockFileSize.
+	maxSize := uint64(tx.db.store.maxBlockFileSize)
+	if targetSize < maxSize {
+		return nil, fmt.Errorf("got target size of %d but it must be greater "+
+			"than %d, the max size of a single block file",
+			targetSize, maxSize)
+	}
+
+	first, last, lastFileSize, err := scanBlockFiles(tx.db.store.basePath)
+	if err != nil {
+		return nil, err
+	}
+
+	// If we have no files on disk or just a single file on disk, return early.
+	if first == last {
+		return nil, nil
+	}
+
+	// Last file number minus the first file number gives us the count of files
+	// on disk minus 1.  We don't want to count the last file since we can't assume
+	// that it is of max size.
+	maxSizeFileCount := last - first
+
+	// If the total size of block files are under the target, return early and
+	// don't prune.
+	totalSize := uint64(lastFileSize) + (maxSize * uint64(maxSizeFileCount))
+	if totalSize <= targetSize {
+		return nil, nil
+	}
+
+	log.Tracef("Using %d more bytes than the target of %d MiB. Pruning files...",
+		totalSize-targetSize,
+		targetSize/(1024*1024))
+
+	deletedFiles := make(map[uint32]struct{})
+
+	// We use < not <= so that the last file is never deleted.  There are other checks in place
+	// but setting it to < here doesn't hurt.
+	for i := uint32(first); i < uint32(last); i++ {
+		err = tx.db.store.deleteFileFunc(i)
+		if err != nil {
+			return nil, fmt.Errorf("PruneBlocks: Failed to delete block file "+
+				"number %d: %v", i, err)
+		}
+
+		// Add the file index to the deleted files map so that we can later
+		// delete the block location index.
+		deletedFiles[i] = struct{}{}
+
+		// If we're already at or below the target usage, break and don't
+		// try to delete more files.
+		totalSize -= maxSize
+		if totalSize <= targetSize {
+			break
+		}
+	}
+
+	// Delete the indexed block locations for the files that we've just deleted.
+	var deletedBlockHashes []chainhash.Hash
+	cursor := tx.blockIdxBucket.Cursor()
+	for ok := cursor.First(); ok; ok = cursor.Next() {
+		loc := deserializeBlockLoc(cursor.Value())
+
+		_, found := deletedFiles[loc.blockFileNum]
+		if found {
+			deletedBlockHashes = append(deletedBlockHashes, *(*chainhash.Hash)(cursor.Key()))
+			err := cursor.Delete()
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+
+	log.Tracef("Finished pruning. Database now at %d bytes", totalSize)
+
+	return deletedBlockHashes, nil
+}
+
 // Commit commits all changes that have been made to the root metadata bucket
 // and all of its sub-buckets to the database cache which is periodically synced
 // to persistent storage.  In addition, it commits all new blocks directly to
diff --git a/database/ffldb/driver_test.go b/database/ffldb/driver_test.go
index 29d31eaf..5b9a7533 100644
--- a/database/ffldb/driver_test.go
+++ b/database/ffldb/driver_test.go
@@ -5,6 +5,7 @@
 package ffldb_test
 
 import (
+	"bytes"
 	"fmt"
 	"os"
 	"path/filepath"
@@ -13,6 +14,7 @@ import (
 
 	"github.com/btcsuite/btcd/btcutil"
 	"github.com/btcsuite/btcd/chaincfg"
+	"github.com/btcsuite/btcd/chaincfg/chainhash"
 	"github.com/btcsuite/btcd/database"
 	"github.com/btcsuite/btcd/database/ffldb"
 )
@@ -253,6 +255,135 @@ func TestPersistence(t *testing.T) {
 	}
 }
 
+// TestPrune tests that the older .fdb files are deleted with a call to prune.
+func TestPrune(t *testing.T) {
+	t.Parallel()
+
+	// Create a new database to run tests against.
+	dbPath := t.TempDir()
+	db, err := database.Create(dbType, dbPath, blockDataNet)
+	if err != nil {
+		t.Errorf("Failed to create test database (%s) %v", dbType, err)
+		return
+	}
+	defer db.Close()
+
+	blockFileSize := uint64(2048)
+
+	testfn := func(t *testing.T, db database.DB) {
+		// Load the test blocks and save in the test context for use throughout
+		// the tests.
+		blocks, err := loadBlocks(t, blockDataFile, blockDataNet)
+		if err != nil {
+			t.Errorf("loadBlocks: Unexpected error: %v", err)
+			return
+		}
+		err = db.Update(func(tx database.Tx) error {
+			for i, block := range blocks {
+				err := tx.StoreBlock(block)
+				if err != nil {
+					return fmt.Errorf("StoreBlock #%d: unexpected error: "+
+						"%v", i, err)
+				}
+			}
+			return nil
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		blockHashMap := make(map[chainhash.Hash][]byte, len(blocks))
+		for _, block := range blocks {
+			bytes, err := block.Bytes()
+			if err != nil {
+				t.Fatal(err)
+			}
+			blockHashMap[*block.Hash()] = bytes
+		}
+
+		err = db.Update(func(tx database.Tx) error {
+			_, err := tx.PruneBlocks(1024)
+			if err == nil {
+				return fmt.Errorf("Expected an error when attempting to prune" +
+					"below the maxFileSize")
+			}
+
+			_, err = tx.PruneBlocks(0)
+			if err == nil {
+				return fmt.Errorf("Expected an error when attempting to prune" +
+					"below the maxFileSize")
+			}
+
+			return nil
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		var deletedBlocks []chainhash.Hash
+
+		// This should leave 3 files on disk.
+		err = db.Update(func(tx database.Tx) error {
+			deletedBlocks, err = tx.PruneBlocks(blockFileSize * 3)
+			return err
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// The only error we can get is a bad pattern error.  Since we're hardcoding
+		// the pattern, we should not have an error at runtime.
+		files, _ := filepath.Glob(filepath.Join(dbPath, "*.fdb"))
+		if len(files) != 3 {
+			t.Fatalf("Expected to find %d files but got %d",
+				3, len(files))
+		}
+
+		// Check that all the blocks that say were deleted are deleted from the
+		// block index bucket as well.
+		err = db.View(func(tx database.Tx) error {
+			for _, deletedBlock := range deletedBlocks {
+				_, err := tx.FetchBlock(&deletedBlock)
+				if dbErr, ok := err.(database.Error); !ok ||
+					dbErr.ErrorCode != database.ErrBlockNotFound {
+
+					return fmt.Errorf("Expected ErrBlockNotFound "+
+						"but got %v", dbErr)
+				}
+			}
+
+			return nil
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// Check that the not deleted blocks are present.
+		for _, deletedBlock := range deletedBlocks {
+			delete(blockHashMap, deletedBlock)
+		}
+		err = db.View(func(tx database.Tx) error {
+			for hash, wantBytes := range blockHashMap {
+				gotBytes, err := tx.FetchBlock(&hash)
+				if err != nil {
+					return err
+				}
+				if !bytes.Equal(gotBytes, wantBytes) {
+					return fmt.Errorf("got bytes %x, want bytes %x",
+						gotBytes, wantBytes)
+				}
+			}
+			return nil
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+	}
+	ffldb.TstRunWithMaxBlockFileSize(db, uint32(blockFileSize), func() {
+		testfn(t, db)
+	})
+}
+
 // TestInterface performs all interfaces tests for this database driver.
 func TestInterface(t *testing.T) {
 	t.Parallel()
diff --git a/database/ffldb/export_test.go b/database/ffldb/export_test.go
index 2d8e4d2a..cbb6dc94 100644
--- a/database/ffldb/export_test.go
+++ b/database/ffldb/export_test.go
@@ -11,7 +11,9 @@ The functions are only exported while the tests are being run.
 
 package ffldb
 
-import "github.com/btcsuite/btcd/database"
+import (
+	"github.com/btcsuite/btcd/database"
+)
 
 // TstRunWithMaxBlockFileSize runs the passed function with the maximum allowed
 // file size for the database set to the provided value.  The value will be set
diff --git a/database/interface.go b/database/interface.go
index d4f1d89d..789c823a 100644
--- a/database/interface.go
+++ b/database/interface.go
@@ -389,6 +389,21 @@ type Tx interface {
 	// implementations.
 	FetchBlockRegions(regions []BlockRegion) ([][]byte, error)
 
+	// PruneBlocks deletes the block files until it reaches the target size
+	// (specificed in bytes).
+	//
+	// The interface contract guarantees at least the following errors will
+	// be returned (other implementation-specific errors are possible):
+	//   - ErrTxNotWritable if attempted against a read-only transaction
+	//   - ErrTxClosed if the transaction has already been closed
+	//
+	// NOTE: The data returned by this function is only valid during a
+	// database transaction.  Attempting to access it after a transaction
+	// has ended results in undefined behavior.  This constraint prevents
+	// additional data copies and allows support for memory-mapped database
+	// implementations.
+	PruneBlocks(targetSize uint64) ([]chainhash.Hash, error)
+
 	// ******************************************************************
 	// Methods related to both atomic metadata storage and block storage.
 	// ******************************************************************

From 57ec43fedc30971ffa924d5fc682cf1a79a4f306 Mon Sep 17 00:00:00 2001
From: Calvin Kim <calvin@kcalvinalvin.info>
Date: Thu, 27 Apr 2023 13:31:14 +0900
Subject: [PATCH 03/14] blockchain: Add pruning support

This change is part of the effort to add pruning support to btcd.

A field for pruning is added and the BlockChain struct is now able to be
configured with pruning.  Prune is called on every block connect and
the prune target field is passed to PruneBlocks func.  There's no check
to keep the latest 288 blocks to abide by the NODE_NETWORK_LIMITED rule.
That'll have to be enforced by the caller creating the BlockChain
struct.
---
 blockchain/chain.go   | 30 ++++++++++++++++++++++++++++++
 blockchain/chainio.go | 15 +++++++++++++++
 2 files changed, 45 insertions(+)

diff --git a/blockchain/chain.go b/blockchain/chain.go
index c013ff3b..3ead971f 100644
--- a/blockchain/chain.go
+++ b/blockchain/chain.go
@@ -115,6 +115,10 @@ type BlockChain struct {
 	// fields in this struct below this point.
 	chainLock sync.RWMutex
 
+	// pruneTarget is the size in bytes the database targets for when the node
+	// is pruned.
+	pruneTarget uint64
+
 	// These fields are related to the memory block index.  They both have
 	// their own locks, however they are often also protected by the chain
 	// lock to help prevent logic races when blocks are being processed.
@@ -600,6 +604,26 @@ func (b *BlockChain) connectBlock(node *blockNode, block *btcutil.Block,
 
 	// Atomically insert info into the database.
 	err = b.db.Update(func(dbTx database.Tx) error {
+		// If the pruneTarget isn't 0, we should attempt to delete older blocks
+		// from the database.
+		if b.pruneTarget != 0 {
+			// When the total block size is under the prune target, prune blocks is
+			// a no-op and the deleted hashes are nil.
+			deletedHashes, err := dbTx.PruneBlocks(b.pruneTarget)
+			if err != nil {
+				return err
+			}
+
+			// Only attempt to delete if we have any deleted blocks.
+			if len(deletedHashes) != 0 {
+				// Delete the spend journals of the pruned blocks.
+				err = dbPruneSpendJournalEntry(dbTx, deletedHashes)
+				if err != nil {
+					return err
+				}
+			}
+		}
+
 		// Update best block state.
 		err := dbPutBestState(dbTx, state, node.workSum)
 		if err != nil {
@@ -1702,6 +1726,11 @@ type Config struct {
 	// This field can be nil if the caller is not interested in using a
 	// signature cache.
 	HashCache *txscript.HashCache
+
+	// Prune specifies the target database usage (in bytes) the database
+	// will target for with block files.  Prune at 0 specifies that no
+	// blocks will be deleted.
+	Prune uint64
 }
 
 // New returns a BlockChain instance using the provided configuration details.
@@ -1757,6 +1786,7 @@ func New(config *Config) (*BlockChain, error) {
 		prevOrphans:         make(map[chainhash.Hash][]*orphanBlock),
 		warningCaches:       newThresholdCaches(vbNumBits),
 		deploymentCaches:    newThresholdCaches(chaincfg.DefinedDeployments),
+		pruneTarget:         config.Prune,
 	}
 
 	// Ensure all the deployments are synchronized with our clock if
diff --git a/blockchain/chainio.go b/blockchain/chainio.go
index 4914da68..21f0d5ba 100644
--- a/blockchain/chainio.go
+++ b/blockchain/chainio.go
@@ -494,6 +494,21 @@ func dbRemoveSpendJournalEntry(dbTx database.Tx, blockHash *chainhash.Hash) erro
 	return spendBucket.Delete(blockHash[:])
 }
 
+// dbPruneSpendJournalEntry uses an existing database transaction to remove all
+// the spend journal entries for the pruned blocks.
+func dbPruneSpendJournalEntry(dbTx database.Tx, blockHashes []chainhash.Hash) error {
+	spendBucket := dbTx.Metadata().Bucket(spendJournalBucketName)
+
+	for _, blockHash := range blockHashes {
+		err := spendBucket.Delete(blockHash[:])
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
 // -----------------------------------------------------------------------------
 // The unspent transaction output (utxo) set consists of an entry for each
 // unspent output using a format that is optimized to reduce space using domain

From 0212c334ce901991395d973fd900996de3312115 Mon Sep 17 00:00:00 2001
From: Calvin Kim <calvin@kcalvinalvin.info>
Date: Thu, 26 Jan 2023 14:29:16 +0900
Subject: [PATCH 04/14] wire, main: Add SFNodeNetworkLimited flag

This change is part of the effort to add pruning support to btcd.

Wire now supports the ability to signal NODE_NETWORK_LIMITED which
signals to peers that the node is able to serve the last 288 blocks.

Since archival nodes have all blocks, they can also signal for
NODE_NETWORK_LIMITED.  SFNodeNetworkLimited flag is added to the default
services.
---
 server.go             |  4 ++--
 wire/protocol.go      | 22 ++++++++++++++--------
 wire/protocol_test.go |  3 ++-
 3 files changed, 18 insertions(+), 11 deletions(-)

diff --git a/server.go b/server.go
index 5ef2e319..932363c1 100644
--- a/server.go
+++ b/server.go
@@ -44,8 +44,8 @@ import (
 const (
 	// defaultServices describes the default services that are supported by
 	// the server.
-	defaultServices = wire.SFNodeNetwork | wire.SFNodeBloom |
-		wire.SFNodeWitness | wire.SFNodeCF
+	defaultServices = wire.SFNodeNetwork | wire.SFNodeNetworkLimited |
+		wire.SFNodeBloom | wire.SFNodeWitness | wire.SFNodeCF
 
 	// defaultRequiredServices describes the default services that are
 	// required to be supported by outbound peers.
diff --git a/wire/protocol.go b/wire/protocol.go
index 3b414ec3..be6fc4ad 100644
--- a/wire/protocol.go
+++ b/wire/protocol.go
@@ -93,18 +93,23 @@ const (
 	// SFNode2X is a flag used to indicate a peer is running the Segwit2X
 	// software.
 	SFNode2X
+
+	// SFNodeNetWorkLimited is a flag used to indicate a peer supports serving
+	// the last 288 blocks.
+	SFNodeNetworkLimited = 1 << 10
 )
 
 // Map of service flags back to their constant names for pretty printing.
 var sfStrings = map[ServiceFlag]string{
-	SFNodeNetwork: "SFNodeNetwork",
-	SFNodeGetUTXO: "SFNodeGetUTXO",
-	SFNodeBloom:   "SFNodeBloom",
-	SFNodeWitness: "SFNodeWitness",
-	SFNodeXthin:   "SFNodeXthin",
-	SFNodeBit5:    "SFNodeBit5",
-	SFNodeCF:      "SFNodeCF",
-	SFNode2X:      "SFNode2X",
+	SFNodeNetwork:        "SFNodeNetwork",
+	SFNodeGetUTXO:        "SFNodeGetUTXO",
+	SFNodeBloom:          "SFNodeBloom",
+	SFNodeWitness:        "SFNodeWitness",
+	SFNodeXthin:          "SFNodeXthin",
+	SFNodeBit5:           "SFNodeBit5",
+	SFNodeCF:             "SFNodeCF",
+	SFNode2X:             "SFNode2X",
+	SFNodeNetworkLimited: "SFNodeNetworkLimited",
 }
 
 // orderedSFStrings is an ordered list of service flags from highest to
@@ -118,6 +123,7 @@ var orderedSFStrings = []ServiceFlag{
 	SFNodeBit5,
 	SFNodeCF,
 	SFNode2X,
+	SFNodeNetworkLimited,
 }
 
 // String returns the ServiceFlag in human-readable form.
diff --git a/wire/protocol_test.go b/wire/protocol_test.go
index 60bd0533..4a57c30c 100644
--- a/wire/protocol_test.go
+++ b/wire/protocol_test.go
@@ -21,7 +21,8 @@ func TestServiceFlagStringer(t *testing.T) {
 		{SFNodeBit5, "SFNodeBit5"},
 		{SFNodeCF, "SFNodeCF"},
 		{SFNode2X, "SFNode2X"},
-		{0xffffffff, "SFNodeNetwork|SFNodeGetUTXO|SFNodeBloom|SFNodeWitness|SFNodeXthin|SFNodeBit5|SFNodeCF|SFNode2X|0xffffff00"},
+		{SFNodeNetworkLimited, "SFNodeNetworkLimited"},
+		{0xffffffff, "SFNodeNetwork|SFNodeGetUTXO|SFNodeBloom|SFNodeWitness|SFNodeXthin|SFNodeBit5|SFNodeCF|SFNode2X|SFNodeNetworkLimited|0xfffffb00"},
 	}
 
 	t.Logf("Running %d tests", len(tests))

From 02469e16a6b0591dafeafc8a2cd42a973d82781f Mon Sep 17 00:00:00 2001
From: Calvin Kim <calvin@kcalvinalvin.info>
Date: Thu, 26 Jan 2023 14:58:57 +0900
Subject: [PATCH 05/14] main: Add prune flag

This change is part of the effort to add pruning support to btcd.

Pruning is now available to the end user via --prune flag.  There are
checks in place so that the user doesn't go below the minimum prune
target of 1536 MiB.  The minimum is set so that we keep at least 288
blocks per the requirement for NODE_NETWORK_LIMITED nodes specified by
BIP0159.  The default value of 0 will disable pruning.
---
 config.go | 10 ++++++++++
 server.go |  4 ++++
 2 files changed, 14 insertions(+)

diff --git a/config.go b/config.go
index 2d8c67e6..4597acfe 100644
--- a/config.go
+++ b/config.go
@@ -66,6 +66,7 @@ const (
 	sampleConfigFilename         = "sample-btcd.conf"
 	defaultTxIndex               = false
 	defaultAddrIndex             = false
+	pruneMinSize                 = 1536
 )
 
 var (
@@ -146,6 +147,7 @@ type config struct {
 	Proxy                string        `long:"proxy" description:"Connect via SOCKS5 proxy (eg. 127.0.0.1:9050)"`
 	ProxyPass            string        `long:"proxypass" default-mask:"-" description:"Password for proxy server"`
 	ProxyUser            string        `long:"proxyuser" description:"Username for proxy server"`
+	Prune                uint64        `long:"prune" description:"Prune already validated blocks from the database. Must specify a target size in MiB (minimum value of 1536, default value of 0 will disable pruning)"`
 	RegressionTest       bool          `long:"regtest" description:"Use the regression test network"`
 	RejectNonStd         bool          `long:"rejectnonstd" description:"Reject non-standard transactions regardless of the default settings for the active network."`
 	RejectReplacement    bool          `long:"rejectreplacement" description:"Reject transactions that attempt to replace existing transactions within the mempool through the Replace-By-Fee (RBF) signaling policy."`
@@ -1137,6 +1139,14 @@ func loadConfig() (*config, []string, error) {
 		}
 	}
 
+	if cfg.Prune != 0 && cfg.Prune < pruneMinSize {
+		err := fmt.Errorf("%s: the minimum value for --prune is %d. Got %d",
+			funcName, pruneMinSize, cfg.Prune)
+		fmt.Fprintln(os.Stderr, err)
+		fmt.Fprintln(os.Stderr, usageMessage)
+		return nil, nil, err
+	}
+
 	// Warn about missing config file only after all other configuration is
 	// done.  This prevents the warning on help messages and invalid
 	// options.  Note this should go directly before the return.
diff --git a/server.go b/server.go
index 932363c1..4e88d36c 100644
--- a/server.go
+++ b/server.go
@@ -2730,6 +2730,9 @@ func newServer(listenAddrs, agentBlacklist, agentWhitelist []string,
 	if cfg.NoCFilters {
 		services &^= wire.SFNodeCF
 	}
+	if cfg.Prune != 0 {
+		services &^= wire.SFNodeNetwork
+	}
 
 	amgr := addrmgr.New(cfg.DataDir, btcdLookup)
 
@@ -2831,6 +2834,7 @@ func newServer(listenAddrs, agentBlacklist, agentWhitelist []string,
 		SigCache:     s.sigCache,
 		IndexManager: indexManager,
 		HashCache:    s.hashCache,
+		Prune:        cfg.Prune * 1024 * 1024,
 	})
 	if err != nil {
 		return nil, err

From a1736b42678aab5e2157b98abc3f64df7c0be358 Mon Sep 17 00:00:00 2001
From: Calvin Kim <calvin@kcalvinalvin.info>
Date: Thu, 2 Mar 2023 15:07:22 +0900
Subject: [PATCH 06/14] main: Disable enabling both --prune and --txindex

You can have a txindex but with the actual blocks gone, they won't be
much of a help.  Consider allowing these option to be both on in the
future where the txindex is only indexing the non-pruned blocks.
---
 config.go | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/config.go b/config.go
index 4597acfe..f754d70c 100644
--- a/config.go
+++ b/config.go
@@ -1147,6 +1147,14 @@ func loadConfig() (*config, []string, error) {
 		return nil, nil, err
 	}
 
+	if cfg.Prune != 0 && cfg.TxIndex {
+		err := fmt.Errorf("%s: the --prune and --txindex options may "+
+			"not be activated at the same time", funcName)
+		fmt.Fprintln(os.Stderr, err)
+		fmt.Fprintln(os.Stderr, usageMessage)
+		return nil, nil, err
+	}
+
 	// Warn about missing config file only after all other configuration is
 	// done.  This prevents the warning on help messages and invalid
 	// options.  Note this should go directly before the return.

From 57903c71c902f757e593266bbb8889bde17debc3 Mon Sep 17 00:00:00 2001
From: Calvin Kim <calvin@kcalvinalvin.info>
Date: Mon, 3 Apr 2023 11:54:47 +0900
Subject: [PATCH 07/14] main: Disable enabling both --prune and --addrindex

You can have a addrindex but with the actual blocks gone, they won't be
much of a help.  Consider allowing these option to be both on in the
future where the addrindex is only indexing the non-pruned blocks.
---
 config.go | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/config.go b/config.go
index f754d70c..67c47dbd 100644
--- a/config.go
+++ b/config.go
@@ -1155,6 +1155,14 @@ func loadConfig() (*config, []string, error) {
 		return nil, nil, err
 	}
 
+	if cfg.Prune != 0 && cfg.AddrIndex {
+		err := fmt.Errorf("%s: the --prune and --addrindex options may "+
+			"not be activated at the same time", funcName)
+		fmt.Fprintln(os.Stderr, err)
+		fmt.Fprintln(os.Stderr, usageMessage)
+		return nil, nil, err
+	}
+
 	// Warn about missing config file only after all other configuration is
 	// done.  This prevents the warning on help messages and invalid
 	// options.  Note this should go directly before the return.

From aaedc11887a0347041208f935d544fd861a92430 Mon Sep 17 00:00:00 2001
From: Calvin Kim <calvin@kcalvinalvin.info>
Date: Tue, 22 Aug 2023 15:49:48 +0900
Subject: [PATCH 08/14] database, database/ffldb: add BeenPruned() method

This change is part of the effort to add pruning support to btcd.

BeenPruned will return true if the database has ever been pruned.  This
allows for accurate prune status to be reported as well as ux
improvements by disallowing the user to accidently remove or enable
indexes.
---
 database/ffldb/db.go          | 15 +++++++++++++++
 database/ffldb/driver_test.go | 31 +++++++++++++++++++++++++++++++
 database/interface.go         |  5 +++++
 3 files changed, 51 insertions(+)

diff --git a/database/ffldb/db.go b/database/ffldb/db.go
index f8ac1e07..1751c936 100644
--- a/database/ffldb/db.go
+++ b/database/ffldb/db.go
@@ -1764,6 +1764,21 @@ func (tx *transaction) PruneBlocks(targetSize uint64) ([]chainhash.Hash, error)
 	return deletedBlockHashes, nil
 }
 
+// BeenPruned returns if the block storage has ever been pruned.
+//
+// This function is part of the database.Tx interface implementation.
+func (tx *transaction) BeenPruned() (bool, error) {
+	first, last, _, err := scanBlockFiles(tx.db.store.basePath)
+	if err != nil {
+		return false, err
+	}
+
+	// If the database is pruned, then the first .fdb will not be there.
+	// We also check that there isn't just 1 file on disk or if there are
+	// no files on disk by checking if first != last.
+	return first != 0 && (first != last), nil
+}
+
 // Commit commits all changes that have been made to the root metadata bucket
 // and all of its sub-buckets to the database cache which is periodically synced
 // to persistent storage.  In addition, it commits all new blocks directly to
diff --git a/database/ffldb/driver_test.go b/database/ffldb/driver_test.go
index 5b9a7533..794e8e19 100644
--- a/database/ffldb/driver_test.go
+++ b/database/ffldb/driver_test.go
@@ -319,6 +319,21 @@ func TestPrune(t *testing.T) {
 		if err != nil {
 			t.Fatal(err)
 		}
+		err = db.View(func(tx database.Tx) error {
+			pruned, err := tx.BeenPruned()
+			if err != nil {
+				return err
+			}
+
+			if pruned {
+				err = fmt.Errorf("The database hasn't been pruned but " +
+					"BeenPruned returned true")
+			}
+			return err
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
 
 		var deletedBlocks []chainhash.Hash
 
@@ -339,6 +354,22 @@ func TestPrune(t *testing.T) {
 				3, len(files))
 		}
 
+		err = db.View(func(tx database.Tx) error {
+			pruned, err := tx.BeenPruned()
+			if err != nil {
+				return err
+			}
+
+			if !pruned {
+				err = fmt.Errorf("The database has been pruned but " +
+					"BeenPruned returned false")
+			}
+			return err
+		})
+		if err != nil {
+			t.Fatal(err)
+		}
+
 		// Check that all the blocks that say were deleted are deleted from the
 		// block index bucket as well.
 		err = db.View(func(tx database.Tx) error {
diff --git a/database/interface.go b/database/interface.go
index 789c823a..7efc7c55 100644
--- a/database/interface.go
+++ b/database/interface.go
@@ -404,6 +404,11 @@ type Tx interface {
 	// implementations.
 	PruneBlocks(targetSize uint64) ([]chainhash.Hash, error)
 
+	// BeenPruned returns if the block storage has ever been pruned.
+	//
+	// Implementation specific errors are possible.
+	BeenPruned() (bool, error)
+
 	// ******************************************************************
 	// Methods related to both atomic metadata storage and block storage.
 	// ******************************************************************

From 56f3463d9d1f1f73ed93eb334c28ad7439416d54 Mon Sep 17 00:00:00 2001
From: Calvin Kim <calvin@kcalvinalvin.info>
Date: Wed, 23 Aug 2023 00:21:19 +0900
Subject: [PATCH 09/14] main: force user to enable pruning if database is
 already pruned

This change is part of the effort to add pruning support to btcd.

Allowing the user to not pass in the --prune flag after pruning results
in inaccurate reporting of the prune status for getblockchaininfo and
for signaling NODE_NETWORK_LIMITED to peers.  Anything that relies on
cfg.Prune to be accurate is at risk of being incorrect.

To solve the current problems and to prevent potential future problems,
just force the user to keep the prune flag on like bitcoind.  In terms
of UX, there isn't that much of a loss since if the user wants to keep
more blocks than they previously did, they can just increase the size
passed to --prune.
---
 btcd.go | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/btcd.go b/btcd.go
index 8f853117..457288b7 100644
--- a/btcd.go
+++ b/btcd.go
@@ -157,6 +157,25 @@ func btcdMain(serverChan chan<- *server) error {
 		return nil
 	}
 
+	// Check if the database had previously been pruned.  If it had been, it's
+	// not possible to newly generate the tx index and addr index.
+	var beenPruned bool
+	db.View(func(dbTx database.Tx) error {
+		beenPruned, err = dbTx.BeenPruned()
+		return err
+	})
+	if err != nil {
+		btcdLog.Errorf("%v", err)
+		return err
+	}
+	if beenPruned && cfg.Prune == 0 {
+		err = fmt.Errorf("--prune cannot be disabled as the node has been "+
+			"previously pruned. You must delete the files in the datadir: \"%s\" "+
+			"and sync from the beginning to disable pruning", cfg.DataDir)
+		btcdLog.Errorf("%v", err)
+		return err
+	}
+
 	// The config file is already created if it did not exist and the log
 	// file has already been opened by now so we only need to allow
 	// creating rpc cert and key files if they don't exist.

From 47261ef205e606f11cddcce3cc4ce4b4b6f8f9aa Mon Sep 17 00:00:00 2001
From: Calvin Kim <calvin@kcalvinalvin.info>
Date: Tue, 22 Aug 2023 15:54:08 +0900
Subject: [PATCH 10/14] main: fetch prune status from db for
 handleGetBlockChainInfo

This change is part of the effort to add pruning support to btcd.

Now that pruning is allowed in btcd, accurately report the prune status
back to the user.
---
 rpcserver.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/rpcserver.go b/rpcserver.go
index b917263d..616adcb6 100644
--- a/rpcserver.go
+++ b/rpcserver.go
@@ -1200,7 +1200,7 @@ func handleGetBlockChainInfo(s *rpcServer, cmd interface{}, closeChan <-chan str
 		BestBlockHash: chainSnapshot.Hash.String(),
 		Difficulty:    getDifficultyRatio(chainSnapshot.Bits, params),
 		MedianTime:    chainSnapshot.MedianTime.Unix(),
-		Pruned:        false,
+		Pruned:        cfg.Prune != 0,
 		SoftForks: &btcjson.SoftForks{
 			Bip9SoftForks: make(map[string]*btcjson.Bip9SoftForkDescription),
 		},

From e27fcac9cd51acb9bd8082b6b74b89204d3130da Mon Sep 17 00:00:00 2001
From: Calvin Kim <calvin@kcalvinalvin.info>
Date: Tue, 22 Aug 2023 15:56:24 +0900
Subject: [PATCH 11/14] blockchain/indexers: add functions to report init
 status

This change is part of the effort to add pruning support to btcd.

The added *Initialized() functions to each of the indexers allow for
callers to check if each of the indexes have been created.  It's
useful for ux improvements where we force the user to manually drop
indexes that aren't compatible with pruning when the user enables
pruning.
---
 blockchain/indexers/addrindex.go | 12 ++++++++++++
 blockchain/indexers/cfindex.go   | 12 ++++++++++++
 blockchain/indexers/txindex.go   | 12 ++++++++++++
 3 files changed, 36 insertions(+)

diff --git a/blockchain/indexers/addrindex.go b/blockchain/indexers/addrindex.go
index 7e9f36f1..a4db7294 100644
--- a/blockchain/indexers/addrindex.go
+++ b/blockchain/indexers/addrindex.go
@@ -991,3 +991,15 @@ func NewAddrIndex(db database.DB, chainParams *chaincfg.Params) *AddrIndex {
 func DropAddrIndex(db database.DB, interrupt <-chan struct{}) error {
 	return dropIndex(db, addrIndexKey, addrIndexName, interrupt)
 }
+
+// AddrIndexInitialized returns true if the address index has been created previously.
+func AddrIndexInitialized(db database.DB) bool {
+	var exists bool
+	db.View(func(dbTx database.Tx) error {
+		bucket := dbTx.Metadata().Bucket(addrIndexKey)
+		exists = bucket != nil
+		return nil
+	})
+
+	return exists
+}
diff --git a/blockchain/indexers/cfindex.go b/blockchain/indexers/cfindex.go
index 21b4bf46..1af1d0a4 100644
--- a/blockchain/indexers/cfindex.go
+++ b/blockchain/indexers/cfindex.go
@@ -355,3 +355,15 @@ func NewCfIndex(db database.DB, chainParams *chaincfg.Params) *CfIndex {
 func DropCfIndex(db database.DB, interrupt <-chan struct{}) error {
 	return dropIndex(db, cfIndexParentBucketKey, cfIndexName, interrupt)
 }
+
+// CfIndexInitialized returns true if the cfindex has been created previously.
+func CfIndexInitialized(db database.DB) bool {
+	var exists bool
+	db.View(func(dbTx database.Tx) error {
+		bucket := dbTx.Metadata().Bucket(cfIndexParentBucketKey)
+		exists = bucket != nil
+		return nil
+	})
+
+	return exists
+}
diff --git a/blockchain/indexers/txindex.go b/blockchain/indexers/txindex.go
index f7d4bf60..3d4e9146 100644
--- a/blockchain/indexers/txindex.go
+++ b/blockchain/indexers/txindex.go
@@ -481,3 +481,15 @@ func DropTxIndex(db database.DB, interrupt <-chan struct{}) error {
 
 	return dropIndex(db, txIndexKey, txIndexName, interrupt)
 }
+
+// TxIndexInitialized returns true if the tx index has been created previously.
+func TxIndexInitialized(db database.DB) bool {
+	var exists bool
+	db.View(func(dbTx database.Tx) error {
+		bucket := dbTx.Metadata().Bucket(txIndexKey)
+		exists = bucket != nil
+		return nil
+	})
+
+	return exists
+}

From f161a31a93e9e721544782d2533e90bf70b74f25 Mon Sep 17 00:00:00 2001
From: Calvin Kim <calvin@kcalvinalvin.info>
Date: Tue, 22 Aug 2023 16:05:54 +0900
Subject: [PATCH 12/14] main: force the user to drop tx and addr indexes for
 pruning

This change is part of the effort to add pruning support to btcd.

The addr and tx indexes are not useful when the node is pruned as the
actual block data that the indexes point to are gone.  If the user had
previously enabled them, then explicitly require an action from the user
to remove the indexes before letting the user enable pruning.
---
 btcd.go | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/btcd.go b/btcd.go
index 457288b7..3d0dafda 100644
--- a/btcd.go
+++ b/btcd.go
@@ -176,6 +176,28 @@ func btcdMain(serverChan chan<- *server) error {
 		return err
 	}
 
+	// Enforce removal of txindex and addrindex if user requested pruning.
+	// This is to require explicit action from the user before removing
+	// indexes that won't be useful when block files are pruned.
+	//
+	// NOTE: The order is important here because dropping the tx index also
+	// drops the address index since it relies on it.  We explicitly make the
+	// user drop both indexes if --addrindex was enabled previously.
+	if cfg.Prune != 0 && indexers.AddrIndexInitialized(db) {
+		err = fmt.Errorf("--prune flag may not be given when the address index " +
+			"has been initialized. Please drop the address index with the " +
+			"--dropaddrindex flag before enabling pruning")
+		btcdLog.Errorf("%v", err)
+		return err
+	}
+	if cfg.Prune != 0 && indexers.TxIndexInitialized(db) {
+		err = fmt.Errorf("--prune flag may not be given when the transaction index " +
+			"has been initialized. Please drop the transaction index with the " +
+			"--droptxindex flag before enabling pruning")
+		btcdLog.Errorf("%v", err)
+		return err
+	}
+
 	// The config file is already created if it did not exist and the log
 	// file has already been opened by now so we only need to allow
 	// creating rpc cert and key files if they don't exist.

From 8f8040e596aa267c5cbad8a815e872f569cf413b Mon Sep 17 00:00:00 2001
From: Calvin Kim <calvin@kcalvinalvin.info>
Date: Tue, 22 Aug 2023 16:09:04 +0900
Subject: [PATCH 13/14] main: return error if user requests addr or tx index
 while pruned

This change is part of the effort to add pruning support to btcd.

It's not possible to generate the addr or tx indexes from scratch if the
block storage had been pruned previously as it's missing the block data.
When the user asks to create these indexes, tell them it's not possible
and the only way it's possible is if they delete and start anew.
---
 btcd.go | 14 ++++++++++++++
 1 file changed, 14 insertions(+)

diff --git a/btcd.go b/btcd.go
index 3d0dafda..0ac2f1bf 100644
--- a/btcd.go
+++ b/btcd.go
@@ -175,6 +175,20 @@ func btcdMain(serverChan chan<- *server) error {
 		btcdLog.Errorf("%v", err)
 		return err
 	}
+	if beenPruned && cfg.TxIndex {
+		err = fmt.Errorf("--txindex cannot be enabled as the node has been "+
+			"previously pruned. You must delete the files in the datadir: \"%s\" "+
+			"and sync from the beginning to enable the desired index", cfg.DataDir)
+		btcdLog.Errorf("%v", err)
+		return err
+	}
+	if beenPruned && cfg.AddrIndex {
+		err = fmt.Errorf("--addrindex cannot be enabled as the node has been "+
+			"previously pruned. You must delete the files in the datadir: \"%s\" "+
+			"and sync from the beginning to enable the desired index", cfg.DataDir)
+		btcdLog.Errorf("%v", err)
+		return err
+	}
 
 	// Enforce removal of txindex and addrindex if user requested pruning.
 	// This is to require explicit action from the user before removing

From 65c729960ae13c9b26aec8c5918b791a66886a04 Mon Sep 17 00:00:00 2001
From: Calvin Kim <calvin@kcalvinalvin.info>
Date: Tue, 22 Aug 2023 16:12:05 +0900
Subject: [PATCH 14/14] main: cfindex related sanity checks when enabled with
 pruning

This change is part of the effort to add pruning support to btcd.

cfIndex is a useful index even if the node has been pruned so it's
allowed to be enabled together with pruning.  However, if the user had
disabled cfindex and enabled pruning, it's not possible to generate
them.  In this case, we tell the user that it's impossible unless the
user deletes and start anew.

Additionally, if the user had enabled cfindex and also enabled pruning
from the start, don't let the user turn the cfindex off without dropping
it explicitly.  This is to make sure that the user isn't left at an
inconsistent state where the cfindex isn't able to catch up to the tip
because the blocks have already been pruned.
---
 btcd.go | 27 +++++++++++++++++++++++++++
 1 file changed, 27 insertions(+)

diff --git a/btcd.go b/btcd.go
index 0ac2f1bf..c7f292cb 100644
--- a/btcd.go
+++ b/btcd.go
@@ -189,6 +189,33 @@ func btcdMain(serverChan chan<- *server) error {
 		btcdLog.Errorf("%v", err)
 		return err
 	}
+	// If we've previously been pruned and the cfindex isn't present, it means that the
+	// user wants to enable the cfindex after the node has already synced up and been
+	// pruned.
+	if beenPruned && !indexers.CfIndexInitialized(db) && !cfg.NoCFilters {
+		err = fmt.Errorf("compact filters cannot be enabled as the node has been "+
+			"previously pruned. You must delete the files in the datadir: \"%s\" "+
+			"and sync from the beginning to enable the desired index. You may "+
+			"use the --nocfilters flag to start the node up without the compact "+
+			"filters", cfg.DataDir)
+		btcdLog.Errorf("%v", err)
+		return err
+	}
+	// If the user wants to disable the cfindex and is pruned or has enabled pruning, force
+	// the user to either drop the cfindex manually or restart the node without the --nocfilters
+	// flag.
+	if (beenPruned || cfg.Prune != 0) && indexers.CfIndexInitialized(db) && cfg.NoCFilters {
+		err = fmt.Errorf("--nocfilters flag was given but the compact filters have " +
+			"previously been enabled on this node and the index data currently " +
+			"exists in the database. The node has also been previously pruned and " +
+			"the database would be left in an inconsistent state if the compact " +
+			"filters don't get indexed now. To disable compact filters, please drop the " +
+			"index completely with the --dropcfindex flag and restart the node. " +
+			"To keep the compact filters, restart the node without the --nocfilters " +
+			"flag")
+		btcdLog.Errorf("%v", err)
+		return err
+	}
 
 	// Enforce removal of txindex and addrindex if user requested pruning.
 	// This is to require explicit action from the user before removing