diff --git a/database/ffldb/db.go b/database/ffldb/db.go index e8b1ce9f..f8ac1e07 100644 --- a/database/ffldb/db.go +++ b/database/ffldb/db.go @@ -1669,6 +1669,101 @@ func (tx *transaction) writePendingAndCommit() error { return tx.db.cache.commitTx(tx) } +// PruneBlocks deletes the block files until it reaches the target size +// (specified in bytes). Throws an error if the target size is below +// the maximum size of a single block file. +// +// This function is part of the database.Tx interface implementation. +func (tx *transaction) PruneBlocks(targetSize uint64) ([]chainhash.Hash, error) { + // Ensure transaction state is valid. + if err := tx.checkClosed(); err != nil { + return nil, err + } + + // Ensure the transaction is writable. + if !tx.writable { + str := "prune blocks requires a writable database transaction" + return nil, makeDbErr(database.ErrTxNotWritable, str, nil) + } + + // Make a local alias for the maxBlockFileSize. + maxSize := uint64(tx.db.store.maxBlockFileSize) + if targetSize < maxSize { + return nil, fmt.Errorf("got target size of %d but it must be greater "+ + "than %d, the max size of a single block file", + targetSize, maxSize) + } + + first, last, lastFileSize, err := scanBlockFiles(tx.db.store.basePath) + if err != nil { + return nil, err + } + + // If we have no files on disk or just a single file on disk, return early. + if first == last { + return nil, nil + } + + // Last file number minus the first file number gives us the count of files + // on disk minus 1. We don't want to count the last file since we can't assume + // that it is of max size. + maxSizeFileCount := last - first + + // If the total size of block files are under the target, return early and + // don't prune. + totalSize := uint64(lastFileSize) + (maxSize * uint64(maxSizeFileCount)) + if totalSize <= targetSize { + return nil, nil + } + + log.Tracef("Using %d more bytes than the target of %d MiB. Pruning files...", + totalSize-targetSize, + targetSize/(1024*1024)) + + deletedFiles := make(map[uint32]struct{}) + + // We use < not <= so that the last file is never deleted. There are other checks in place + // but setting it to < here doesn't hurt. + for i := uint32(first); i < uint32(last); i++ { + err = tx.db.store.deleteFileFunc(i) + if err != nil { + return nil, fmt.Errorf("PruneBlocks: Failed to delete block file "+ + "number %d: %v", i, err) + } + + // Add the file index to the deleted files map so that we can later + // delete the block location index. + deletedFiles[i] = struct{}{} + + // If we're already at or below the target usage, break and don't + // try to delete more files. + totalSize -= maxSize + if totalSize <= targetSize { + break + } + } + + // Delete the indexed block locations for the files that we've just deleted. + var deletedBlockHashes []chainhash.Hash + cursor := tx.blockIdxBucket.Cursor() + for ok := cursor.First(); ok; ok = cursor.Next() { + loc := deserializeBlockLoc(cursor.Value()) + + _, found := deletedFiles[loc.blockFileNum] + if found { + deletedBlockHashes = append(deletedBlockHashes, *(*chainhash.Hash)(cursor.Key())) + err := cursor.Delete() + if err != nil { + return nil, err + } + } + } + + log.Tracef("Finished pruning. Database now at %d bytes", totalSize) + + return deletedBlockHashes, nil +} + // Commit commits all changes that have been made to the root metadata bucket // and all of its sub-buckets to the database cache which is periodically synced // to persistent storage. In addition, it commits all new blocks directly to diff --git a/database/ffldb/driver_test.go b/database/ffldb/driver_test.go index 29d31eaf..5b9a7533 100644 --- a/database/ffldb/driver_test.go +++ b/database/ffldb/driver_test.go @@ -5,6 +5,7 @@ package ffldb_test import ( + "bytes" "fmt" "os" "path/filepath" @@ -13,6 +14,7 @@ import ( "github.com/btcsuite/btcd/btcutil" "github.com/btcsuite/btcd/chaincfg" + "github.com/btcsuite/btcd/chaincfg/chainhash" "github.com/btcsuite/btcd/database" "github.com/btcsuite/btcd/database/ffldb" ) @@ -253,6 +255,135 @@ func TestPersistence(t *testing.T) { } } +// TestPrune tests that the older .fdb files are deleted with a call to prune. +func TestPrune(t *testing.T) { + t.Parallel() + + // Create a new database to run tests against. + dbPath := t.TempDir() + db, err := database.Create(dbType, dbPath, blockDataNet) + if err != nil { + t.Errorf("Failed to create test database (%s) %v", dbType, err) + return + } + defer db.Close() + + blockFileSize := uint64(2048) + + testfn := func(t *testing.T, db database.DB) { + // Load the test blocks and save in the test context for use throughout + // the tests. + blocks, err := loadBlocks(t, blockDataFile, blockDataNet) + if err != nil { + t.Errorf("loadBlocks: Unexpected error: %v", err) + return + } + err = db.Update(func(tx database.Tx) error { + for i, block := range blocks { + err := tx.StoreBlock(block) + if err != nil { + return fmt.Errorf("StoreBlock #%d: unexpected error: "+ + "%v", i, err) + } + } + return nil + }) + if err != nil { + t.Fatal(err) + } + + blockHashMap := make(map[chainhash.Hash][]byte, len(blocks)) + for _, block := range blocks { + bytes, err := block.Bytes() + if err != nil { + t.Fatal(err) + } + blockHashMap[*block.Hash()] = bytes + } + + err = db.Update(func(tx database.Tx) error { + _, err := tx.PruneBlocks(1024) + if err == nil { + return fmt.Errorf("Expected an error when attempting to prune" + + "below the maxFileSize") + } + + _, err = tx.PruneBlocks(0) + if err == nil { + return fmt.Errorf("Expected an error when attempting to prune" + + "below the maxFileSize") + } + + return nil + }) + if err != nil { + t.Fatal(err) + } + + var deletedBlocks []chainhash.Hash + + // This should leave 3 files on disk. + err = db.Update(func(tx database.Tx) error { + deletedBlocks, err = tx.PruneBlocks(blockFileSize * 3) + return err + }) + if err != nil { + t.Fatal(err) + } + + // The only error we can get is a bad pattern error. Since we're hardcoding + // the pattern, we should not have an error at runtime. + files, _ := filepath.Glob(filepath.Join(dbPath, "*.fdb")) + if len(files) != 3 { + t.Fatalf("Expected to find %d files but got %d", + 3, len(files)) + } + + // Check that all the blocks that say were deleted are deleted from the + // block index bucket as well. + err = db.View(func(tx database.Tx) error { + for _, deletedBlock := range deletedBlocks { + _, err := tx.FetchBlock(&deletedBlock) + if dbErr, ok := err.(database.Error); !ok || + dbErr.ErrorCode != database.ErrBlockNotFound { + + return fmt.Errorf("Expected ErrBlockNotFound "+ + "but got %v", dbErr) + } + } + + return nil + }) + if err != nil { + t.Fatal(err) + } + + // Check that the not deleted blocks are present. + for _, deletedBlock := range deletedBlocks { + delete(blockHashMap, deletedBlock) + } + err = db.View(func(tx database.Tx) error { + for hash, wantBytes := range blockHashMap { + gotBytes, err := tx.FetchBlock(&hash) + if err != nil { + return err + } + if !bytes.Equal(gotBytes, wantBytes) { + return fmt.Errorf("got bytes %x, want bytes %x", + gotBytes, wantBytes) + } + } + return nil + }) + if err != nil { + t.Fatal(err) + } + } + ffldb.TstRunWithMaxBlockFileSize(db, uint32(blockFileSize), func() { + testfn(t, db) + }) +} + // TestInterface performs all interfaces tests for this database driver. func TestInterface(t *testing.T) { t.Parallel() diff --git a/database/ffldb/export_test.go b/database/ffldb/export_test.go index 2d8e4d2a..cbb6dc94 100644 --- a/database/ffldb/export_test.go +++ b/database/ffldb/export_test.go @@ -11,7 +11,9 @@ The functions are only exported while the tests are being run. package ffldb -import "github.com/btcsuite/btcd/database" +import ( + "github.com/btcsuite/btcd/database" +) // TstRunWithMaxBlockFileSize runs the passed function with the maximum allowed // file size for the database set to the provided value. The value will be set diff --git a/database/interface.go b/database/interface.go index d4f1d89d..789c823a 100644 --- a/database/interface.go +++ b/database/interface.go @@ -389,6 +389,21 @@ type Tx interface { // implementations. FetchBlockRegions(regions []BlockRegion) ([][]byte, error) + // PruneBlocks deletes the block files until it reaches the target size + // (specificed in bytes). + // + // The interface contract guarantees at least the following errors will + // be returned (other implementation-specific errors are possible): + // - ErrTxNotWritable if attempted against a read-only transaction + // - ErrTxClosed if the transaction has already been closed + // + // NOTE: The data returned by this function is only valid during a + // database transaction. Attempting to access it after a transaction + // has ended results in undefined behavior. This constraint prevents + // additional data copies and allows support for memory-mapped database + // implementations. + PruneBlocks(targetSize uint64) ([]chainhash.Hash, error) + // ****************************************************************** // Methods related to both atomic metadata storage and block storage. // ******************************************************************