manual geth v1.11.1 merge

core/rawdb/accessors_chain.go

@@ -259,8 +259,7 @@ func WriteLastPivotNumber(db ethdb.KeyValueWriter, pivot uint64) {
 }
 
 // ReadTxIndexTail retrieves the number of oldest indexed block
-// whose transaction indices has been indexed. If the corresponding entry
-// is non-existent in database it means the indexing has been finished.
+// whose transaction indices has been indexed.
 func ReadTxIndexTail(db ethdb.KeyValueReader) *uint64 {
 	data, _ := db.Get(txIndexTailKey)
 	if len(data) != 8 {
@@ -670,10 +669,11 @@ func DeleteReceipts(db ethdb.KeyValueWriter, hash common.Hash, number uint64) {
 
 // storedReceiptRLP is the storage encoding of a receipt.
 // Re-definition in core/types/receipt.go.
+// TODO: Re-use the existing definition.
 type storedReceiptRLP struct {
 	PostStateOrStatus []byte
 	CumulativeGasUsed uint64
-	Logs              []*types.LogForStorage
+	Logs              []*types.Log
 }
 
 // ReceiptLogs is a barebone version of ReceiptForStorage which only keeps
@@ -689,10 +689,7 @@ func (r *receiptLogs) DecodeRLP(s *rlp.Stream) error {
 	if err := s.Decode(&stored); err != nil {
 		return err
 	}
-	r.Logs = make([]*types.Log, len(stored.Logs))
-	for i, log := range stored.Logs {
-		r.Logs[i] = (*types.Log)(log)
-	}
+	r.Logs = stored.Logs
 	return nil
 }
 
@@ -717,9 +714,9 @@ func deriveLogFields(receipts []*receiptLogs, hash common.Hash, number uint64, t
 	return nil
 }
 
-// ReadLogs retrieves the logs for all transactions in a block. The log fields
-// are populated with metadata. In case the receipts or the block body
-// are not found, a nil is returned.
+// ReadLogs retrieves the logs for all transactions in a block. In case
+// receipts is not found, a nil is returned.
+// Note: ReadLogs does not derive unstored log fields.
 func ReadLogs(db ethdb.Reader, hash common.Hash, number uint64, config *params.ChainConfig) [][]*types.Log {
 	// Retrieve the flattened receipt slice
 	data := ReadReceiptsRLP(db, hash, number)
@@ -728,39 +725,10 @@ func ReadLogs(db ethdb.Reader, hash common.Hash, number uint64, config *params.C
 	}
 	receipts := []*receiptLogs{}
 	if err := rlp.DecodeBytes(data, &receipts); err != nil {
-		// Receipts might be in the legacy format, try decoding that.
-		// TODO: to be removed after users migrated
-		if logs := readLegacyLogs(db, hash, number, config); logs != nil {
-			return logs
-		}
 		log.Error("Invalid receipt array RLP", "hash", hash, "err", err)
 		return nil
 	}
 
-	body := ReadBody(db, hash, number)
-	if body == nil {
-		log.Error("Missing body but have receipt", "hash", hash, "number", number)
-		return nil
-	}
-	if err := deriveLogFields(receipts, hash, number, body.Transactions); err != nil {
-		log.Error("Failed to derive block receipts fields", "hash", hash, "number", number, "err", err)
-		return nil
-	}
-	logs := make([][]*types.Log, len(receipts))
-	for i, receipt := range receipts {
-		logs[i] = receipt.Logs
-	}
-	return logs
-}
-
-// readLegacyLogs is a temporary workaround for when trying to read logs
-// from a block which has its receipt stored in the legacy format. It'll
-// be removed after users have migrated their freezer databases.
-func readLegacyLogs(db ethdb.Reader, hash common.Hash, number uint64, config *params.ChainConfig) [][]*types.Log {
-	receipts := ReadReceipts(db, hash, number, config)
-	if receipts == nil {
-		return nil
-	}
 	logs := make([][]*types.Log, len(receipts))
 	for i, receipt := range receipts {
 		logs[i] = receipt.Logs
@@ -783,7 +751,7 @@ func ReadBlock(db ethdb.Reader, hash common.Hash, number uint64) *types.Block {
 	if body == nil {
 		return nil
 	}
-	return types.NewBlockWithHeader(header).WithBody(body.Transactions, body.Uncles)
+	return types.NewBlockWithHeader(header).WithBody(body.Transactions, body.Uncles).WithWithdrawals(body.Withdrawals)
 }
 
 // WriteBlock serializes a block into the database, header and body separately.
@@ -883,7 +851,7 @@ func ReadBadBlock(db ethdb.Reader, hash common.Hash) *types.Block {
 	}
 	for _, bad := range badBlocks {
 		if bad.Header.Hash() == hash {
-			return types.NewBlockWithHeader(bad.Header).WithBody(bad.Body.Transactions, bad.Body.Uncles)
+			return types.NewBlockWithHeader(bad.Header).WithBody(bad.Body.Transactions, bad.Body.Uncles).WithWithdrawals(bad.Body.Withdrawals)
 		}
 	}
 	return nil
@@ -902,7 +870,7 @@ func ReadAllBadBlocks(db ethdb.Reader) []*types.Block {
 	}
 	var blocks []*types.Block
 	for _, bad := range badBlocks {
-		blocks = append(blocks, types.NewBlockWithHeader(bad.Header).WithBody(bad.Body.Transactions, bad.Body.Uncles))
+		blocks = append(blocks, types.NewBlockWithHeader(bad.Header).WithBody(bad.Body.Transactions, bad.Body.Uncles).WithWithdrawals(bad.Body.Withdrawals))
 	}
 	return blocks
 }

core/rawdb/accessors_chain_test.go

@@ -750,10 +750,6 @@ func TestReadLogs(t *testing.T) {
 		t.Fatalf("unexpected number of logs[1] returned, have %d want %d", have, want)
 	}
 
-	// Fill in log fields so we can compare their rlp encoding
-	if err := types.Receipts(receipts).DeriveFields(params.TestChainConfig, hash, 0, body.Transactions); err != nil {
-		t.Fatal(err)
-	}
 	for i, pr := range receipts {
 		for j, pl := range pr.Logs {
 			rlpHave, err := rlp.EncodeToBytes(newFullLogRLP(logs[i][j]))

core/rawdb/accessors_state.go

@@ -28,6 +28,17 @@ func ReadPreimage(db ethdb.KeyValueReader, hash common.Hash) []byte {
 	return data
 }
 
+// WritePreimages writes the provided set of preimages to the database.
+func WritePreimages(db ethdb.KeyValueWriter, preimages map[common.Hash][]byte) {
+	for hash, preimage := range preimages {
+		if err := db.Put(preimageKey(hash), preimage); err != nil {
+			log.Crit("Failed to store trie preimage", "err", err)
+		}
+	}
+	preimageCounter.Inc(int64(len(preimages)))
+	preimageHitCounter.Inc(int64(len(preimages)))
+}
+
 // ReadCode retrieves the contract code of the provided code hash.
 func ReadCode(db ethdb.KeyValueReader, hash common.Hash) []byte {
 	// Try with the prefixed code scheme first, if not then try with legacy
@@ -48,12 +59,6 @@ func ReadCodeWithPrefix(db ethdb.KeyValueReader, hash common.Hash) []byte {
 	return data
 }
 
-// ReadTrieNode retrieves the trie node of the provided hash.
-func ReadTrieNode(db ethdb.KeyValueReader, hash common.Hash) []byte {
-	data, _ := db.Get(hash.Bytes())
-	return data
-}
-
 // HasCode checks if the contract code corresponding to the
 // provided code hash is present in the db.
 func HasCode(db ethdb.KeyValueReader, hash common.Hash) bool {
@@ -74,23 +79,6 @@ func HasCodeWithPrefix(db ethdb.KeyValueReader, hash common.Hash) bool {
 	return ok
 }
 
-// HasTrieNode checks if the trie node with the provided hash is present in db.
-func HasTrieNode(db ethdb.KeyValueReader, hash common.Hash) bool {
-	ok, _ := db.Has(hash.Bytes())
-	return ok
-}
-
-// WritePreimages writes the provided set of preimages to the database.
-func WritePreimages(db ethdb.KeyValueWriter, preimages map[common.Hash][]byte) {
-	for hash, preimage := range preimages {
-		if err := db.Put(preimageKey(hash), preimage); err != nil {
-			log.Crit("Failed to store trie preimage", "err", err)
-		}
-	}
-	preimageCounter.Inc(int64(len(preimages)))
-	preimageHitCounter.Inc(int64(len(preimages)))
-}
-
 // WriteCode writes the provided contract code database.
 func WriteCode(db ethdb.KeyValueWriter, hash common.Hash, code []byte) {
 	if err := db.Put(codeKey(hash), code); err != nil {
@@ -98,23 +86,9 @@ func WriteCode(db ethdb.KeyValueWriter, hash common.Hash, code []byte) {
 	}
 }
 
-// WriteTrieNode writes the provided trie node database.
-func WriteTrieNode(db ethdb.KeyValueWriter, hash common.Hash, node []byte) {
-	if err := db.Put(hash.Bytes(), node); err != nil {
-		log.Crit("Failed to store trie node", "err", err)
-	}
-}
-
 // DeleteCode deletes the specified contract code from the database.
 func DeleteCode(db ethdb.KeyValueWriter, hash common.Hash) {
 	if err := db.Delete(codeKey(hash)); err != nil {
 		log.Crit("Failed to delete contract code", "err", err)
 	}
 }
-
-// DeleteTrieNode deletes the specified trie node from the database.
-func DeleteTrieNode(db ethdb.KeyValueWriter, hash common.Hash) {
-	if err := db.Delete(hash.Bytes()); err != nil {
-		log.Crit("Failed to delete trie node", "err", err)
-	}
-}

core/rawdb/accessors_trie.go

@@ -0,0 +1,263 @@
+// Copyright 2022 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>
+
+package rawdb
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/log"
+	"golang.org/x/crypto/sha3"
+)
+
+// HashScheme is the legacy hash-based state scheme with which trie nodes are
+// stored in the disk with node hash as the database key. The advantage of this
+// scheme is that different versions of trie nodes can be stored in disk, which
+// is very beneficial for constructing archive nodes. The drawback is it will
+// store different trie nodes on the same path to different locations on the disk
+// with no data locality, and it's unfriendly for designing state pruning.
+//
+// Now this scheme is still kept for backward compatibility, and it will be used
+// for archive node and some other tries(e.g. light trie).
+const HashScheme = "hashScheme"
+
+// PathScheme is the new path-based state scheme with which trie nodes are stored
+// in the disk with node path as the database key. This scheme will only store one
+// version of state data in the disk, which means that the state pruning operation
+// is native. At the same time, this scheme will put adjacent trie nodes in the same
+// area of the disk with good data locality property. But this scheme needs to rely
+// on extra state diffs to survive deep reorg.
+const PathScheme = "pathScheme"
+
+// nodeHasher used to derive the hash of trie node.
+type nodeHasher struct{ sha crypto.KeccakState }
+
+var hasherPool = sync.Pool{
+	New: func() interface{} { return &nodeHasher{sha: sha3.NewLegacyKeccak256().(crypto.KeccakState)} },
+}
+
+func newNodeHasher() *nodeHasher       { return hasherPool.Get().(*nodeHasher) }
+func returnHasherToPool(h *nodeHasher) { hasherPool.Put(h) }
+
+func (h *nodeHasher) hashData(data []byte) (n common.Hash) {
+	h.sha.Reset()
+	h.sha.Write(data)
+	h.sha.Read(n[:])
+	return n
+}
+
+// ReadAccountTrieNode retrieves the account trie node and the associated node
+// hash with the specified node path.
+func ReadAccountTrieNode(db ethdb.KeyValueReader, path []byte) ([]byte, common.Hash) {
+	data, err := db.Get(accountTrieNodeKey(path))
+	if err != nil {
+		return nil, common.Hash{}
+	}
+	hasher := newNodeHasher()
+	defer returnHasherToPool(hasher)
+	return data, hasher.hashData(data)
+}
+
+// HasAccountTrieNode checks the account trie node presence with the specified
+// node path and the associated node hash.
+func HasAccountTrieNode(db ethdb.KeyValueReader, path []byte, hash common.Hash) bool {
+	data, err := db.Get(accountTrieNodeKey(path))
+	if err != nil {
+		return false
+	}
+	hasher := newNodeHasher()
+	defer returnHasherToPool(hasher)
+	return hasher.hashData(data) == hash
+}
+
+// WriteAccountTrieNode writes the provided account trie node into database.
+func WriteAccountTrieNode(db ethdb.KeyValueWriter, path []byte, node []byte) {
+	if err := db.Put(accountTrieNodeKey(path), node); err != nil {
+		log.Crit("Failed to store account trie node", "err", err)
+	}
+}
+
+// DeleteAccountTrieNode deletes the specified account trie node from the database.
+func DeleteAccountTrieNode(db ethdb.KeyValueWriter, path []byte) {
+	if err := db.Delete(accountTrieNodeKey(path)); err != nil {
+		log.Crit("Failed to delete account trie node", "err", err)
+	}
+}
+
+// ReadStorageTrieNode retrieves the storage trie node and the associated node
+// hash with the specified node path.
+func ReadStorageTrieNode(db ethdb.KeyValueReader, accountHash common.Hash, path []byte) ([]byte, common.Hash) {
+	data, err := db.Get(storageTrieNodeKey(accountHash, path))
+	if err != nil {
+		return nil, common.Hash{}
+	}
+	hasher := newNodeHasher()
+	defer returnHasherToPool(hasher)
+	return data, hasher.hashData(data)
+}
+
+// HasStorageTrieNode checks the storage trie node presence with the provided
+// node path and the associated node hash.
+func HasStorageTrieNode(db ethdb.KeyValueReader, accountHash common.Hash, path []byte, hash common.Hash) bool {
+	data, err := db.Get(storageTrieNodeKey(accountHash, path))
+	if err != nil {
+		return false
+	}
+	hasher := newNodeHasher()
+	defer returnHasherToPool(hasher)
+	return hasher.hashData(data) == hash
+}
+
+// WriteStorageTrieNode writes the provided storage trie node into database.
+func WriteStorageTrieNode(db ethdb.KeyValueWriter, accountHash common.Hash, path []byte, node []byte) {
+	if err := db.Put(storageTrieNodeKey(accountHash, path), node); err != nil {
+		log.Crit("Failed to store storage trie node", "err", err)
+	}
+}
+
+// DeleteStorageTrieNode deletes the specified storage trie node from the database.
+func DeleteStorageTrieNode(db ethdb.KeyValueWriter, accountHash common.Hash, path []byte) {
+	if err := db.Delete(storageTrieNodeKey(accountHash, path)); err != nil {
+		log.Crit("Failed to delete storage trie node", "err", err)
+	}
+}
+
+// ReadLegacyTrieNode retrieves the legacy trie node with the given
+// associated node hash.
+func ReadLegacyTrieNode(db ethdb.KeyValueReader, hash common.Hash) []byte {
+	data, err := db.Get(hash.Bytes())
+	if err != nil {
+		return nil
+	}
+	return data
+}
+
+// HasLegacyTrieNode checks if the trie node with the provided hash is present in db.
+func HasLegacyTrieNode(db ethdb.KeyValueReader, hash common.Hash) bool {
+	ok, _ := db.Has(hash.Bytes())
+	return ok
+}
+
+// WriteLegacyTrieNode writes the provided legacy trie node to database.
+func WriteLegacyTrieNode(db ethdb.KeyValueWriter, hash common.Hash, node []byte) {
+	if err := db.Put(hash.Bytes(), node); err != nil {
+		log.Crit("Failed to store legacy trie node", "err", err)
+	}
+}
+
+// DeleteLegacyTrieNode deletes the specified legacy trie node from database.
+func DeleteLegacyTrieNode(db ethdb.KeyValueWriter, hash common.Hash) {
+	if err := db.Delete(hash.Bytes()); err != nil {
+		log.Crit("Failed to delete legacy trie node", "err", err)
+	}
+}
+
+// HasTrieNode checks the trie node presence with the provided node info and
+// the associated node hash.
+func HasTrieNode(db ethdb.KeyValueReader, owner common.Hash, path []byte, hash common.Hash, scheme string) bool {
+	switch scheme {
+	case HashScheme:
+		return HasLegacyTrieNode(db, hash)
+	case PathScheme:
+		if owner == (common.Hash{}) {
+			return HasAccountTrieNode(db, path, hash)
+		}
+		return HasStorageTrieNode(db, owner, path, hash)
+	default:
+		panic(fmt.Sprintf("Unknown scheme %v", scheme))
+	}
+}
+
+// ReadTrieNode retrieves the trie node from database with the provided node info
+// and associated node hash.
+// hashScheme-based lookup requires the following:
+//   - hash
+//
+// pathScheme-based lookup requires the following:
+//   - owner
+//   - path
+func ReadTrieNode(db ethdb.KeyValueReader, owner common.Hash, path []byte, hash common.Hash, scheme string) []byte {
+	switch scheme {
+	case HashScheme:
+		return ReadLegacyTrieNode(db, hash)
+	case PathScheme:
+		var (
+			blob  []byte
+			nHash common.Hash
+		)
+		if owner == (common.Hash{}) {
+			blob, nHash = ReadAccountTrieNode(db, path)
+		} else {
+			blob, nHash = ReadStorageTrieNode(db, owner, path)
+		}
+		if nHash != hash {
+			return nil
+		}
+		return blob
+	default:
+		panic(fmt.Sprintf("Unknown scheme %v", scheme))
+	}
+}
+
+// WriteTrieNode writes the trie node into database with the provided node info
+// and associated node hash.
+// hashScheme-based lookup requires the following:
+//   - hash
+//
+// pathScheme-based lookup requires the following:
+//   - owner
+//   - path
+func WriteTrieNode(db ethdb.KeyValueWriter, owner common.Hash, path []byte, hash common.Hash, node []byte, scheme string) {
+	switch scheme {
+	case HashScheme:
+		WriteLegacyTrieNode(db, hash, node)
+	case PathScheme:
+		if owner == (common.Hash{}) {
+			WriteAccountTrieNode(db, path, node)
+		} else {
+			WriteStorageTrieNode(db, owner, path, node)
+		}
+	default:
+		panic(fmt.Sprintf("Unknown scheme %v", scheme))
+	}
+}
+
+// DeleteTrieNode deletes the trie node from database with the provided node info
+// and associated node hash.
+// hashScheme-based lookup requires the following:
+//   - hash
+//
+// pathScheme-based lookup requires the following:
+//   - owner
+//   - path
+func DeleteTrieNode(db ethdb.KeyValueWriter, owner common.Hash, path []byte, hash common.Hash, scheme string) {
+	switch scheme {
+	case HashScheme:
+		DeleteLegacyTrieNode(db, hash)
+	case PathScheme:
+		if owner == (common.Hash{}) {
+			DeleteAccountTrieNode(db, path)
+		} else {
+			DeleteStorageTrieNode(db, owner, path)
+		}
+	default:
+		panic(fmt.Sprintf("Unknown scheme %v", scheme))
+	}
+}

core/rawdb/ancient_scheme.go

@@ -16,8 +16,6 @@
 
 package rawdb
 
-import "fmt"
-
 // The list of table names of chain freezer.
 const (
 	// chainFreezerHeaderTable indicates the name of the freezer header table.
@@ -53,34 +51,3 @@ var (
 
 // freezers the collections of all builtin freezers.
 var freezers = []string{chainFreezerName}
-
-// InspectFreezerTable dumps out the index of a specific freezer table. The passed
-// ancient indicates the path of root ancient directory where the chain freezer can
-// be opened. Start and end specify the range for dumping out indexes.
-// Note this function can only be used for debugging purposes.
-func InspectFreezerTable(ancient string, freezerName string, tableName string, start, end int64) error {
-	var (
-		path   string
-		tables map[string]bool
-	)
-	switch freezerName {
-	case chainFreezerName:
-		path, tables = resolveChainFreezerDir(ancient), chainFreezerNoSnappy
-	default:
-		return fmt.Errorf("unknown freezer, supported ones: %v", freezers)
-	}
-	noSnappy, exist := tables[tableName]
-	if !exist {
-		var names []string
-		for name := range tables {
-			names = append(names, name)
-		}
-		return fmt.Errorf("unknown table, supported ones: %v", names)
-	}
-	table, err := newFreezerTable(path, tableName, noSnappy, true)
-	if err != nil {
-		return err
-	}
-	table.dumpIndexStdout(start, end)
-	return nil
-}

core/rawdb/ancient_utils.go

@@ -0,0 +1,121 @@
+// Copyright 2022 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package rawdb
+
+import (
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/ethdb"
+)
+
+type tableSize struct {
+	name string
+	size common.StorageSize
+}
+
+// freezerInfo contains the basic information of the freezer.
+type freezerInfo struct {
+	name  string      // The identifier of freezer
+	head  uint64      // The number of last stored item in the freezer
+	tail  uint64      // The number of first stored item in the freezer
+	sizes []tableSize // The storage size per table
+}
+
+// count returns the number of stored items in the freezer.
+func (info *freezerInfo) count() uint64 {
+	return info.head - info.tail + 1
+}
+
+// size returns the storage size of the entire freezer.
+func (info *freezerInfo) size() common.StorageSize {
+	var total common.StorageSize
+	for _, table := range info.sizes {
+		total += table.size
+	}
+	return total
+}
+
+// inspectFreezers inspects all freezers registered in the system.
+func inspectFreezers(db ethdb.Database) ([]freezerInfo, error) {
+	var infos []freezerInfo
+	for _, freezer := range freezers {
+		switch freezer {
+		case chainFreezerName:
+			// Chain ancient store is a bit special. It's always opened along
+			// with the key-value store, inspect the chain store directly.
+			info := freezerInfo{name: freezer}
+			// Retrieve storage size of every contained table.
+			for table := range chainFreezerNoSnappy {
+				size, err := db.AncientSize(table)
+				if err != nil {
+					return nil, err
+				}
+				info.sizes = append(info.sizes, tableSize{name: table, size: common.StorageSize(size)})
+			}
+			// Retrieve the number of last stored item
+			ancients, err := db.Ancients()
+			if err != nil {
+				return nil, err
+			}
+			info.head = ancients - 1
+
+			// Retrieve the number of first stored item
+			tail, err := db.Tail()
+			if err != nil {
+				return nil, err
+			}
+			info.tail = tail
+			infos = append(infos, info)
+
+		default:
+			return nil, fmt.Errorf("unknown freezer, supported ones: %v", freezers)
+		}
+	}
+	return infos, nil
+}
+
+// InspectFreezerTable dumps out the index of a specific freezer table. The passed
+// ancient indicates the path of root ancient directory where the chain freezer can
+// be opened. Start and end specify the range for dumping out indexes.
+// Note this function can only be used for debugging purposes.
+func InspectFreezerTable(ancient string, freezerName string, tableName string, start, end int64) error {
+	var (
+		path   string
+		tables map[string]bool
+	)
+	switch freezerName {
+	case chainFreezerName:
+		path, tables = resolveChainFreezerDir(ancient), chainFreezerNoSnappy
+	default:
+		return fmt.Errorf("unknown freezer, supported ones: %v", freezers)
+	}
+	noSnappy, exist := tables[tableName]
+	if !exist {
+		var names []string
+		for name := range tables {
+			names = append(names, name)
+		}
+		return fmt.Errorf("unknown table, supported ones: %v", names)
+	}
+	table, err := newFreezerTable(path, tableName, noSnappy, true)
+	if err != nil {
+		return err
+	}
+	table.dumpIndexStdout(start, end)
+	return nil
+}

core/rawdb/chain_freezer.go

@@ -70,14 +70,13 @@ func newChainFreezer(datadir string, namespace string, readonly bool, maxTableSi
 
 // Close closes the chain freezer instance and terminates the background thread.
 func (f *chainFreezer) Close() error {
-	err := f.Freezer.Close()
 	select {
 	case <-f.quit:
 	default:
 		close(f.quit)
 	}
 	f.wg.Wait()
-	return err
+	return f.Freezer.Close()
 }
 
 // freeze is a background thread that periodically checks the blockchain for any
@@ -86,12 +85,14 @@ func (f *chainFreezer) Close() error {
 // This functionality is deliberately broken off from block importing to avoid
 // incurring additional data shuffling delays on block propagation.
 func (f *chainFreezer) freeze(db ethdb.KeyValueStore) {
-	nfdb := &nofreezedb{KeyValueStore: db}
-
 	var (
 		backoff   bool
 		triggered chan struct{} // Used in tests
+		nfdb      = &nofreezedb{KeyValueStore: db}
 	)
+	timer := time.NewTimer(freezerRecheckInterval)
+	defer timer.Stop()
+
 	for {
 		select {
 		case <-f.quit:
@@ -106,8 +107,9 @@ func (f *chainFreezer) freeze(db ethdb.KeyValueStore) {
 				triggered = nil
 			}
 			select {
-			case <-time.NewTimer(freezerRecheckInterval).C:
+			case <-timer.C:
 				backoff = false
+				timer.Reset(freezerRecheckInterval)
 			case triggered = <-f.trigger:
 				backoff = false
 			case <-f.quit:

core/rawdb/chain_iterator.go

@@ -191,7 +191,7 @@ func indexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan
 		// in to be [to-1]. Therefore, setting lastNum to means that the
 		// prqueue gap-evaluation will work correctly
 		lastNum = to
-		queue   = prque.New(nil)
+		queue   = prque.New[int64, *blockTxHashes](nil)
 		// for stats reporting
 		blocks, txs = 0, 0
 	)
@@ -210,7 +210,7 @@ func indexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan
 				break
 			}
 			// Next block available, pop it off and index it
-			delivery := queue.PopItem().(*blockTxHashes)
+			delivery := queue.PopItem()
 			lastNum = delivery.number
 			WriteTxLookupEntries(batch, delivery.number, delivery.hashes)
 			blocks++
@@ -243,7 +243,7 @@ func indexTransactions(db ethdb.Database, from uint64, to uint64, interrupt chan
 	case <-interrupt:
 		log.Debug("Transaction indexing interrupted", "blocks", blocks, "txs", txs, "tail", lastNum, "elapsed", common.PrettyDuration(time.Since(start)))
 	default:
-		log.Info("Indexed transactions", "blocks", blocks, "txs", txs, "tail", lastNum, "elapsed", common.PrettyDuration(time.Since(start)))
+		log.Debug("Indexed transactions", "blocks", blocks, "txs", txs, "tail", lastNum, "elapsed", common.PrettyDuration(time.Since(start)))
 	}
 }
 
@@ -282,7 +282,7 @@ func unindexTransactions(db ethdb.Database, from uint64, to uint64, interrupt ch
 		// we expect the first number to come in to be [from]. Therefore, setting
 		// nextNum to from means that the prqueue gap-evaluation will work correctly
 		nextNum = from
-		queue   = prque.New(nil)
+		queue   = prque.New[int64, *blockTxHashes](nil)
 		// for stats reporting
 		blocks, txs = 0, 0
 	)
@@ -299,7 +299,7 @@ func unindexTransactions(db ethdb.Database, from uint64, to uint64, interrupt ch
 			if hook != nil && !hook(nextNum) {
 				break
 			}
-			delivery := queue.PopItem().(*blockTxHashes)
+			delivery := queue.PopItem()
 			nextNum = delivery.number + 1
 			DeleteTxLookupEntries(batch, delivery.hashes)
 			txs += len(delivery.hashes)
@@ -335,7 +335,7 @@ func unindexTransactions(db ethdb.Database, from uint64, to uint64, interrupt ch
 	case <-interrupt:
 		log.Debug("Transaction unindexing interrupted", "blocks", blocks, "txs", txs, "tail", to, "elapsed", common.PrettyDuration(time.Since(start)))
 	default:
-		log.Info("Unindexed transactions", "blocks", blocks, "txs", txs, "tail", to, "elapsed", common.PrettyDuration(time.Since(start)))
+		log.Debug("Unindexed transactions", "blocks", blocks, "txs", txs, "tail", to, "elapsed", common.PrettyDuration(time.Since(start)))
 	}
 }
 

core/rawdb/database.go

@@ -22,6 +22,8 @@ import (
 	"fmt"
 	"os"
 	"path"
+	"path/filepath"
+	"strings"
 	"sync/atomic"
 	"time"
 
@@ -240,8 +242,8 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, ancient string, namespace st
 			if kvhash, _ := db.Get(headerHashKey(frozen)); len(kvhash) == 0 {
 				// Subsequent header after the freezer limit is missing from the database.
 				// Reject startup if the database has a more recent head.
-				if *ReadHeaderNumber(db, ReadHeadHeaderHash(db)) > frozen-1 {
-					return nil, fmt.Errorf("gap (#%d) in the chain between ancients and leveldb", frozen)
+				if ldbNum := *ReadHeaderNumber(db, ReadHeadHeaderHash(db)); ldbNum > frozen-1 {
+					return nil, fmt.Errorf("gap in the chain between ancients (#%d) and leveldb (#%d) ", frozen, ldbNum)
 				}
 				// Database contains only older data than the freezer, this happens if the
 				// state was wiped and reinited from an existing freezer.
@@ -301,19 +303,84 @@ func NewLevelDBDatabase(file string, cache int, handles int, namespace string, r
 	if err != nil {
 		return nil, err
 	}
+	log.Info("Using LevelDB as the backing database")
 	return NewDatabase(db), nil
 }
 
-// NewLevelDBDatabaseWithFreezer creates a persistent key-value database with a
-// freezer moving immutable chain segments into cold storage. The passed ancient
-// indicates the path of root ancient directory where the chain freezer can be
-// opened.
-func NewLevelDBDatabaseWithFreezer(file string, cache int, handles int, ancient string, namespace string, readonly bool) (ethdb.Database, error) {
-	kvdb, err := leveldb.New(file, cache, handles, namespace, readonly)
+const (
+	dbPebble  = "pebble"
+	dbLeveldb = "leveldb"
+)
+
+// hasPreexistingDb checks the given data directory whether a database is already
+// instantiated at that location, and if so, returns the type of database (or the
+// empty string).
+func hasPreexistingDb(path string) string {
+	if _, err := os.Stat(filepath.Join(path, "CURRENT")); err != nil {
+		return "" // No pre-existing db
+	}
+	if matches, err := filepath.Glob(filepath.Join(path, "OPTIONS*")); len(matches) > 0 || err != nil {
+		if err != nil {
+			panic(err) // only possible if the pattern is malformed
+		}
+		return dbPebble
+	}
+	return dbLeveldb
+}
+
+// OpenOptions contains the options to apply when opening a database.
+// OBS: If AncientsDirectory is empty, it indicates that no freezer is to be used.
+type OpenOptions struct {
+	Type              string // "leveldb" | "pebble"
+	Directory         string // the datadir
+	AncientsDirectory string // the ancients-dir
+	Namespace         string // the namespace for database relevant metrics
+	Cache             int    // the capacity(in megabytes) of the data caching
+	Handles           int    // number of files to be open simultaneously
+	ReadOnly          bool
+}
+
+// openKeyValueDatabase opens a disk-based key-value database, e.g. leveldb or pebble.
+//
+//	                      type == null          type != null
+//	                   +----------------------------------------
+//	db is non-existent |  leveldb default  |  specified type
+//	db is existent     |  from db          |  specified type (if compatible)
+func openKeyValueDatabase(o OpenOptions) (ethdb.Database, error) {
+	existingDb := hasPreexistingDb(o.Directory)
+	if len(existingDb) != 0 && len(o.Type) != 0 && o.Type != existingDb {
+		return nil, fmt.Errorf("db.engine choice was %v but found pre-existing %v database in specified data directory", o.Type, existingDb)
+	}
+	if o.Type == dbPebble || existingDb == dbPebble {
+		if PebbleEnabled {
+			log.Info("Using pebble as the backing database")
+			return NewPebbleDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly)
+		} else {
+			return nil, errors.New("db.engine 'pebble' not supported on this platform")
+		}
+	}
+	if len(o.Type) != 0 && o.Type != dbLeveldb {
+		return nil, fmt.Errorf("unknown db.engine %v", o.Type)
+	}
+	log.Info("Using leveldb as the backing database")
+	// Use leveldb, either as default (no explicit choice), or pre-existing, or chosen explicitly
+	return NewLevelDBDatabase(o.Directory, o.Cache, o.Handles, o.Namespace, o.ReadOnly)
+}
+
+// Open opens both a disk-based key-value database such as leveldb or pebble, but also
+// integrates it with a freezer database -- if the AncientDir option has been
+// set on the provided OpenOptions.
+// The passed o.AncientDir indicates the path of root ancient directory where
+// the chain freezer can be opened.
+func Open(o OpenOptions) (ethdb.Database, error) {
+	kvdb, err := openKeyValueDatabase(o)
 	if err != nil {
 		return nil, err
 	}
-	frdb, err := NewDatabaseWithFreezer(kvdb, ancient, namespace, readonly)
+	if len(o.AncientsDirectory) == 0 {
+		return kvdb, nil
+	}
+	frdb, err := NewDatabaseWithFreezer(kvdb, o.AncientsDirectory, o.Namespace, o.ReadOnly)
 	if err != nil {
 		kvdb.Close()
 		return nil, err
@@ -379,13 +446,6 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
 		beaconHeaders   stat
 		cliqueSnaps     stat
 
-		// Ancient store statistics
-		ancientHeadersSize  common.StorageSize
-		ancientBodiesSize   common.StorageSize
-		ancientReceiptsSize common.StorageSize
-		ancientTdsSize      common.StorageSize
-		ancientHashesSize   common.StorageSize
-
 		// Les statistic
 		chtTrieNodes   stat
 		bloomTrieNodes stat
@@ -439,15 +499,15 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
 			bloomBits.Add(size)
 		case bytes.HasPrefix(key, skeletonHeaderPrefix) && len(key) == (len(skeletonHeaderPrefix)+8):
 			beaconHeaders.Add(size)
-		case bytes.HasPrefix(key, []byte("clique-")) && len(key) == 7+common.HashLength:
+		case bytes.HasPrefix(key, CliqueSnapshotPrefix) && len(key) == 7+common.HashLength:
 			cliqueSnaps.Add(size)
-		case bytes.HasPrefix(key, []byte("cht-")) ||
-			bytes.HasPrefix(key, []byte("chtIndexV2-")) ||
-			bytes.HasPrefix(key, []byte("chtRootV2-")): // Canonical hash trie
+		case bytes.HasPrefix(key, ChtTablePrefix) ||
+			bytes.HasPrefix(key, ChtIndexTablePrefix) ||
+			bytes.HasPrefix(key, ChtPrefix): // Canonical hash trie
 			chtTrieNodes.Add(size)
-		case bytes.HasPrefix(key, []byte("blt-")) ||
-			bytes.HasPrefix(key, []byte("bltIndex-")) ||
-			bytes.HasPrefix(key, []byte("bltRoot-")): // Bloomtrie sub
+		case bytes.HasPrefix(key, BloomTrieTablePrefix) ||
+			bytes.HasPrefix(key, BloomTrieIndexPrefix) ||
+			bytes.HasPrefix(key, BloomTriePrefix): // Bloomtrie sub
 			bloomTrieNodes.Add(size)
 		default:
 			var accounted bool
@@ -473,20 +533,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
 			logged = time.Now()
 		}
 	}
-	// Inspect append-only file store then.
-	ancientSizes := []*common.StorageSize{&ancientHeadersSize, &ancientBodiesSize, &ancientReceiptsSize, &ancientHashesSize, &ancientTdsSize}
-	for i, category := range []string{chainFreezerHeaderTable, chainFreezerBodiesTable, chainFreezerReceiptTable, chainFreezerHashTable, chainFreezerDifficultyTable} {
-		if size, err := db.AncientSize(category); err == nil {
-			*ancientSizes[i] += common.StorageSize(size)
-			total += common.StorageSize(size)
-		}
-	}
-	// Get number of ancient rows inside the freezer
-	ancients := counter(0)
-	if count, err := db.Ancients(); err == nil {
-		ancients = counter(count)
-	}
-	// Display the database statistic.
+	// Display the database statistic of key-value store.
 	stats := [][]string{
 		{"Key-Value store", "Headers", headers.Size(), headers.Count()},
 		{"Key-Value store", "Bodies", bodies.Size(), bodies.Count()},
@@ -504,14 +551,25 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
 		{"Key-Value store", "Beacon sync headers", beaconHeaders.Size(), beaconHeaders.Count()},
 		{"Key-Value store", "Clique snapshots", cliqueSnaps.Size(), cliqueSnaps.Count()},
 		{"Key-Value store", "Singleton metadata", metadata.Size(), metadata.Count()},
-		{"Ancient store", "Headers", ancientHeadersSize.String(), ancients.String()},
-		{"Ancient store", "Bodies", ancientBodiesSize.String(), ancients.String()},
-		{"Ancient store", "Receipt lists", ancientReceiptsSize.String(), ancients.String()},
-		{"Ancient store", "Difficulties", ancientTdsSize.String(), ancients.String()},
-		{"Ancient store", "Block number->hash", ancientHashesSize.String(), ancients.String()},
 		{"Light client", "CHT trie nodes", chtTrieNodes.Size(), chtTrieNodes.Count()},
 		{"Light client", "Bloom trie nodes", bloomTrieNodes.Size(), bloomTrieNodes.Count()},
 	}
+	// Inspect all registered append-only file store then.
+	ancients, err := inspectFreezers(db)
+	if err != nil {
+		return err
+	}
+	for _, ancient := range ancients {
+		for _, table := range ancient.sizes {
+			stats = append(stats, []string{
+				fmt.Sprintf("Ancient store (%s)", strings.Title(ancient.name)),
+				strings.Title(table.name),
+				table.size.String(),
+				fmt.Sprintf("%d", ancient.count()),
+			})
+		}
+		total += ancient.size()
+	}
 	table := tablewriter.NewWriter(os.Stdout)
 	table.SetHeader([]string{"Database", "Category", "Size", "Items"})
 	table.SetFooter([]string{"", "Total", total.String(), " "})
@@ -521,6 +579,5 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
 	if unaccounted.size > 0 {
 		log.Error("Database contains unaccounted data", "size", unaccounted.size, "count", unaccounted.count)
 	}
-
 	return nil
 }

core/rawdb/databases_64bit.go

@@ -0,0 +1,37 @@
+// Copyright 2023 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>
+
+//go:build arm64 || amd64
+
+package rawdb
+
+import (
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/ethdb/pebble"
+)
+
+// Pebble is unsuported on 32bit architecture
+const PebbleEnabled = true
+
+// NewPebbleDBDatabase creates a persistent key-value database without a freezer
+// moving immutable chain segments into cold storage.
+func NewPebbleDBDatabase(file string, cache int, handles int, namespace string, readonly bool) (ethdb.Database, error) {
+	db, err := pebble.New(file, cache, handles, namespace, readonly)
+	if err != nil {
+		return nil, err
+	}
+	return NewDatabase(db), nil
+}

core/rawdb/databases_non64bit.go

@@ -0,0 +1,34 @@
+// Copyright 2023 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+//go:build !(arm64 || amd64)
+
+package rawdb
+
+import (
+	"errors"
+
+	"github.com/ethereum/go-ethereum/ethdb"
+)
+
+// Pebble is unsuported on 32bit architecture
+const PebbleEnabled = false
+
+// NewPebbleDBDatabase creates a persistent key-value database without a freezer
+// moving immutable chain segments into cold storage.
+func NewPebbleDBDatabase(file string, cache int, handles int, namespace string, readonly bool) (ethdb.Database, error) {
+	return nil, errors.New("pebble is not supported on this platform")
+}

core/rawdb/freezer.go

@@ -57,10 +57,10 @@ const freezerTableSize = 2 * 1000 * 1000 * 1000
 // Freezer is a memory mapped append-only database to store immutable ordered
 // data into flat files:
 //
-// - The append-only nature ensures that disk writes are minimized.
-// - The memory mapping ensures we can max out system memory for caching without
-//   reserving it for go-ethereum. This would also reduce the memory requirements
-//   of Geth, and thus also GC overhead.
+//   - The append-only nature ensures that disk writes are minimized.
+//   - The memory mapping ensures we can max out system memory for caching without
+//     reserving it for go-ethereum. This would also reduce the memory requirements
+//     of Geth, and thus also GC overhead.
 type Freezer struct {
 	// WARNING: The `frozen` and `tail` fields are accessed atomically. On 32 bit platforms, only
 	// 64-bit aligned fields can be atomic. The struct is guaranteed to be so aligned,
@@ -188,9 +188,9 @@ func (f *Freezer) Ancient(kind string, number uint64) ([]byte, error) {
 
 // AncientRange retrieves multiple items in sequence, starting from the index 'start'.
 // It will return
-//  - at most 'max' items,
-//  - at least 1 item (even if exceeding the maxByteSize), but will otherwise
-//   return as many items as fit into maxByteSize.
+//   - at most 'max' items,
+//   - at least 1 item (even if exceeding the maxByteSize), but will otherwise
+//     return as many items as fit into maxByteSize.
 func (f *Freezer) AncientRange(kind string, start, count, maxBytes uint64) ([][]byte, error) {
 	if table := f.tables[kind]; table != nil {
 		return table.RetrieveItems(start, count, maxBytes)
@@ -321,30 +321,35 @@ func (f *Freezer) Sync() error {
 	return nil
 }
 
-// validate checks that every table has the same length.
+// validate checks that every table has the same boundary.
 // Used instead of `repair` in readonly mode.
 func (f *Freezer) validate() error {
 	if len(f.tables) == 0 {
 		return nil
 	}
 	var (
-		length uint64
-		name   string
+		head uint64
+		tail uint64
+		name string
 	)
-	// Hack to get length of any table
+	// Hack to get boundary of any table
 	for kind, table := range f.tables {
-		length = atomic.LoadUint64(&table.items)
+		head = atomic.LoadUint64(&table.items)
+		tail = atomic.LoadUint64(&table.itemHidden)
 		name = kind
 		break
 	}
-	// Now check every table against that length
+	// Now check every table against those boundaries.
 	for kind, table := range f.tables {
-		items := atomic.LoadUint64(&table.items)
-		if length != items {
-			return fmt.Errorf("freezer tables %s and %s have differing lengths: %d != %d", kind, name, items, length)
+		if head != atomic.LoadUint64(&table.items) {
+			return fmt.Errorf("freezer tables %s and %s have differing head: %d != %d", kind, name, atomic.LoadUint64(&table.items), head)
+		}
+		if tail != atomic.LoadUint64(&table.itemHidden) {
+			return fmt.Errorf("freezer tables %s and %s have differing tail: %d != %d", kind, name, atomic.LoadUint64(&table.itemHidden), tail)
 		}
 	}
-	atomic.StoreUint64(&f.frozen, length)
+	atomic.StoreUint64(&f.frozen, head)
+	atomic.StoreUint64(&f.tail, tail)
 	return nil
 }
 

core/rawdb/freezer_resettable.go

@@ -0,0 +1,233 @@
+// Copyright 2022 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package rawdb
+
+import (
+	"os"
+	"path/filepath"
+	"sync"
+
+	"github.com/ethereum/go-ethereum/ethdb"
+)
+
+const tmpSuffix = ".tmp"
+
+// freezerOpenFunc is the function used to open/create a freezer.
+type freezerOpenFunc = func() (*Freezer, error)
+
+// ResettableFreezer is a wrapper of the freezer which makes the
+// freezer resettable.
+type ResettableFreezer struct {
+	freezer *Freezer
+	opener  freezerOpenFunc
+	datadir string
+	lock    sync.RWMutex
+}
+
+// NewResettableFreezer creates a resettable freezer, note freezer is
+// only resettable if the passed file directory is exclusively occupied
+// by the freezer. And also the user-configurable ancient root directory
+// is **not** supported for reset since it might be a mount and rename
+// will cause a copy of hundreds of gigabyte into local directory. It
+// needs some other file based solutions.
+//
+// The reset function will delete directory atomically and re-create the
+// freezer from scratch.
+func NewResettableFreezer(datadir string, namespace string, readonly bool, maxTableSize uint32, tables map[string]bool) (*ResettableFreezer, error) {
+	if err := cleanup(datadir); err != nil {
+		return nil, err
+	}
+	opener := func() (*Freezer, error) {
+		return NewFreezer(datadir, namespace, readonly, maxTableSize, tables)
+	}
+	freezer, err := opener()
+	if err != nil {
+		return nil, err
+	}
+	return &ResettableFreezer{
+		freezer: freezer,
+		opener:  opener,
+		datadir: datadir,
+	}, nil
+}
+
+// Reset deletes the file directory exclusively occupied by the freezer and
+// recreate the freezer from scratch. The atomicity of directory deletion
+// is guaranteed by the rename operation, the leftover directory will be
+// cleaned up in next startup in case crash happens after rename.
+func (f *ResettableFreezer) Reset() error {
+	f.lock.Lock()
+	defer f.lock.Unlock()
+
+	if err := f.freezer.Close(); err != nil {
+		return err
+	}
+	tmp := tmpName(f.datadir)
+	if err := os.Rename(f.datadir, tmp); err != nil {
+		return err
+	}
+	if err := os.RemoveAll(tmp); err != nil {
+		return err
+	}
+	freezer, err := f.opener()
+	if err != nil {
+		return err
+	}
+	f.freezer = freezer
+	return nil
+}
+
+// Close terminates the chain freezer, unmapping all the data files.
+func (f *ResettableFreezer) Close() error {
+	f.lock.RLock()
+	defer f.lock.RUnlock()
+
+	return f.freezer.Close()
+}
+
+// HasAncient returns an indicator whether the specified ancient data exists
+// in the freezer
+func (f *ResettableFreezer) HasAncient(kind string, number uint64) (bool, error) {
+	f.lock.RLock()
+	defer f.lock.RUnlock()
+
+	return f.freezer.HasAncient(kind, number)
+}
+
+// Ancient retrieves an ancient binary blob from the append-only immutable files.
+func (f *ResettableFreezer) Ancient(kind string, number uint64) ([]byte, error) {
+	f.lock.RLock()
+	defer f.lock.RUnlock()
+
+	return f.freezer.Ancient(kind, number)
+}
+
+// AncientRange retrieves multiple items in sequence, starting from the index 'start'.
+// It will return
+//   - at most 'max' items,
+//   - at least 1 item (even if exceeding the maxByteSize), but will otherwise
+//     return as many items as fit into maxByteSize
+func (f *ResettableFreezer) AncientRange(kind string, start, count, maxBytes uint64) ([][]byte, error) {
+	f.lock.RLock()
+	defer f.lock.RUnlock()
+
+	return f.freezer.AncientRange(kind, start, count, maxBytes)
+}
+
+// Ancients returns the length of the frozen items.
+func (f *ResettableFreezer) Ancients() (uint64, error) {
+	f.lock.RLock()
+	defer f.lock.RUnlock()
+
+	return f.freezer.Ancients()
+}
+
+// Tail returns the number of first stored item in the freezer.
+func (f *ResettableFreezer) Tail() (uint64, error) {
+	f.lock.RLock()
+	defer f.lock.RUnlock()
+
+	return f.freezer.Tail()
+}
+
+// AncientSize returns the ancient size of the specified category.
+func (f *ResettableFreezer) AncientSize(kind string) (uint64, error) {
+	f.lock.RLock()
+	defer f.lock.RUnlock()
+
+	return f.freezer.AncientSize(kind)
+}
+
+// ReadAncients runs the given read operation while ensuring that no writes take place
+// on the underlying freezer.
+func (f *ResettableFreezer) ReadAncients(fn func(ethdb.AncientReaderOp) error) (err error) {
+	f.lock.RLock()
+	defer f.lock.RUnlock()
+
+	return f.freezer.ReadAncients(fn)
+}
+
+// ModifyAncients runs the given write operation.
+func (f *ResettableFreezer) ModifyAncients(fn func(ethdb.AncientWriteOp) error) (writeSize int64, err error) {
+	f.lock.RLock()
+	defer f.lock.RUnlock()
+
+	return f.freezer.ModifyAncients(fn)
+}
+
+// TruncateHead discards any recent data above the provided threshold number.
+func (f *ResettableFreezer) TruncateHead(items uint64) error {
+	f.lock.RLock()
+	defer f.lock.RUnlock()
+
+	return f.freezer.TruncateHead(items)
+}
+
+// TruncateTail discards any recent data below the provided threshold number.
+func (f *ResettableFreezer) TruncateTail(tail uint64) error {
+	f.lock.RLock()
+	defer f.lock.RUnlock()
+
+	return f.freezer.TruncateTail(tail)
+}
+
+// Sync flushes all data tables to disk.
+func (f *ResettableFreezer) Sync() error {
+	f.lock.RLock()
+	defer f.lock.RUnlock()
+
+	return f.freezer.Sync()
+}
+
+// MigrateTable processes the entries in a given table in sequence
+// converting them to a new format if they're of an old format.
+func (f *ResettableFreezer) MigrateTable(kind string, convert convertLegacyFn) error {
+	f.lock.RLock()
+	defer f.lock.RUnlock()
+
+	return f.freezer.MigrateTable(kind, convert)
+}
+
+// cleanup removes the directory located in the specified path
+// has the name with deletion marker suffix.
+func cleanup(path string) error {
+	parent := filepath.Dir(path)
+	if _, err := os.Lstat(parent); os.IsNotExist(err) {
+		return nil
+	}
+	dir, err := os.Open(parent)
+	if err != nil {
+		return err
+	}
+	names, err := dir.Readdirnames(0)
+	if err != nil {
+		return err
+	}
+	if cerr := dir.Close(); cerr != nil {
+		return cerr
+	}
+	for _, name := range names {
+		if name == filepath.Base(path)+tmpSuffix {
+			return os.RemoveAll(filepath.Join(parent, name))
+		}
+	}
+	return nil
+}
+
+func tmpName(path string) string {
+	return filepath.Join(filepath.Dir(path), filepath.Base(path)+tmpSuffix)
+}

core/rawdb/freezer_resettable_test.go

@@ -0,0 +1,107 @@
+// Copyright 2022 The go-ethereum Authors
+// This file is part of the go-ethereum library.
+//
+// The go-ethereum library is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Lesser General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// The go-ethereum library is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Lesser General Public License for more details.
+//
+// You should have received a copy of the GNU Lesser General Public License
+// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
+
+package rawdb
+
+import (
+	"bytes"
+	"os"
+	"testing"
+
+	"github.com/ethereum/go-ethereum/ethdb"
+)
+
+func TestResetFreezer(t *testing.T) {
+	items := []struct {
+		id   uint64
+		blob []byte
+	}{
+		{0, bytes.Repeat([]byte{0}, 2048)},
+		{1, bytes.Repeat([]byte{1}, 2048)},
+		{2, bytes.Repeat([]byte{2}, 2048)},
+	}
+	f, _ := NewResettableFreezer(t.TempDir(), "", false, 2048, freezerTestTableDef)
+	defer f.Close()
+
+	f.ModifyAncients(func(op ethdb.AncientWriteOp) error {
+		for _, item := range items {
+			op.AppendRaw("test", item.id, item.blob)
+		}
+		return nil
+	})
+	for _, item := range items {
+		blob, _ := f.Ancient("test", item.id)
+		if !bytes.Equal(blob, item.blob) {
+			t.Fatal("Unexpected blob")
+		}
+	}
+
+	// Reset freezer
+	f.Reset()
+	count, _ := f.Ancients()
+	if count != 0 {
+		t.Fatal("Failed to reset freezer")
+	}
+	for _, item := range items {
+		blob, _ := f.Ancient("test", item.id)
+		if len(blob) != 0 {
+			t.Fatal("Unexpected blob")
+		}
+	}
+
+	// Fill the freezer
+	f.ModifyAncients(func(op ethdb.AncientWriteOp) error {
+		for _, item := range items {
+			op.AppendRaw("test", item.id, item.blob)
+		}
+		return nil
+	})
+	for _, item := range items {
+		blob, _ := f.Ancient("test", item.id)
+		if !bytes.Equal(blob, item.blob) {
+			t.Fatal("Unexpected blob")
+		}
+	}
+}
+
+func TestFreezerCleanup(t *testing.T) {
+	items := []struct {
+		id   uint64
+		blob []byte
+	}{
+		{0, bytes.Repeat([]byte{0}, 2048)},
+		{1, bytes.Repeat([]byte{1}, 2048)},
+		{2, bytes.Repeat([]byte{2}, 2048)},
+	}
+	datadir := t.TempDir()
+	f, _ := NewResettableFreezer(datadir, "", false, 2048, freezerTestTableDef)
+	f.ModifyAncients(func(op ethdb.AncientWriteOp) error {
+		for _, item := range items {
+			op.AppendRaw("test", item.id, item.blob)
+		}
+		return nil
+	})
+	f.Close()
+	os.Rename(datadir, tmpName(datadir))
+
+	// Open the freezer again, trigger cleanup operation
+	f, _ = NewResettableFreezer(datadir, "", false, 2048, freezerTestTableDef)
+	f.Close()
+
+	if _, err := os.Lstat(tmpName(datadir)); !os.IsNotExist(err) {
+		t.Fatal("Failed to cleanup leftover directory")
+	}
+}

core/rawdb/freezer_table.go

@@ -148,20 +148,12 @@ func newTable(path string, name string, readMeter metrics.Meter, writeMeter metr
 		meta  *os.File
 	)
 	if readonly {
-		// Will fail if table doesn't exist
+		// Will fail if table index file or meta file is not existent
 		index, err = openFreezerFileForReadOnly(filepath.Join(path, idxName))
 		if err != nil {
 			return nil, err
 		}
-		// TODO(rjl493456442) change it to read-only mode. Open the metadata file
-		// in rw mode. It's a temporary solution for now and should be changed
-		// whenever the tail deletion is actually used. The reason for this hack is
-		// the additional meta file for each freezer table is added in order to support
-		// tail deletion, but for most legacy nodes this file is missing. This check
-		// will suddenly break lots of database relevant commands. So the metadata file
-		// is always opened for mutation and nothing else will be written except
-		// the initialization.
-		meta, err = openFreezerFileForAppend(filepath.Join(path, fmt.Sprintf("%s.meta", name)))
+		meta, err = openFreezerFileForReadOnly(filepath.Join(path, fmt.Sprintf("%s.meta", name)))
 		if err != nil {
 			return nil, err
 		}
@@ -237,6 +229,7 @@ func (t *freezerTable) repair() error {
 		lastIndex   indexEntry
 		contentSize int64
 		contentExp  int64
+		verbose     bool
 	)
 	// Read index zero, determine what file is the earliest
 	// and what item offset to use
@@ -280,9 +273,10 @@ func (t *freezerTable) repair() error {
 	// Keep truncating both files until they come in sync
 	contentExp = int64(lastIndex.offset)
 	for contentExp != contentSize {
+		verbose = true
 		// Truncate the head file to the last offset pointer
 		if contentExp < contentSize {
-			t.logger.Warn("Truncating dangling head", "indexed", common.StorageSize(contentExp), "stored", common.StorageSize(contentSize))
+			t.logger.Warn("Truncating dangling head", "indexed", contentExp, "stored", contentSize)
 			if err := truncateFreezerFile(t.head, contentExp); err != nil {
 				return err
 			}
@@ -290,7 +284,7 @@ func (t *freezerTable) repair() error {
 		}
 		// Truncate the index to point within the head file
 		if contentExp > contentSize {
-			t.logger.Warn("Truncating dangling indexes", "indexed", common.StorageSize(contentExp), "stored", common.StorageSize(contentSize))
+			t.logger.Warn("Truncating dangling indexes", "indexes", offsetsSize/indexEntrySize, "indexed", contentExp, "stored", contentSize)
 			if err := truncateFreezerFile(t.index, offsetsSize-indexEntrySize); err != nil {
 				return err
 			}
@@ -351,7 +345,11 @@ func (t *freezerTable) repair() error {
 	if err := t.preopen(); err != nil {
 		return err
 	}
-	t.logger.Debug("Chain freezer table opened", "items", t.items, "size", common.StorageSize(t.headBytes))
+	if verbose {
+		t.logger.Info("Chain freezer table opened", "items", t.items, "size", t.headBytes)
+	} else {
+		t.logger.Debug("Chain freezer table opened", "items", t.items, "size", common.StorageSize(t.headBytes))
+	}
 	return nil
 }
 
@@ -561,21 +559,31 @@ func (t *freezerTable) Close() error {
 	defer t.lock.Unlock()
 
 	var errs []error
-	if err := t.index.Close(); err != nil {
-		errs = append(errs, err)
-	}
-	t.index = nil
-
-	if err := t.meta.Close(); err != nil {
-		errs = append(errs, err)
-	}
-	t.meta = nil
-
-	for _, f := range t.files {
-		if err := f.Close(); err != nil {
-			errs = append(errs, err)
+	doClose := func(f *os.File, sync bool, close bool) {
+		if sync && !t.readonly {
+			if err := f.Sync(); err != nil {
+				errs = append(errs, err)
+			}
+		}
+		if close {
+			if err := f.Close(); err != nil {
+				errs = append(errs, err)
+			}
 		}
 	}
+	// Trying to fsync a file opened in rdonly causes "Access denied"
+	// error on Windows.
+	doClose(t.index, true, true)
+	doClose(t.meta, true, true)
+	// The preopened non-head data-files are all opened in readonly.
+	// The head is opened in rw-mode, so we sync it here - but since it's also
+	// part of t.files, it will be closed in the loop below.
+	doClose(t.head, true, false) // sync but do not close
+	for _, f := range t.files {
+		doClose(f, false, true) // close but do not sync
+	}
+	t.index = nil
+	t.meta = nil
 	t.head = nil
 
 	if errs != nil {
@@ -732,7 +740,7 @@ func (t *freezerTable) retrieveItems(start, count, maxBytes uint64) ([]byte, []i
 	defer t.lock.RUnlock()
 
 	// Ensure the table and the item are accessible
-	if t.index == nil || t.head == nil {
+	if t.index == nil || t.head == nil || t.meta == nil {
 		return nil, nil, errClosed
 	}
 	var (
@@ -860,8 +868,11 @@ func (t *freezerTable) advanceHead() error {
 	if err != nil {
 		return err
 	}
-
-	// Close old file, and reopen in RDONLY mode.
+	// Commit the contents of the old file to stable storage and
+	// tear it down. It will be re-opened in read-only mode.
+	if err := t.head.Sync(); err != nil {
+		return err
+	}
 	t.releaseFile(t.headId)
 	t.openFile(t.headId, openFreezerFileForReadOnly)
 
@@ -875,13 +886,22 @@ func (t *freezerTable) advanceHead() error {
 // Sync pushes any pending data from memory out to disk. This is an expensive
 // operation, so use it with care.
 func (t *freezerTable) Sync() error {
-	if err := t.index.Sync(); err != nil {
-		return err
+	t.lock.Lock()
+	defer t.lock.Unlock()
+	if t.index == nil || t.head == nil || t.meta == nil {
+		return errClosed
 	}
-	if err := t.meta.Sync(); err != nil {
-		return err
+	var err error
+	trackError := func(e error) {
+		if e != nil && err == nil {
+			err = e
+		}
 	}
-	return t.head.Sync()
+
+	trackError(t.index.Sync())
+	trackError(t.meta.Sync())
+	trackError(t.head.Sync())
+	return err
 }
 
 func (t *freezerTable) dumpIndexStdout(start, stop int64) {
@@ -901,7 +921,8 @@ func (t *freezerTable) dumpIndex(w io.Writer, start, stop int64) {
 		fmt.Fprintf(w, "Failed to decode freezer table %v\n", err)
 		return
 	}
-	fmt.Fprintf(w, "Version %d deleted %d, hidden %d\n", meta.Version, atomic.LoadUint64(&t.itemOffset), atomic.LoadUint64(&t.itemHidden))
+	fmt.Fprintf(w, "Version %d count %d, deleted %d, hidden %d\n", meta.Version,
+		atomic.LoadUint64(&t.items), atomic.LoadUint64(&t.itemOffset), atomic.LoadUint64(&t.itemHidden))
 
 	buf := make([]byte, indexEntrySize)
 

core/rawdb/freezer_table_test.go

@@ -27,17 +27,12 @@ import (
 	"sync/atomic"
 	"testing"
 	"testing/quick"
-	"time"
 
 	"github.com/davecgh/go-spew/spew"
 	"github.com/ethereum/go-ethereum/metrics"
 	"github.com/stretchr/testify/require"
 )
 
-func init() {
-	rand.Seed(time.Now().Unix())
-}
-
 // TestFreezerBasics test initializing a freezertable from scratch, writing to the table,
 // and reading it back.
 func TestFreezerBasics(t *testing.T) {

core/rawdb/freezer_test.go

@@ -190,7 +190,7 @@ func TestFreezerConcurrentModifyTruncate(t *testing.T) {
 
 	var item = make([]byte, 256)
 
-	for i := 0; i < 1000; i++ {
+	for i := 0; i < 10; i++ {
 		// First reset and write 100 items.
 		if err := f.TruncateHead(0); err != nil {
 			t.Fatal("truncate failed:", err)
@@ -407,3 +407,25 @@ func TestRenameWindows(t *testing.T) {
 		t.Errorf("unexpected file contents. Got %v\n", buf)
 	}
 }
+
+func TestFreezerCloseSync(t *testing.T) {
+	t.Parallel()
+	f, _ := newFreezerForTesting(t, map[string]bool{"a": true, "b": true})
+	defer f.Close()
+
+	// Now, close and sync. This mimics the behaviour if the node is shut down,
+	// just as the chain freezer is writing.
+	// 1: thread-1: chain treezer writes, via freezeRange (holds lock)
+	// 2: thread-2: Close called, waits for write to finish
+	// 3: thread-1: finishes writing, releases lock
+	// 4: thread-2: obtains lock, completes Close()
+	// 5: thread-1: calls f.Sync()
+	if err := f.Close(); err != nil {
+		t.Fatal(err)
+	}
+	if err := f.Sync(); err == nil {
+		t.Fatalf("want error, have nil")
+	} else if have, want := err.Error(), "[closed closed]"; have != want {
+		t.Fatalf("want %v, have %v", have, want)
+	}
+}

core/rawdb/schema.go

@@ -100,12 +100,26 @@ var (
 	CodePrefix            = []byte("c") // CodePrefix + code hash -> account code
 	skeletonHeaderPrefix  = []byte("S") // skeletonHeaderPrefix + num (uint64 big endian) -> header
 
+	// Path-based trie node scheme.
+	trieNodeAccountPrefix = []byte("A") // trieNodeAccountPrefix + hexPath -> trie node
+	trieNodeStoragePrefix = []byte("O") // trieNodeStoragePrefix + accountHash + hexPath -> trie node
+
 	PreimagePrefix = []byte("secure-key-")       // PreimagePrefix + hash -> preimage
 	configPrefix   = []byte("ethereum-config-")  // config prefix for the db
 	genesisPrefix  = []byte("ethereum-genesis-") // genesis state prefix for the db
 
-	// Chain index prefixes (use `i` + single byte to avoid mixing data types).
-	BloomBitsIndexPrefix = []byte("iB") // BloomBitsIndexPrefix is the data table of a chain indexer to track its progress
+	// BloomBitsIndexPrefix is the data table of a chain indexer to track its progress
+	BloomBitsIndexPrefix = []byte("iB")
+
+	ChtPrefix           = []byte("chtRootV2-") // ChtPrefix + chtNum (uint64 big endian) -> trie root hash
+	ChtTablePrefix      = []byte("cht-")
+	ChtIndexTablePrefix = []byte("chtIndexV2-")
+
+	BloomTriePrefix      = []byte("bltRoot-") // BloomTriePrefix + bloomTrieNum (uint64 big endian) -> trie root hash
+	BloomTrieTablePrefix = []byte("blt-")
+	BloomTrieIndexPrefix = []byte("bltIndex-")
+
+	CliqueSnapshotPrefix = []byte("clique-")
 
 	preimageCounter    = metrics.NewRegisteredCounter("db/preimage/total", nil)
 	preimageHitCounter = metrics.NewRegisteredCounter("db/preimage/hits", nil)
@@ -224,3 +238,13 @@ func configKey(hash common.Hash) []byte {
 func genesisStateSpecKey(hash common.Hash) []byte {
 	return append(genesisPrefix, hash.Bytes()...)
 }
+
+// accountTrieNodeKey = trieNodeAccountPrefix + nodePath.
+func accountTrieNodeKey(path []byte) []byte {
+	return append(trieNodeAccountPrefix, path...)
+}
+
+// storageTrieNodeKey = trieNodeStoragePrefix + accountHash + nodePath.
+func storageTrieNodeKey(accountHash common.Hash, path []byte) []byte {
+	return append(append(trieNodeStoragePrefix, accountHash.Bytes()...), path...)
+}