Patch for concurrent iterator & others (onto v1.11.6) #386
@ -22,7 +22,6 @@ import (
|
|||||||
"os"
|
"os"
|
||||||
"os/signal"
|
"os/signal"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
"sort"
|
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"syscall"
|
"syscall"
|
||||||
@ -160,8 +159,8 @@ WARNING: This is a low-level operation which may cause database corruption!`,
|
|||||||
dbDumpFreezerIndex = &cli.Command{
|
dbDumpFreezerIndex = &cli.Command{
|
||||||
Action: freezerInspect,
|
Action: freezerInspect,
|
||||||
Name: "freezer-index",
|
Name: "freezer-index",
|
||||||
Usage: "Dump out the index of a given freezer type",
|
Usage: "Dump out the index of a specific freezer table",
|
||||||
ArgsUsage: "<type> <start (int)> <end (int)>",
|
ArgsUsage: "<freezer-type> <table-type> <start (int)> <end (int)>",
|
||||||
Flags: flags.Merge([]cli.Flag{
|
Flags: flags.Merge([]cli.Flag{
|
||||||
utils.SyncModeFlag,
|
utils.SyncModeFlag,
|
||||||
}, utils.NetworkFlags, utils.DatabasePathFlags),
|
}, utils.NetworkFlags, utils.DatabasePathFlags),
|
||||||
@ -275,7 +274,7 @@ func inspect(ctx *cli.Context) error {
|
|||||||
start []byte
|
start []byte
|
||||||
)
|
)
|
||||||
if ctx.NArg() > 2 {
|
if ctx.NArg() > 2 {
|
||||||
return fmt.Errorf("Max 2 arguments: %v", ctx.Command.ArgsUsage)
|
return fmt.Errorf("max 2 arguments: %v", ctx.Command.ArgsUsage)
|
||||||
}
|
}
|
||||||
if ctx.NArg() >= 1 {
|
if ctx.NArg() >= 1 {
|
||||||
if d, err := hexutil.Decode(ctx.Args().Get(0)); err != nil {
|
if d, err := hexutil.Decode(ctx.Args().Get(0)); err != nil {
|
||||||
@ -536,43 +535,35 @@ func dbDumpTrie(ctx *cli.Context) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func freezerInspect(ctx *cli.Context) error {
|
func freezerInspect(ctx *cli.Context) error {
|
||||||
var (
|
if ctx.NArg() < 4 {
|
||||||
start, end int64
|
|
||||||
disableSnappy bool
|
|
||||||
err error
|
|
||||||
)
|
|
||||||
if ctx.NArg() < 3 {
|
|
||||||
return fmt.Errorf("required arguments: %v", ctx.Command.ArgsUsage)
|
return fmt.Errorf("required arguments: %v", ctx.Command.ArgsUsage)
|
||||||
}
|
}
|
||||||
kind := ctx.Args().Get(0)
|
var (
|
||||||
if noSnap, ok := rawdb.FreezerNoSnappy[kind]; !ok {
|
freezer = ctx.Args().Get(0)
|
||||||
var options []string
|
table = ctx.Args().Get(1)
|
||||||
for opt := range rawdb.FreezerNoSnappy {
|
)
|
||||||
options = append(options, opt)
|
start, err := strconv.ParseInt(ctx.Args().Get(2), 10, 64)
|
||||||
}
|
if err != nil {
|
||||||
sort.Strings(options)
|
log.Info("Could not read start-param", "err", err)
|
||||||
return fmt.Errorf("Could read freezer-type '%v'. Available options: %v", kind, options)
|
|
||||||
} else {
|
|
||||||
disableSnappy = noSnap
|
|
||||||
}
|
|
||||||
if start, err = strconv.ParseInt(ctx.Args().Get(1), 10, 64); err != nil {
|
|
||||||
log.Info("Could read start-param", "error", err)
|
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
if end, err = strconv.ParseInt(ctx.Args().Get(2), 10, 64); err != nil {
|
end, err := strconv.ParseInt(ctx.Args().Get(3), 10, 64)
|
||||||
log.Info("Could read count param", "error", err)
|
if err != nil {
|
||||||
|
log.Info("Could not read count param", "err", err)
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
stack, _ := makeConfigNode(ctx)
|
stack, _ := makeConfigNode(ctx)
|
||||||
defer stack.Close()
|
defer stack.Close()
|
||||||
path := filepath.Join(stack.ResolvePath("chaindata"), "ancient")
|
|
||||||
log.Info("Opening freezer", "location", path, "name", kind)
|
db := utils.MakeChainDatabase(ctx, stack, true)
|
||||||
if f, err := rawdb.NewFreezerTable(path, kind, disableSnappy, true); err != nil {
|
defer db.Close()
|
||||||
|
|
||||||
|
ancient, err := db.AncientDatadir()
|
||||||
|
if err != nil {
|
||||||
|
log.Info("Failed to retrieve ancient root", "err", err)
|
||||||
return err
|
return err
|
||||||
} else {
|
|
||||||
f.DumpIndex(start, end)
|
|
||||||
}
|
}
|
||||||
return nil
|
return rawdb.InspectFreezerTable(ancient, freezer, table, start, end)
|
||||||
}
|
}
|
||||||
|
|
||||||
func importLDBdata(ctx *cli.Context) error {
|
func importLDBdata(ctx *cli.Context) error {
|
||||||
|
@ -91,7 +91,7 @@ var (
|
|||||||
}
|
}
|
||||||
AncientFlag = &flags.DirectoryFlag{
|
AncientFlag = &flags.DirectoryFlag{
|
||||||
Name: "datadir.ancient",
|
Name: "datadir.ancient",
|
||||||
Usage: "Data directory for ancient chain segments (default = inside chaindata)",
|
Usage: "Root directory for ancient data (default = inside chaindata)",
|
||||||
Category: flags.EthCategory,
|
Category: flags.EthCategory,
|
||||||
}
|
}
|
||||||
MinFreeDiskSpaceFlag = &flags.DirectoryFlag{
|
MinFreeDiskSpaceFlag = &flags.DirectoryFlag{
|
||||||
|
@ -37,7 +37,7 @@ import (
|
|||||||
func ReadCanonicalHash(db ethdb.Reader, number uint64) common.Hash {
|
func ReadCanonicalHash(db ethdb.Reader, number uint64) common.Hash {
|
||||||
var data []byte
|
var data []byte
|
||||||
db.ReadAncients(func(reader ethdb.AncientReaderOp) error {
|
db.ReadAncients(func(reader ethdb.AncientReaderOp) error {
|
||||||
data, _ = reader.Ancient(freezerHashTable, number)
|
data, _ = reader.Ancient(chainFreezerHashTable, number)
|
||||||
if len(data) == 0 {
|
if len(data) == 0 {
|
||||||
// Get it by hash from leveldb
|
// Get it by hash from leveldb
|
||||||
data, _ = db.Get(headerHashKey(number))
|
data, _ = db.Get(headerHashKey(number))
|
||||||
@ -335,7 +335,7 @@ func ReadHeaderRange(db ethdb.Reader, number uint64, count uint64) []rlp.RawValu
|
|||||||
}
|
}
|
||||||
// read remaining from ancients
|
// read remaining from ancients
|
||||||
max := count * 700
|
max := count * 700
|
||||||
data, err := db.AncientRange(freezerHeaderTable, i+1-count, count, max)
|
data, err := db.AncientRange(chainFreezerHeaderTable, i+1-count, count, max)
|
||||||
if err == nil && uint64(len(data)) == count {
|
if err == nil && uint64(len(data)) == count {
|
||||||
// the data is on the order [h, h+1, .., n] -- reordering needed
|
// the data is on the order [h, h+1, .., n] -- reordering needed
|
||||||
for i := range data {
|
for i := range data {
|
||||||
@ -352,7 +352,7 @@ func ReadHeaderRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValu
|
|||||||
// First try to look up the data in ancient database. Extra hash
|
// First try to look up the data in ancient database. Extra hash
|
||||||
// comparison is necessary since ancient database only maintains
|
// comparison is necessary since ancient database only maintains
|
||||||
// the canonical data.
|
// the canonical data.
|
||||||
data, _ = reader.Ancient(freezerHeaderTable, number)
|
data, _ = reader.Ancient(chainFreezerHeaderTable, number)
|
||||||
if len(data) > 0 && crypto.Keccak256Hash(data) == hash {
|
if len(data) > 0 && crypto.Keccak256Hash(data) == hash {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@ -428,7 +428,7 @@ func deleteHeaderWithoutNumber(db ethdb.KeyValueWriter, hash common.Hash, number
|
|||||||
// isCanon is an internal utility method, to check whether the given number/hash
|
// isCanon is an internal utility method, to check whether the given number/hash
|
||||||
// is part of the ancient (canon) set.
|
// is part of the ancient (canon) set.
|
||||||
func isCanon(reader ethdb.AncientReaderOp, number uint64, hash common.Hash) bool {
|
func isCanon(reader ethdb.AncientReaderOp, number uint64, hash common.Hash) bool {
|
||||||
h, err := reader.Ancient(freezerHashTable, number)
|
h, err := reader.Ancient(chainFreezerHashTable, number)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return false
|
return false
|
||||||
}
|
}
|
||||||
@ -444,7 +444,7 @@ func ReadBodyRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue
|
|||||||
db.ReadAncients(func(reader ethdb.AncientReaderOp) error {
|
db.ReadAncients(func(reader ethdb.AncientReaderOp) error {
|
||||||
// Check if the data is in ancients
|
// Check if the data is in ancients
|
||||||
if isCanon(reader, number, hash) {
|
if isCanon(reader, number, hash) {
|
||||||
data, _ = reader.Ancient(freezerBodiesTable, number)
|
data, _ = reader.Ancient(chainFreezerBodiesTable, number)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
// If not, try reading from leveldb
|
// If not, try reading from leveldb
|
||||||
@ -459,7 +459,7 @@ func ReadBodyRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue
|
|||||||
func ReadCanonicalBodyRLP(db ethdb.Reader, number uint64) rlp.RawValue {
|
func ReadCanonicalBodyRLP(db ethdb.Reader, number uint64) rlp.RawValue {
|
||||||
var data []byte
|
var data []byte
|
||||||
db.ReadAncients(func(reader ethdb.AncientReaderOp) error {
|
db.ReadAncients(func(reader ethdb.AncientReaderOp) error {
|
||||||
data, _ = reader.Ancient(freezerBodiesTable, number)
|
data, _ = reader.Ancient(chainFreezerBodiesTable, number)
|
||||||
if len(data) > 0 {
|
if len(data) > 0 {
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
@ -527,7 +527,7 @@ func ReadTdRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue {
|
|||||||
db.ReadAncients(func(reader ethdb.AncientReaderOp) error {
|
db.ReadAncients(func(reader ethdb.AncientReaderOp) error {
|
||||||
// Check if the data is in ancients
|
// Check if the data is in ancients
|
||||||
if isCanon(reader, number, hash) {
|
if isCanon(reader, number, hash) {
|
||||||
data, _ = reader.Ancient(freezerDifficultyTable, number)
|
data, _ = reader.Ancient(chainFreezerDifficultyTable, number)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
// If not, try reading from leveldb
|
// If not, try reading from leveldb
|
||||||
@ -587,7 +587,7 @@ func ReadReceiptsRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawVa
|
|||||||
db.ReadAncients(func(reader ethdb.AncientReaderOp) error {
|
db.ReadAncients(func(reader ethdb.AncientReaderOp) error {
|
||||||
// Check if the data is in ancients
|
// Check if the data is in ancients
|
||||||
if isCanon(reader, number, hash) {
|
if isCanon(reader, number, hash) {
|
||||||
data, _ = reader.Ancient(freezerReceiptTable, number)
|
data, _ = reader.Ancient(chainFreezerReceiptTable, number)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
// If not, try reading from leveldb
|
// If not, try reading from leveldb
|
||||||
@ -819,19 +819,19 @@ func WriteAncientBlocks(db ethdb.AncientWriter, blocks []*types.Block, receipts
|
|||||||
|
|
||||||
func writeAncientBlock(op ethdb.AncientWriteOp, block *types.Block, header *types.Header, receipts []*types.ReceiptForStorage, td *big.Int) error {
|
func writeAncientBlock(op ethdb.AncientWriteOp, block *types.Block, header *types.Header, receipts []*types.ReceiptForStorage, td *big.Int) error {
|
||||||
num := block.NumberU64()
|
num := block.NumberU64()
|
||||||
if err := op.AppendRaw(freezerHashTable, num, block.Hash().Bytes()); err != nil {
|
if err := op.AppendRaw(chainFreezerHashTable, num, block.Hash().Bytes()); err != nil {
|
||||||
return fmt.Errorf("can't add block %d hash: %v", num, err)
|
return fmt.Errorf("can't add block %d hash: %v", num, err)
|
||||||
}
|
}
|
||||||
if err := op.Append(freezerHeaderTable, num, header); err != nil {
|
if err := op.Append(chainFreezerHeaderTable, num, header); err != nil {
|
||||||
return fmt.Errorf("can't append block header %d: %v", num, err)
|
return fmt.Errorf("can't append block header %d: %v", num, err)
|
||||||
}
|
}
|
||||||
if err := op.Append(freezerBodiesTable, num, block.Body()); err != nil {
|
if err := op.Append(chainFreezerBodiesTable, num, block.Body()); err != nil {
|
||||||
return fmt.Errorf("can't append block body %d: %v", num, err)
|
return fmt.Errorf("can't append block body %d: %v", num, err)
|
||||||
}
|
}
|
||||||
if err := op.Append(freezerReceiptTable, num, receipts); err != nil {
|
if err := op.Append(chainFreezerReceiptTable, num, receipts); err != nil {
|
||||||
return fmt.Errorf("can't append block %d receipts: %v", num, err)
|
return fmt.Errorf("can't append block %d receipts: %v", num, err)
|
||||||
}
|
}
|
||||||
if err := op.Append(freezerDifficultyTable, num, td); err != nil {
|
if err := op.Append(chainFreezerDifficultyTable, num, td); err != nil {
|
||||||
return fmt.Errorf("can't append block %d total difficulty: %v", num, err)
|
return fmt.Errorf("can't append block %d total difficulty: %v", num, err)
|
||||||
}
|
}
|
||||||
return nil
|
return nil
|
||||||
|
86
core/rawdb/ancient_scheme.go
Normal file
86
core/rawdb/ancient_scheme.go
Normal file
@ -0,0 +1,86 @@
|
|||||||
|
// Copyright 2022 The go-ethereum Authors
|
||||||
|
// This file is part of the go-ethereum library.
|
||||||
|
//
|
||||||
|
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||||
|
// it under the terms of the GNU Lesser General Public License as published by
|
||||||
|
// the Free Software Foundation, either version 3 of the License, or
|
||||||
|
// (at your option) any later version.
|
||||||
|
//
|
||||||
|
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||||
|
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||||
|
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||||
|
// GNU Lesser General Public License for more details.
|
||||||
|
//
|
||||||
|
// You should have received a copy of the GNU Lesser General Public License
|
||||||
|
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||||
|
|
||||||
|
package rawdb
|
||||||
|
|
||||||
|
import "fmt"
|
||||||
|
|
||||||
|
// The list of table names of chain freezer.
|
||||||
|
const (
|
||||||
|
// chainFreezerHeaderTable indicates the name of the freezer header table.
|
||||||
|
chainFreezerHeaderTable = "headers"
|
||||||
|
|
||||||
|
// chainFreezerHashTable indicates the name of the freezer canonical hash table.
|
||||||
|
chainFreezerHashTable = "hashes"
|
||||||
|
|
||||||
|
// chainFreezerBodiesTable indicates the name of the freezer block body table.
|
||||||
|
chainFreezerBodiesTable = "bodies"
|
||||||
|
|
||||||
|
// chainFreezerReceiptTable indicates the name of the freezer receipts table.
|
||||||
|
chainFreezerReceiptTable = "receipts"
|
||||||
|
|
||||||
|
// chainFreezerDifficultyTable indicates the name of the freezer total difficulty table.
|
||||||
|
chainFreezerDifficultyTable = "diffs"
|
||||||
|
)
|
||||||
|
|
||||||
|
// chainFreezerNoSnappy configures whether compression is disabled for the ancient-tables.
|
||||||
|
// Hashes and difficulties don't compress well.
|
||||||
|
var chainFreezerNoSnappy = map[string]bool{
|
||||||
|
chainFreezerHeaderTable: false,
|
||||||
|
chainFreezerHashTable: true,
|
||||||
|
chainFreezerBodiesTable: false,
|
||||||
|
chainFreezerReceiptTable: false,
|
||||||
|
chainFreezerDifficultyTable: true,
|
||||||
|
}
|
||||||
|
|
||||||
|
// The list of identifiers of ancient stores.
|
||||||
|
var (
|
||||||
|
chainFreezerName = "chain" // the folder name of chain segment ancient store.
|
||||||
|
)
|
||||||
|
|
||||||
|
// freezers the collections of all builtin freezers.
|
||||||
|
var freezers = []string{chainFreezerName}
|
||||||
|
|
||||||
|
// InspectFreezerTable dumps out the index of a specific freezer table. The passed
|
||||||
|
// ancient indicates the path of root ancient directory where the chain freezer can
|
||||||
|
// be opened. Start and end specify the range for dumping out indexes.
|
||||||
|
// Note this function can only be used for debugging purposes.
|
||||||
|
func InspectFreezerTable(ancient string, freezerName string, tableName string, start, end int64) error {
|
||||||
|
var (
|
||||||
|
path string
|
||||||
|
tables map[string]bool
|
||||||
|
)
|
||||||
|
switch freezerName {
|
||||||
|
case chainFreezerName:
|
||||||
|
path, tables = resolveChainFreezerDir(ancient), chainFreezerNoSnappy
|
||||||
|
default:
|
||||||
|
return fmt.Errorf("unknown freezer, supported ones: %v", freezers)
|
||||||
|
}
|
||||||
|
noSnappy, exist := tables[tableName]
|
||||||
|
if !exist {
|
||||||
|
var names []string
|
||||||
|
for name := range tables {
|
||||||
|
names = append(names, name)
|
||||||
|
}
|
||||||
|
return fmt.Errorf("unknown table, supported ones: %v", names)
|
||||||
|
}
|
||||||
|
table, err := newFreezerTable(path, tableName, noSnappy, true)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
table.dumpIndexStdout(start, end)
|
||||||
|
return nil
|
||||||
|
}
|
@ -278,19 +278,19 @@ func (f *chainFreezer) freezeRange(nfdb *nofreezedb, number, limit uint64) (hash
|
|||||||
}
|
}
|
||||||
|
|
||||||
// Write to the batch.
|
// Write to the batch.
|
||||||
if err := op.AppendRaw(freezerHashTable, number, hash[:]); err != nil {
|
if err := op.AppendRaw(chainFreezerHashTable, number, hash[:]); err != nil {
|
||||||
return fmt.Errorf("can't write hash to Freezer: %v", err)
|
return fmt.Errorf("can't write hash to Freezer: %v", err)
|
||||||
}
|
}
|
||||||
if err := op.AppendRaw(freezerHeaderTable, number, header); err != nil {
|
if err := op.AppendRaw(chainFreezerHeaderTable, number, header); err != nil {
|
||||||
return fmt.Errorf("can't write header to Freezer: %v", err)
|
return fmt.Errorf("can't write header to Freezer: %v", err)
|
||||||
}
|
}
|
||||||
if err := op.AppendRaw(freezerBodiesTable, number, body); err != nil {
|
if err := op.AppendRaw(chainFreezerBodiesTable, number, body); err != nil {
|
||||||
return fmt.Errorf("can't write body to Freezer: %v", err)
|
return fmt.Errorf("can't write body to Freezer: %v", err)
|
||||||
}
|
}
|
||||||
if err := op.AppendRaw(freezerReceiptTable, number, receipts); err != nil {
|
if err := op.AppendRaw(chainFreezerReceiptTable, number, receipts); err != nil {
|
||||||
return fmt.Errorf("can't write receipts to Freezer: %v", err)
|
return fmt.Errorf("can't write receipts to Freezer: %v", err)
|
||||||
}
|
}
|
||||||
if err := op.AppendRaw(freezerDifficultyTable, number, td); err != nil {
|
if err := op.AppendRaw(chainFreezerDifficultyTable, number, td); err != nil {
|
||||||
return fmt.Errorf("can't write td to Freezer: %v", err)
|
return fmt.Errorf("can't write td to Freezer: %v", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -50,7 +50,7 @@ func InitDatabaseFromFreezer(db ethdb.Database) {
|
|||||||
if i+count > frozen {
|
if i+count > frozen {
|
||||||
count = frozen - i
|
count = frozen - i
|
||||||
}
|
}
|
||||||
data, err := db.AncientRange(freezerHashTable, i, count, 32*count)
|
data, err := db.AncientRange(chainFreezerHashTable, i, count, 32*count)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Crit("Failed to init database from freezer", "err", err)
|
log.Crit("Failed to init database from freezer", "err", err)
|
||||||
}
|
}
|
||||||
|
@ -21,6 +21,7 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"os"
|
"os"
|
||||||
|
"path"
|
||||||
"sync/atomic"
|
"sync/atomic"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@ -34,10 +35,16 @@ import (
|
|||||||
|
|
||||||
// freezerdb is a database wrapper that enabled freezer data retrievals.
|
// freezerdb is a database wrapper that enabled freezer data retrievals.
|
||||||
type freezerdb struct {
|
type freezerdb struct {
|
||||||
|
ancientRoot string
|
||||||
ethdb.KeyValueStore
|
ethdb.KeyValueStore
|
||||||
ethdb.AncientStore
|
ethdb.AncientStore
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// AncientDatadir returns the path of root ancient directory.
|
||||||
|
func (frdb *freezerdb) AncientDatadir() (string, error) {
|
||||||
|
return frdb.ancientRoot, nil
|
||||||
|
}
|
||||||
|
|
||||||
// Close implements io.Closer, closing both the fast key-value store as well as
|
// Close implements io.Closer, closing both the fast key-value store as well as
|
||||||
// the slow ancient tables.
|
// the slow ancient tables.
|
||||||
func (frdb *freezerdb) Close() error {
|
func (frdb *freezerdb) Close() error {
|
||||||
@ -162,12 +169,36 @@ func NewDatabase(db ethdb.KeyValueStore) ethdb.Database {
|
|||||||
return &nofreezedb{KeyValueStore: db}
|
return &nofreezedb{KeyValueStore: db}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// resolveChainFreezerDir is a helper function which resolves the absolute path
|
||||||
|
// of chain freezer by considering backward compatibility.
|
||||||
|
func resolveChainFreezerDir(ancient string) string {
|
||||||
|
// Check if the chain freezer is already present in the specified
|
||||||
|
// sub folder, if not then two possibilities:
|
||||||
|
// - chain freezer is not initialized
|
||||||
|
// - chain freezer exists in legacy location (root ancient folder)
|
||||||
|
freezer := path.Join(ancient, chainFreezerName)
|
||||||
|
if !common.FileExist(freezer) {
|
||||||
|
if !common.FileExist(ancient) {
|
||||||
|
// The entire ancient store is not initialized, still use the sub
|
||||||
|
// folder for initialization.
|
||||||
|
} else {
|
||||||
|
// Ancient root is already initialized, then we hold the assumption
|
||||||
|
// that chain freezer is also initialized and located in root folder.
|
||||||
|
// In this case fallback to legacy location.
|
||||||
|
freezer = ancient
|
||||||
|
log.Info("Found legacy ancient chain path", "location", ancient)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return freezer
|
||||||
|
}
|
||||||
|
|
||||||
// NewDatabaseWithFreezer creates a high level database on top of a given key-
|
// NewDatabaseWithFreezer creates a high level database on top of a given key-
|
||||||
// value data store with a freezer moving immutable chain segments into cold
|
// value data store with a freezer moving immutable chain segments into cold
|
||||||
// storage.
|
// storage. The passed ancient indicates the path of root ancient directory
|
||||||
func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace string, readonly bool) (ethdb.Database, error) {
|
// where the chain freezer can be opened.
|
||||||
|
func NewDatabaseWithFreezer(db ethdb.KeyValueStore, ancient string, namespace string, readonly bool) (ethdb.Database, error) {
|
||||||
// Create the idle freezer instance
|
// Create the idle freezer instance
|
||||||
frdb, err := newChainFreezer(freezer, namespace, readonly, freezerTableSize, FreezerNoSnappy)
|
frdb, err := newChainFreezer(resolveChainFreezerDir(ancient), namespace, readonly, freezerTableSize, chainFreezerNoSnappy)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
@ -198,7 +229,7 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace st
|
|||||||
// If the freezer already contains something, ensure that the genesis blocks
|
// If the freezer already contains something, ensure that the genesis blocks
|
||||||
// match, otherwise we might mix up freezers across chains and destroy both
|
// match, otherwise we might mix up freezers across chains and destroy both
|
||||||
// the freezer and the key-value store.
|
// the freezer and the key-value store.
|
||||||
frgenesis, err := frdb.Ancient(freezerHashTable, 0)
|
frgenesis, err := frdb.Ancient(chainFreezerHashTable, 0)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, fmt.Errorf("failed to retrieve genesis from ancient %v", err)
|
return nil, fmt.Errorf("failed to retrieve genesis from ancient %v", err)
|
||||||
} else if !bytes.Equal(kvgenesis, frgenesis) {
|
} else if !bytes.Equal(kvgenesis, frgenesis) {
|
||||||
@ -244,6 +275,7 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace st
|
|||||||
}()
|
}()
|
||||||
}
|
}
|
||||||
return &freezerdb{
|
return &freezerdb{
|
||||||
|
ancientRoot: ancient,
|
||||||
KeyValueStore: db,
|
KeyValueStore: db,
|
||||||
AncientStore: frdb,
|
AncientStore: frdb,
|
||||||
}, nil
|
}, nil
|
||||||
@ -273,13 +305,15 @@ func NewLevelDBDatabase(file string, cache int, handles int, namespace string, r
|
|||||||
}
|
}
|
||||||
|
|
||||||
// NewLevelDBDatabaseWithFreezer creates a persistent key-value database with a
|
// NewLevelDBDatabaseWithFreezer creates a persistent key-value database with a
|
||||||
// freezer moving immutable chain segments into cold storage.
|
// freezer moving immutable chain segments into cold storage. The passed ancient
|
||||||
func NewLevelDBDatabaseWithFreezer(file string, cache int, handles int, freezer string, namespace string, readonly bool) (ethdb.Database, error) {
|
// indicates the path of root ancient directory where the chain freezer can be
|
||||||
|
// opened.
|
||||||
|
func NewLevelDBDatabaseWithFreezer(file string, cache int, handles int, ancient string, namespace string, readonly bool) (ethdb.Database, error) {
|
||||||
kvdb, err := leveldb.New(file, cache, handles, namespace, readonly)
|
kvdb, err := leveldb.New(file, cache, handles, namespace, readonly)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
frdb, err := NewDatabaseWithFreezer(kvdb, freezer, namespace, readonly)
|
frdb, err := NewDatabaseWithFreezer(kvdb, ancient, namespace, readonly)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
kvdb.Close()
|
kvdb.Close()
|
||||||
return nil, err
|
return nil, err
|
||||||
@ -441,7 +475,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error {
|
|||||||
}
|
}
|
||||||
// Inspect append-only file store then.
|
// Inspect append-only file store then.
|
||||||
ancientSizes := []*common.StorageSize{&ancientHeadersSize, &ancientBodiesSize, &ancientReceiptsSize, &ancientHashesSize, &ancientTdsSize}
|
ancientSizes := []*common.StorageSize{&ancientHeadersSize, &ancientBodiesSize, &ancientReceiptsSize, &ancientHashesSize, &ancientTdsSize}
|
||||||
for i, category := range []string{freezerHeaderTable, freezerBodiesTable, freezerReceiptTable, freezerHashTable, freezerDifficultyTable} {
|
for i, category := range []string{chainFreezerHeaderTable, chainFreezerBodiesTable, chainFreezerReceiptTable, chainFreezerHashTable, chainFreezerDifficultyTable} {
|
||||||
if size, err := db.AncientSize(category); err == nil {
|
if size, err := db.AncientSize(category); err == nil {
|
||||||
*ancientSizes[i] += common.StorageSize(size)
|
*ancientSizes[i] += common.StorageSize(size)
|
||||||
total += common.StorageSize(size)
|
total += common.StorageSize(size)
|
||||||
|
@ -68,8 +68,6 @@ type Freezer struct {
|
|||||||
frozen uint64 // Number of blocks already frozen
|
frozen uint64 // Number of blocks already frozen
|
||||||
tail uint64 // Number of the first stored item in the freezer
|
tail uint64 // Number of the first stored item in the freezer
|
||||||
|
|
||||||
datadir string // Path of root directory of ancient store
|
|
||||||
|
|
||||||
// This lock synchronizes writers and the truncate operation, as well as
|
// This lock synchronizes writers and the truncate operation, as well as
|
||||||
// the "atomic" (batched) read operations.
|
// the "atomic" (batched) read operations.
|
||||||
writeLock sync.RWMutex
|
writeLock sync.RWMutex
|
||||||
@ -111,7 +109,6 @@ func NewFreezer(datadir string, namespace string, readonly bool, maxTableSize ui
|
|||||||
readonly: readonly,
|
readonly: readonly,
|
||||||
tables: make(map[string]*freezerTable),
|
tables: make(map[string]*freezerTable),
|
||||||
instanceLock: lock,
|
instanceLock: lock,
|
||||||
datadir: datadir,
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// Create the tables.
|
// Create the tables.
|
||||||
@ -429,7 +426,7 @@ func (f *Freezer) MigrateTable(kind string, convert convertLegacyFn) error {
|
|||||||
// Set up new dir for the migrated table, the content of which
|
// Set up new dir for the migrated table, the content of which
|
||||||
// we'll at the end move over to the ancients dir.
|
// we'll at the end move over to the ancients dir.
|
||||||
migrationPath := filepath.Join(ancientsPath, "migration")
|
migrationPath := filepath.Join(ancientsPath, "migration")
|
||||||
newTable, err := NewFreezerTable(migrationPath, kind, table.noCompression, false)
|
newTable, err := newFreezerTable(migrationPath, kind, table.noCompression, false)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
@ -486,11 +483,5 @@ func (f *Freezer) MigrateTable(kind string, convert convertLegacyFn) error {
|
|||||||
if err := os.Remove(migrationPath); err != nil {
|
if err := os.Remove(migrationPath); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// AncientDatadir returns the root directory path of the ancient store.
|
|
||||||
func (f *Freezer) AncientDatadir() (string, error) {
|
|
||||||
return f.datadir, nil
|
|
||||||
}
|
|
||||||
|
@ -123,8 +123,8 @@ type freezerTable struct {
|
|||||||
lock sync.RWMutex // Mutex protecting the data file descriptors
|
lock sync.RWMutex // Mutex protecting the data file descriptors
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewFreezerTable opens the given path as a freezer table.
|
// newFreezerTable opens the given path as a freezer table.
|
||||||
func NewFreezerTable(path, name string, disableSnappy, readonly bool) (*freezerTable, error) {
|
func newFreezerTable(path, name string, disableSnappy, readonly bool) (*freezerTable, error) {
|
||||||
return newTable(path, name, metrics.NilMeter{}, metrics.NilMeter{}, metrics.NilGauge{}, freezerTableSize, disableSnappy, readonly)
|
return newTable(path, name, metrics.NilMeter{}, metrics.NilMeter{}, metrics.NilGauge{}, freezerTableSize, disableSnappy, readonly)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -884,9 +884,7 @@ func (t *freezerTable) Sync() error {
|
|||||||
return t.head.Sync()
|
return t.head.Sync()
|
||||||
}
|
}
|
||||||
|
|
||||||
// DumpIndex is a debug print utility function, mainly for testing. It can also
|
func (t *freezerTable) dumpIndexStdout(start, stop int64) {
|
||||||
// be used to analyse a live freezer table index.
|
|
||||||
func (t *freezerTable) DumpIndex(start, stop int64) {
|
|
||||||
t.dumpIndex(os.Stdout, start, stop)
|
t.dumpIndex(os.Stdout, start, stop)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -902,7 +902,7 @@ func TestSequentialRead(t *testing.T) {
|
|||||||
}
|
}
|
||||||
// Write 15 bytes 30 times
|
// Write 15 bytes 30 times
|
||||||
writeChunks(t, f, 30, 15)
|
writeChunks(t, f, 30, 15)
|
||||||
f.DumpIndex(0, 30)
|
f.dumpIndexStdout(0, 30)
|
||||||
f.Close()
|
f.Close()
|
||||||
}
|
}
|
||||||
{ // Open it, iterate, verify iteration
|
{ // Open it, iterate, verify iteration
|
||||||
|
@ -111,33 +111,6 @@ var (
|
|||||||
preimageHitCounter = metrics.NewRegisteredCounter("db/preimage/hits", nil)
|
preimageHitCounter = metrics.NewRegisteredCounter("db/preimage/hits", nil)
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
|
||||||
// freezerHeaderTable indicates the name of the freezer header table.
|
|
||||||
freezerHeaderTable = "headers"
|
|
||||||
|
|
||||||
// freezerHashTable indicates the name of the freezer canonical hash table.
|
|
||||||
freezerHashTable = "hashes"
|
|
||||||
|
|
||||||
// freezerBodiesTable indicates the name of the freezer block body table.
|
|
||||||
freezerBodiesTable = "bodies"
|
|
||||||
|
|
||||||
// freezerReceiptTable indicates the name of the freezer receipts table.
|
|
||||||
freezerReceiptTable = "receipts"
|
|
||||||
|
|
||||||
// freezerDifficultyTable indicates the name of the freezer total difficulty table.
|
|
||||||
freezerDifficultyTable = "diffs"
|
|
||||||
)
|
|
||||||
|
|
||||||
// FreezerNoSnappy configures whether compression is disabled for the ancient-tables.
|
|
||||||
// Hashes and difficulties don't compress well.
|
|
||||||
var FreezerNoSnappy = map[string]bool{
|
|
||||||
freezerHeaderTable: false,
|
|
||||||
freezerHashTable: true,
|
|
||||||
freezerBodiesTable: false,
|
|
||||||
freezerReceiptTable: false,
|
|
||||||
freezerDifficultyTable: true,
|
|
||||||
}
|
|
||||||
|
|
||||||
// LegacyTxLookupEntry is the legacy TxLookupEntry definition with some unnecessary
|
// LegacyTxLookupEntry is the legacy TxLookupEntry definition with some unnecessary
|
||||||
// fields.
|
// fields.
|
||||||
type LegacyTxLookupEntry struct {
|
type LegacyTxLookupEntry struct {
|
||||||
|
@ -142,7 +142,9 @@ type AncientWriteOp interface {
|
|||||||
|
|
||||||
// AncientStater wraps the Stat method of a backing data store.
|
// AncientStater wraps the Stat method of a backing data store.
|
||||||
type AncientStater interface {
|
type AncientStater interface {
|
||||||
// AncientDatadir returns the root directory path of the ancient store.
|
// AncientDatadir returns the path of root ancient directory. Empty string
|
||||||
|
// will be returned if ancient store is not enabled at all. The returned
|
||||||
|
// path can be used to construct the path of other freezers.
|
||||||
AncientDatadir() (string, error)
|
AncientDatadir() (string, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -172,7 +174,6 @@ type Stater interface {
|
|||||||
type AncientStore interface {
|
type AncientStore interface {
|
||||||
AncientReader
|
AncientReader
|
||||||
AncientWriter
|
AncientWriter
|
||||||
AncientStater
|
|
||||||
io.Closer
|
io.Closer
|
||||||
}
|
}
|
||||||
|
|
||||||
|
22
node/node.go
22
node/node.go
@ -703,7 +703,7 @@ func (n *Node) OpenDatabase(name string, cache, handles int, namespace string, r
|
|||||||
// also attaching a chain freezer to it that moves ancient chain data from the
|
// also attaching a chain freezer to it that moves ancient chain data from the
|
||||||
// database to immutable append-only files. If the node is an ephemeral one, a
|
// database to immutable append-only files. If the node is an ephemeral one, a
|
||||||
// memory database is returned.
|
// memory database is returned.
|
||||||
func (n *Node) OpenDatabaseWithFreezer(name string, cache, handles int, freezer, namespace string, readonly bool) (ethdb.Database, error) {
|
func (n *Node) OpenDatabaseWithFreezer(name string, cache, handles int, ancient string, namespace string, readonly bool) (ethdb.Database, error) {
|
||||||
n.lock.Lock()
|
n.lock.Lock()
|
||||||
defer n.lock.Unlock()
|
defer n.lock.Unlock()
|
||||||
if n.state == closedState {
|
if n.state == closedState {
|
||||||
@ -715,14 +715,7 @@ func (n *Node) OpenDatabaseWithFreezer(name string, cache, handles int, freezer,
|
|||||||
if n.config.DataDir == "" {
|
if n.config.DataDir == "" {
|
||||||
db = rawdb.NewMemoryDatabase()
|
db = rawdb.NewMemoryDatabase()
|
||||||
} else {
|
} else {
|
||||||
root := n.ResolvePath(name)
|
db, err = rawdb.NewLevelDBDatabaseWithFreezer(n.ResolvePath(name), cache, handles, n.ResolveAncient(name, ancient), namespace, readonly)
|
||||||
switch {
|
|
||||||
case freezer == "":
|
|
||||||
freezer = filepath.Join(root, "ancient")
|
|
||||||
case !filepath.IsAbs(freezer):
|
|
||||||
freezer = n.ResolvePath(freezer)
|
|
||||||
}
|
|
||||||
db, err = rawdb.NewLevelDBDatabaseWithFreezer(root, cache, handles, freezer, namespace, readonly)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
if err == nil {
|
if err == nil {
|
||||||
@ -736,6 +729,17 @@ func (n *Node) ResolvePath(x string) string {
|
|||||||
return n.config.ResolvePath(x)
|
return n.config.ResolvePath(x)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ResolveAncient returns the absolute path of the root ancient directory.
|
||||||
|
func (n *Node) ResolveAncient(name string, ancient string) string {
|
||||||
|
switch {
|
||||||
|
case ancient == "":
|
||||||
|
ancient = filepath.Join(n.ResolvePath(name), "ancient")
|
||||||
|
case !filepath.IsAbs(ancient):
|
||||||
|
ancient = n.ResolvePath(ancient)
|
||||||
|
}
|
||||||
|
return ancient
|
||||||
|
}
|
||||||
|
|
||||||
// closeTrackingDB wraps the Close method of a database. When the database is closed by the
|
// closeTrackingDB wraps the Close method of a database. When the database is closed by the
|
||||||
// service, the wrapper removes it from the node's database map. This ensures that Node
|
// service, the wrapper removes it from the node's database map. This ensures that Node
|
||||||
// won't auto-close the database if it is closed by the service that opened it.
|
// won't auto-close the database if it is closed by the service that opened it.
|
||||||
|
Loading…
Reference in New Issue
Block a user