cerc-io/plugeth
12
0
Fork 1
Code Issues 4 Pull Requests Packages Projects Releases Wiki Activity

Merge commit '02eb36afc' into merge/geth-v1.13.12

Browse Source
This commit is contained in:
philip-morlier
2024-02-09 07:15:32 -08:00
parent 5065089c3d 02eb36afc2
commit 7e8ed6b43d
109 changed files with 3054 additions and 412 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/clef/datatypes.md
+1 -1
View File
@@ -75,7 +75,7 @@ Example:
},
{
"type": "Info",
"message": "User should see this aswell"
"message": "User should see this as well"
}
],
"meta": {
cmd/devp2p/internal/ethtest/conn.go
+1 -1
View File
@@ -166,7 +166,7 @@ func (c *Conn) ReadEth() (any, error) {
case eth.TransactionsMsg:
msg = new(eth.TransactionsPacket)
case eth.NewPooledTransactionHashesMsg:
msg = new(eth.NewPooledTransactionHashesPacket68)
msg = new(eth.NewPooledTransactionHashesPacket)
case eth.GetPooledTransactionsMsg:
msg = new(eth.GetPooledTransactionsPacket)
case eth.PooledTransactionsMsg:
cmd/devp2p/internal/ethtest/suite.go
+5 -5
View File
@@ -710,7 +710,7 @@ func (s *Suite) TestNewPooledTxs(t *utesting.T) {
}
// Send announcement.
ann := eth.NewPooledTransactionHashesPacket68{Types: txTypes, Sizes: sizes, Hashes: hashes}
ann := eth.NewPooledTransactionHashesPacket{Types: txTypes, Sizes: sizes, Hashes: hashes}
err = conn.Write(ethProto, eth.NewPooledTransactionHashesMsg, ann)
if err != nil {
t.Fatalf("failed to write to connection: %v", err)
@@ -728,7 +728,7 @@ func (s *Suite) TestNewPooledTxs(t *utesting.T) {
t.Fatalf("unexpected number of txs requested: wanted %d, got %d", len(hashes), len(msg.GetPooledTransactionsRequest))
}
return
case *eth.NewPooledTransactionHashesPacket68:
case *eth.NewPooledTransactionHashesPacket:
continue
case *eth.TransactionsPacket:
continue
@@ -796,12 +796,12 @@ func (s *Suite) TestBlobViolations(t *utesting.T) {
t2 = s.makeBlobTxs(2, 3, 0x2)
)
for _, test := range []struct {
ann eth.NewPooledTransactionHashesPacket68
ann eth.NewPooledTransactionHashesPacket
resp eth.PooledTransactionsResponse
}{
// Invalid tx size.
{
ann: eth.NewPooledTransactionHashesPacket68{
ann: eth.NewPooledTransactionHashesPacket{
Types: []byte{types.BlobTxType, types.BlobTxType},
Sizes: []uint32{uint32(t1[0].Size()), uint32(t1[1].Size() + 10)},
Hashes: []common.Hash{t1[0].Hash(), t1[1].Hash()},
@@ -810,7 +810,7 @@ func (s *Suite) TestBlobViolations(t *utesting.T) {
},
// Wrong tx type.
{
ann: eth.NewPooledTransactionHashesPacket68{
ann: eth.NewPooledTransactionHashesPacket{
Types: []byte{types.DynamicFeeTxType, types.BlobTxType},
Sizes: []uint32{uint32(t2[0].Size()), uint32(t2[1].Size())},
Hashes: []common.Hash{t2[0].Hash(), t2[1].Hash()},
cmd/devp2p/internal/ethtest/transaction.go
+2 -2
View File
@@ -70,7 +70,7 @@ func (s *Suite) sendTxs(txs []*types.Transaction) error {
for _, tx := range *msg {
got[tx.Hash()] = true
}
case *eth.NewPooledTransactionHashesPacket68:
case *eth.NewPooledTransactionHashesPacket:
for _, hash := range msg.Hashes {
got[hash] = true
}
@@ -146,7 +146,7 @@ func (s *Suite) sendInvalidTxs(txs []*types.Transaction) error {
return fmt.Errorf("received bad tx: %s", tx.Hash())
}
}
case *eth.NewPooledTransactionHashesPacket68:
case *eth.NewPooledTransactionHashesPacket:
for _, hash := range msg.Hashes {
if _, ok := invalids[hash]; ok {
return fmt.Errorf("received bad tx: %s", hash)
cmd/era/main.go
+324
View File
@@ -0,0 +1,324 @@
// Copyright 2023 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"encoding/json"
"fmt"
"math/big"
"os"
"path"
"strconv"
"strings"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/internal/era"
"github.com/ethereum/go-ethereum/internal/ethapi"
"github.com/ethereum/go-ethereum/internal/flags"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/trie"
"github.com/urfave/cli/v2"
)
var app = flags.NewApp("go-ethereum era tool")
var (
dirFlag = &cli.StringFlag{
Name: "dir",
Usage: "directory storing all relevant era1 files",
Value: "eras",
}
networkFlag = &cli.StringFlag{
Name: "network",
Usage: "network name associated with era1 files",
Value: "mainnet",
}
eraSizeFlag = &cli.IntFlag{
Name: "size",
Usage: "number of blocks per era",
Value: era.MaxEra1Size,
}
txsFlag = &cli.BoolFlag{
Name: "txs",
Usage: "print full transaction values",
}
)
var (
blockCommand = &cli.Command{
Name: "block",
Usage: "get block data",
ArgsUsage: "<number>",
Action: block,
Flags: []cli.Flag{
txsFlag,
},
}
infoCommand = &cli.Command{
Name: "info",
ArgsUsage: "<epoch>",
Usage: "get epoch information",
Action: info,
}
verifyCommand = &cli.Command{
Name: "verify",
ArgsUsage: "<expected>",
Usage: "verifies each era1 against expected accumulator root",
Action: verify,
}
)
func init() {
app.Commands = []*cli.Command{
blockCommand,
infoCommand,
verifyCommand,
}
app.Flags = []cli.Flag{
dirFlag,
networkFlag,
eraSizeFlag,
}
}
func main() {
if err := app.Run(os.Args); err != nil {
fmt.Fprintf(os.Stderr, "%v\n", err)
os.Exit(1)
}
}
// block prints the specified block from an era1 store.
func block(ctx *cli.Context) error {
num, err := strconv.ParseUint(ctx.Args().First(), 10, 64)
if err != nil {
return fmt.Errorf("invalid block number: %w", err)
}
e, err := open(ctx, num/uint64(ctx.Int(eraSizeFlag.Name)))
if err != nil {
return fmt.Errorf("error opening era1: %w", err)
}
defer e.Close()
// Read block with number.
block, err := e.GetBlockByNumber(num)
if err != nil {
return fmt.Errorf("error reading block %d: %w", num, err)
}
// Convert block to JSON and print.
val := ethapi.RPCMarshalBlock(block, ctx.Bool(txsFlag.Name), ctx.Bool(txsFlag.Name), params.MainnetChainConfig)
b, err := json.MarshalIndent(val, "", " ")
if err != nil {
return fmt.Errorf("error marshaling json: %w", err)
}
fmt.Println(string(b))
return nil
}
// info prints some high-level information about the era1 file.
func info(ctx *cli.Context) error {
epoch, err := strconv.ParseUint(ctx.Args().First(), 10, 64)
if err != nil {
return fmt.Errorf("invalid epoch number: %w", err)
}
e, err := open(ctx, epoch)
if err != nil {
return err
}
defer e.Close()
acc, err := e.Accumulator()
if err != nil {
return fmt.Errorf("error reading accumulator: %w", err)
}
td, err := e.InitialTD()
if err != nil {
return fmt.Errorf("error reading total difficulty: %w", err)
}
info := struct {
Accumulator common.Hash `json:"accumulator"`
TotalDifficulty *big.Int `json:"totalDifficulty"`
StartBlock uint64 `json:"startBlock"`
Count uint64 `json:"count"`
}{
acc, td, e.Start(), e.Count(),
}
b, _ := json.MarshalIndent(info, "", " ")
fmt.Println(string(b))
return nil
}
// open opens an era1 file at a certain epoch.
func open(ctx *cli.Context, epoch uint64) (*era.Era, error) {
var (
dir = ctx.String(dirFlag.Name)
network = ctx.String(networkFlag.Name)
)
entries, err := era.ReadDir(dir, network)
if err != nil {
return nil, fmt.Errorf("error reading era dir: %w", err)
}
if epoch >= uint64(len(entries)) {
return nil, fmt.Errorf("epoch out-of-bounds: last %d, want %d", len(entries)-1, epoch)
}
return era.Open(path.Join(dir, entries[epoch]))
}
// verify checks each era1 file in a directory to ensure it is well-formed and
// that the accumulator matches the expected value.
func verify(ctx *cli.Context) error {
if ctx.Args().Len() != 1 {
return fmt.Errorf("missing accumulators file")
}
roots, err := readHashes(ctx.Args().First())
if err != nil {
return fmt.Errorf("unable to read expected roots file: %w", err)
}
var (
dir = ctx.String(dirFlag.Name)
network = ctx.String(networkFlag.Name)
start = time.Now()
reported = time.Now()
)
entries, err := era.ReadDir(dir, network)
if err != nil {
return fmt.Errorf("error reading %s: %w", dir, err)
}
if len(entries) != len(roots) {
return fmt.Errorf("number of era1 files should match the number of accumulator hashes")
}
// Verify each epoch matches the expected root.
for i, want := range roots {
// Wrap in function so defers don't stack.
err := func() error {
name := entries[i]
e, err := era.Open(path.Join(dir, name))
if err != nil {
return fmt.Errorf("error opening era1 file %s: %w", name, err)
}
defer e.Close()
// Read accumulator and check against expected.
if got, err := e.Accumulator(); err != nil {
return fmt.Errorf("error retrieving accumulator for %s: %w", name, err)
} else if got != want {
return fmt.Errorf("invalid root %s: got %s, want %s", name, got, want)
}
// Recompute accumulator.
if err := checkAccumulator(e); err != nil {
return fmt.Errorf("error verify era1 file %s: %w", name, err)
}
// Give the user some feedback that something is happening.
if time.Since(reported) >= 8*time.Second {
fmt.Printf("Verifying Era1 files \t\t verified=%d,\t elapsed=%s\n", i, common.PrettyDuration(time.Since(start)))
reported = time.Now()
}
return nil
}()
if err != nil {
return err
}
}
return nil
}
// checkAccumulator verifies the accumulator matches the data in the Era.
func checkAccumulator(e *era.Era) error {
var (
err error
want common.Hash
td *big.Int
tds = make([]*big.Int, 0)
hashes = make([]common.Hash, 0)
)
if want, err = e.Accumulator(); err != nil {
return fmt.Errorf("error reading accumulator: %w", err)
}
if td, err = e.InitialTD(); err != nil {
return fmt.Errorf("error reading total difficulty: %w", err)
}
it, err := era.NewIterator(e)
if err != nil {
return fmt.Errorf("error making era iterator: %w", err)
}
// To fully verify an era the following attributes must be checked:
// 1) the block index is constructed correctly
// 2) the tx root matches the value in the block
// 3) the receipts root matches the value in the block
// 4) the starting total difficulty value is correct
// 5) the accumulator is correct by recomputing it locally, which verifies
// the blocks are all correct (via hash)
//
// The attributes 1), 2), and 3) are checked for each block. 4) and 5) require
// accumulation across the entire set and are verified at the end.
for it.Next() {
// 1) next() walks the block index, so we're able to implicitly verify it.
if it.Error() != nil {
return fmt.Errorf("error reading block %d: %w", it.Number(), err)
}
block, receipts, err := it.BlockAndReceipts()
if it.Error() != nil {
return fmt.Errorf("error reading block %d: %w", it.Number(), err)
}
// 2) recompute tx root and verify against header.
tr := types.DeriveSha(block.Transactions(), trie.NewStackTrie(nil))
if tr != block.TxHash() {
return fmt.Errorf("tx root in block %d mismatch: want %s, got %s", block.NumberU64(), block.TxHash(), tr)
}
// 3) recompute receipt root and check value against block.
rr := types.DeriveSha(receipts, trie.NewStackTrie(nil))
if rr != block.ReceiptHash() {
return fmt.Errorf("receipt root in block %d mismatch: want %s, got %s", block.NumberU64(), block.ReceiptHash(), rr)
}
hashes = append(hashes, block.Hash())
td.Add(td, block.Difficulty())
tds = append(tds, new(big.Int).Set(td))
}
// 4+5) Verify accumulator and total difficulty.
got, err := era.ComputeAccumulator(hashes, tds)
if err != nil {
return fmt.Errorf("error computing accumulator: %w", err)
}
if got != want {
return fmt.Errorf("expected accumulator root does not match calculated: got %s, want %s", got, want)
}
return nil
}
// readHashes reads a file of newline-delimited hashes.
func readHashes(f string) ([]common.Hash, error) {
b, err := os.ReadFile(f)
if err != nil {
return nil, fmt.Errorf("unable to open accumulators file")
}
s := strings.Split(string(b), "\n")
// Remove empty last element, if present.
if s[len(s)-1] == "" {
s = s[:len(s)-1]
}
// Convert to hashes.
r := make([]common.Hash, len(s))
for i := range s {
r[i] = common.HexToHash(s[i])
}
return r, nil
}
cmd/geth/chaincmd.go
+119
View File
@@ -35,10 +35,12 @@ import (
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/internal/era"
"github.com/ethereum/go-ethereum/internal/flags"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/node"
"github.com/ethereum/go-ethereum/params"
"github.com/urfave/cli/v2"
)
@@ -122,6 +124,33 @@ Optional second and third arguments control the first and
last block to write. In this mode, the file will be appended
if already existing. If the file ends with .gz, the output will
be gzipped.`,
}
importHistoryCommand = &cli.Command{
Action: importHistory,
Name: "import-history",
Usage: "Import an Era archive",
ArgsUsage: "<dir>",
Flags: flags.Merge([]cli.Flag{
utils.TxLookupLimitFlag,
},
utils.DatabaseFlags,
utils.NetworkFlags,
),
Description: `
The import-history command will import blocks and their corresponding receipts
from Era archives.
`,
}
exportHistoryCommand = &cli.Command{
Action: exportHistory,
Name: "export-history",
Usage: "Export blockchain history to Era archives",
ArgsUsage: "<dir> <first> <last>",
Flags: flags.Merge(utils.DatabaseFlags),
Description: `
The export-history command will export blocks and their corresponding receipts
into Era archives. Eras are typically packaged in steps of 8192 blocks.
`,
}
importPreimagesCommand = &cli.Command{
Action: importPreimages,
@@ -364,7 +393,97 @@ func exportChain(ctx *cli.Context) error {
}
err = utils.ExportAppendChain(chain, fp, uint64(first), uint64(last))
}
if err != nil {
utils.Fatalf("Export error: %v\n", err)
}
fmt.Printf("Export done in %v\n", time.Since(start))
return nil
}
func importHistory(ctx *cli.Context) error {
if ctx.Args().Len() != 1 {
utils.Fatalf("usage: %s", ctx.Command.ArgsUsage)
}
stack, _ := makeConfigNode(ctx)
defer stack.Close()
chain, db := utils.MakeChain(ctx, stack, false)
defer db.Close()
var (
start = time.Now()
dir = ctx.Args().Get(0)
network string
)
// Determine network.
if utils.IsNetworkPreset(ctx) {
switch {
case ctx.Bool(utils.MainnetFlag.Name):
network = "mainnet"
case ctx.Bool(utils.SepoliaFlag.Name):
network = "sepolia"
case ctx.Bool(utils.GoerliFlag.Name):
network = "goerli"
}
} else {
// No network flag set, try to determine network based on files
// present in directory.
var networks []string
for _, n := range params.NetworkNames {
entries, err := era.ReadDir(dir, n)
if err != nil {
return fmt.Errorf("error reading %s: %w", dir, err)
}
if len(entries) > 0 {
networks = append(networks, n)
}
}
if len(networks) == 0 {
return fmt.Errorf("no era1 files found in %s", dir)
}
if len(networks) > 1 {
return fmt.Errorf("multiple networks found, use a network flag to specify desired network")
}
network = networks[0]
}
if err := utils.ImportHistory(chain, db, dir, network); err != nil {
return err
}
fmt.Printf("Import done in %v\n", time.Since(start))
return nil
}
// exportHistory exports chain history in Era archives at a specified
// directory.
func exportHistory(ctx *cli.Context) error {
if ctx.Args().Len() != 3 {
utils.Fatalf("usage: %s", ctx.Command.ArgsUsage)
}
stack, _ := makeConfigNode(ctx)
defer stack.Close()
chain, _ := utils.MakeChain(ctx, stack, true)
start := time.Now()
var (
dir = ctx.Args().Get(0)
first, ferr = strconv.ParseInt(ctx.Args().Get(1), 10, 64)
last, lerr = strconv.ParseInt(ctx.Args().Get(2), 10, 64)
)
if ferr != nil || lerr != nil {
utils.Fatalf("Export error in parsing parameters: block number not an integer\n")
}
if first < 0 || last < 0 {
utils.Fatalf("Export error: block number must be greater than 0\n")
}
if head := chain.CurrentSnapBlock(); uint64(last) > head.Number.Uint64() {
utils.Fatalf("Export error: block number %d larger than head block %d\n", uint64(last), head.Number.Uint64())
}
err := utils.ExportHistory(chain, dir, uint64(first), uint64(last), uint64(era.MaxEra1Size))
if err != nil {
utils.Fatalf("Export error: %v\n", err)
}
cmd/geth/main.go
+2
View File
@@ -215,6 +215,8 @@ func init() {
initCommand,
importCommand,
exportCommand,
importHistoryCommand,
exportHistoryCommand,
importPreimagesCommand,
removedbCommand,
dumpCommand,
cmd/utils/cmd.go
+191
View File
@@ -19,12 +19,15 @@ package utils
import (
"bufio"
"bytes"
"compress/gzip"
"crypto/sha256"
"errors"
"fmt"
"io"
"os"
"os/signal"
"path"
"runtime"
"strings"
"syscall"
@@ -39,8 +42,10 @@ import (
"github.com/ethereum/go-ethereum/eth/ethconfig"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/internal/debug"
"github.com/ethereum/go-ethereum/internal/era"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/node"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rlp"
"github.com/urfave/cli/v2"
)
@@ -228,6 +233,105 @@ func ImportChain(chain *core.BlockChain, fn string) error {
return nil
}
func readList(filename string) ([]string, error) {
b, err := os.ReadFile(filename)
if err != nil {
return nil, err
}
return strings.Split(string(b), "\n"), nil
}
// ImportHistory imports Era1 files containing historical block information,
// starting from genesis.
func ImportHistory(chain *core.BlockChain, db ethdb.Database, dir string, network string) error {
if chain.CurrentSnapBlock().Number.BitLen() != 0 {
return fmt.Errorf("history import only supported when starting from genesis")
}
entries, err := era.ReadDir(dir, network)
if err != nil {
return fmt.Errorf("error reading %s: %w", dir, err)
}
checksums, err := readList(path.Join(dir, "checksums.txt"))
if err != nil {
return fmt.Errorf("unable to read checksums.txt: %w", err)
}
if len(checksums) != len(entries) {
return fmt.Errorf("expected equal number of checksums and entries, have: %d checksums, %d entries", len(checksums), len(entries))
}
var (
start = time.Now()
reported = time.Now()
imported = 0
forker = core.NewForkChoice(chain, nil)
h = sha256.New()
buf = bytes.NewBuffer(nil)
)
for i, filename := range entries {
err := func() error {
f, err := os.Open(path.Join(dir, filename))
if err != nil {
return fmt.Errorf("unable to open era: %w", err)
}
defer f.Close()
// Validate checksum.
if _, err := io.Copy(h, f); err != nil {
return fmt.Errorf("unable to recalculate checksum: %w", err)
}
if have, want := common.BytesToHash(h.Sum(buf.Bytes()[:])).Hex(), checksums[i]; have != want {
return fmt.Errorf("checksum mismatch: have %s, want %s", have, want)
}
h.Reset()
buf.Reset()
// Import all block data from Era1.
e, err := era.From(f)
if err != nil {
return fmt.Errorf("error opening era: %w", err)
}
it, err := era.NewIterator(e)
if err != nil {
return fmt.Errorf("error making era reader: %w", err)
}
for it.Next() {
block, err := it.Block()
if err != nil {
return fmt.Errorf("error reading block %d: %w", it.Number(), err)
}
if block.Number().BitLen() == 0 {
continue // skip genesis
}
receipts, err := it.Receipts()
if err != nil {
return fmt.Errorf("error reading receipts %d: %w", it.Number(), err)
}
if status, err := chain.HeaderChain().InsertHeaderChain([]*types.Header{block.Header()}, start, forker); err != nil {
return fmt.Errorf("error inserting header %d: %w", it.Number(), err)
} else if status != core.CanonStatTy {
return fmt.Errorf("error inserting header %d, not canon: %v", it.Number(), status)
}
if _, err := chain.InsertReceiptChain([]*types.Block{block}, []types.Receipts{receipts}, 2^64-1); err != nil {
return fmt.Errorf("error inserting body %d: %w", it.Number(), err)
}
imported += 1
// Give the user some feedback that something is happening.
if time.Since(reported) >= 8*time.Second {
log.Info("Importing Era files", "head", it.Number(), "imported", imported, "elapsed", common.PrettyDuration(time.Since(start)))
imported = 0
reported = time.Now()
}
}
return nil
}()
if err != nil {
return err
}
}
return nil
}
func missingBlocks(chain *core.BlockChain, blocks []*types.Block) []*types.Block {
head := chain.CurrentBlock()
for i, block := range blocks {
@@ -297,6 +401,93 @@ func ExportAppendChain(blockchain *core.BlockChain, fn string, first uint64, las
return nil
}
// ExportHistory exports blockchain history into the specified directory,
// following the Era format.
func ExportHistory(bc *core.BlockChain, dir string, first, last, step uint64) error {
log.Info("Exporting blockchain history", "dir", dir)
if head := bc.CurrentBlock().Number.Uint64(); head < last {
log.Warn("Last block beyond head, setting last = head", "head", head, "last", last)
last = head
}
network := "unknown"
if name, ok := params.NetworkNames[bc.Config().ChainID.String()]; ok {
network = name
}
if err := os.MkdirAll(dir, os.ModePerm); err != nil {
return fmt.Errorf("error creating output directory: %w", err)
}
var (
start = time.Now()
reported = time.Now()
h = sha256.New()
buf = bytes.NewBuffer(nil)
checksums []string
)
for i := first; i <= last; i += step {
err := func() error {
filename := path.Join(dir, era.Filename(network, int(i/step), common.Hash{}))
f, err := os.Create(filename)
if err != nil {
return fmt.Errorf("could not create era file: %w", err)
}
defer f.Close()
w := era.NewBuilder(f)
for j := uint64(0); j < step && j <= last-i; j++ {
var (
n = i + j
block = bc.GetBlockByNumber(n)
)
if block == nil {
return fmt.Errorf("export failed on #%d: not found", n)
}
receipts := bc.GetReceiptsByHash(block.Hash())
if receipts == nil {
return fmt.Errorf("export failed on #%d: receipts not found", n)
}
td := bc.GetTd(block.Hash(), block.NumberU64())
if td == nil {
return fmt.Errorf("export failed on #%d: total difficulty not found", n)
}
if err := w.Add(block, receipts, td); err != nil {
return err
}
}
root, err := w.Finalize()
if err != nil {
return fmt.Errorf("export failed to finalize %d: %w", step/i, err)
}
// Set correct filename with root.
os.Rename(filename, path.Join(dir, era.Filename(network, int(i/step), root)))
// Compute checksum of entire Era1.
if _, err := f.Seek(0, io.SeekStart); err != nil {
return err
}
if _, err := io.Copy(h, f); err != nil {
return fmt.Errorf("unable to calculate checksum: %w", err)
}
checksums = append(checksums, common.BytesToHash(h.Sum(buf.Bytes()[:])).Hex())
h.Reset()
buf.Reset()
return nil
}()
if err != nil {
return err
}
if time.Since(reported) >= 8*time.Second {
log.Info("Exporting blocks", "exported", i, "elapsed", common.PrettyDuration(time.Since(start)))
reported = time.Now()
}
}
os.WriteFile(path.Join(dir, "checksums.txt"), []byte(strings.Join(checksums, "\n")), os.ModePerm)
log.Info("Exported blockchain to", "dir", dir)
return nil
}
// ImportPreimages imports a batch of exported hash preimages into the database.
// It's a part of the deprecated functionality, should be removed in the future.
func ImportPreimages(db ethdb.Database, fn string) error {
cmd/utils/history_test.go
+184
View File
@@ -0,0 +1,184 @@
// Copyright 2023 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.
package utils
import (
"bytes"
"crypto/sha256"
"io"
"math/big"
"os"
"path"
"strings"
"testing"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/consensus/ethash"
"github.com/ethereum/go-ethereum/core"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/internal/era"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/trie"
)
var (
count uint64 = 128
step uint64 = 16
)
func TestHistoryImportAndExport(t *testing.T) {
var (
key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291")
address = crypto.PubkeyToAddress(key.PublicKey)
genesis = &core.Genesis{
Config: params.TestChainConfig,
Alloc: core.GenesisAlloc{address: {Balance: big.NewInt(1000000000000000000)}},
}
signer = types.LatestSigner(genesis.Config)
)
// Generate chain.
db, blocks, _ := core.GenerateChainWithGenesis(genesis, ethash.NewFaker(), int(count), func(i int, g *core.BlockGen) {
if i == 0 {
return
}
tx, err := types.SignNewTx(key, signer, &types.DynamicFeeTx{
ChainID: genesis.Config.ChainID,
Nonce: uint64(i - 1),
GasTipCap: common.Big0,
GasFeeCap: g.PrevBlock(0).BaseFee(),
Gas: 50000,
To: &common.Address{0xaa},
Value: big.NewInt(int64(i)),
Data: nil,
AccessList: nil,
})
if err != nil {
t.Fatalf("error creating tx: %v", err)
}
g.AddTx(tx)
})
// Initialize BlockChain.
chain, err := core.NewBlockChain(db, nil, genesis, nil, ethash.NewFaker(), vm.Config{}, nil, nil)
if err != nil {
t.Fatalf("unable to initialize chain: %v", err)
}
if _, err := chain.InsertChain(blocks); err != nil {
t.Fatalf("error insterting chain: %v", err)
}
// Make temp directory for era files.
dir, err := os.MkdirTemp("", "history-export-test")
if err != nil {
t.Fatalf("error creating temp test directory: %v", err)
}
defer os.RemoveAll(dir)
// Export history to temp directory.
if err := ExportHistory(chain, dir, 0, count, step); err != nil {
t.Fatalf("error exporting history: %v", err)
}
// Read checksums.
b, err := os.ReadFile(path.Join(dir, "checksums.txt"))
if err != nil {
t.Fatalf("failed to read checksums: %v", err)
}
checksums := strings.Split(string(b), "\n")
// Verify each Era.
entries, _ := era.ReadDir(dir, "mainnet")
for i, filename := range entries {
func() {
f, err := os.Open(path.Join(dir, filename))
if err != nil {
t.Fatalf("error opening era file: %v", err)
}
var (
h = sha256.New()
buf = bytes.NewBuffer(nil)
)
if _, err := io.Copy(h, f); err != nil {
t.Fatalf("unable to recalculate checksum: %v", err)
}
if got, want := common.BytesToHash(h.Sum(buf.Bytes()[:])).Hex(), checksums[i]; got != want {
t.Fatalf("checksum %d does not match: got %s, want %s", i, got, want)
}
e, err := era.From(f)
if err != nil {
t.Fatalf("error opening era: %v", err)
}
defer e.Close()
it, err := era.NewIterator(e)
if err != nil {
t.Fatalf("error making era reader: %v", err)
}
for j := 0; it.Next(); j++ {
n := i*int(step) + j
if it.Error() != nil {
t.Fatalf("error reading block entry %d: %v", n, it.Error())
}
block, receipts, err := it.BlockAndReceipts()
if err != nil {
t.Fatalf("error reading block entry %d: %v", n, err)
}
want := chain.GetBlockByNumber(uint64(n))
if want, got := uint64(n), block.NumberU64(); want != got {
t.Fatalf("blocks out of order: want %d, got %d", want, got)
}
if want.Hash() != block.Hash() {
t.Fatalf("block hash mismatch %d: want %s, got %s", n, want.Hash().Hex(), block.Hash().Hex())
}
if got := types.DeriveSha(block.Transactions(), trie.NewStackTrie(nil)); got != want.TxHash() {
t.Fatalf("tx hash %d mismatch: want %s, got %s", n, want.TxHash(), got)
}
if got := types.CalcUncleHash(block.Uncles()); got != want.UncleHash() {
t.Fatalf("uncle hash %d mismatch: want %s, got %s", n, want.UncleHash(), got)
}
if got := types.DeriveSha(receipts, trie.NewStackTrie(nil)); got != want.ReceiptHash() {
t.Fatalf("receipt root %d mismatch: want %s, got %s", n, want.ReceiptHash(), got)
}
}
}()
}
// Now import Era.
freezer := t.TempDir()
db2, err := rawdb.NewDatabaseWithFreezer(rawdb.NewMemoryDatabase(), freezer, "", false)
if err != nil {
panic(err)
}
t.Cleanup(func() {
db2.Close()
})
genesis.MustCommit(db2, trie.NewDatabase(db, trie.HashDefaults))
imported, err := core.NewBlockChain(db2, nil, genesis, nil, ethash.NewFaker(), vm.Config{}, nil, nil)
if err != nil {
t.Fatalf("unable to initialize chain: %v", err)
}
if err := ImportHistory(imported, db2, dir, "mainnet"); err != nil {
t.Fatalf("failed to import chain: %v", err)
}
if have, want := imported.CurrentHeader(), chain.CurrentHeader(); have.Hash() != want.Hash() {
t.Fatalf("imported chain does not match expected, have (%d, %s) want (%d, %s)", have.Number, have.Hash(), want.Number, want.Hash())
}
}