cerc-io/go-ethereum
18
0
Fork 0
Code Issues 9 Pull Requests Packages Projects Releases 66 Wiki Activity

Patch for concurrent iterator & others (onto v1.11.6) #386

Closed
roysc wants to merge 1565 commits from v1.11.6-statediff-v5 into master
pull from: v1.11.6-statediff-v5
merge into: cerc-io:master
Conversation 0 Commits 1565 Files Changed 1471 +8 -476
9 changed files with 8 additions and 476 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files
Showing only changes of commit 10347c6b54 - Show all commits

22
cmd/geth/config.go
View File

@ -31,7 +31,6 @@ import (
"github.com/ethereum/go-ethereum/accounts/scwallet"
"github.com/ethereum/go-ethereum/accounts/usbwallet"
"github.com/ethereum/go-ethereum/cmd/utils"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/eth/ethconfig"
"github.com/ethereum/go-ethereum/internal/ethapi"
"github.com/ethereum/go-ethereum/internal/flags"
@ -166,26 +165,7 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) {
cfg.Eth.OverrideTerminalTotalDifficultyPassed = &override
}
backend, eth := utils.RegisterEthService(stack, &cfg.Eth)
// Warn users to migrate if they have a legacy freezer format.
if eth != nil && !ctx.IsSet(utils.IgnoreLegacyReceiptsFlag.Name) {
firstIdx := uint64(0)
// Hack to speed up check for mainnet because we know
// the first non-empty block.
ghash := rawdb.ReadCanonicalHash(eth.ChainDb(), 0)
if cfg.Eth.NetworkId == 1 && ghash == params.MainnetGenesisHash {
firstIdx = 46147
}
isLegacy, firstLegacy, err := dbHasLegacyReceipts(eth.ChainDb(), firstIdx)
if err != nil {
log.Error("Failed to check db for legacy receipts", "err", err)
} else if isLegacy {
stack.Close()
log.Error("Database has receipts with a legacy format", "firstLegacy", firstLegacy)
utils.Fatalf("Aborting. Please run `geth db freezer-migrate`.")
}
}
backend, _ := utils.RegisterEthService(stack, &cfg.Eth)
// Configure log filter RPC API.
filterSystem := utils.RegisterFilterAPI(stack, backend, &cfg.Eth)

102
cmd/geth/dbcmd.go
View File

@ -33,7 +33,6 @@ import (
"github.com/ethereum/go-ethereum/console/prompt"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/state/snapshot"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/internal/flags"
@ -69,7 +68,6 @@ Remove blockchain and state databases`,
dbImportCmd,
dbExportCmd,
dbMetadataCmd,
dbMigrateFreezerCmd,
dbCheckStateContentCmd,
},
}
@ -195,17 +193,6 @@ WARNING: This is a low-level operation which may cause database corruption!`,
}, utils.NetworkFlags, utils.DatabasePathFlags),
Description: "Shows metadata about the chain status.",
}
dbMigrateFreezerCmd = &cli.Command{
Action: freezerMigrate,
Name: "freezer-migrate",
Usage: "Migrate legacy parts of the freezer. (WARNING: may take a long time)",
ArgsUsage: "",
Flags: flags.Merge([]cli.Flag{
utils.SyncModeFlag,
}, utils.NetworkFlags, utils.DatabasePathFlags),
Description: `The freezer-migrate command checks your database for receipts in a legacy format and updates those.
WARNING: please back-up the receipt files in your ancients before running this command.`,
}
)
func removeDB(ctx *cli.Context) error {
@ -756,92 +743,3 @@ func showMetaData(ctx *cli.Context) error {
table.Render()
return nil
}
func freezerMigrate(ctx *cli.Context) error {
stack, _ := makeConfigNode(ctx)
defer stack.Close()
db := utils.MakeChainDatabase(ctx, stack, false)
defer db.Close()
// Check first block for legacy receipt format
numAncients, err := db.Ancients()
if err != nil {
return err
}
if numAncients < 1 {
log.Info("No receipts in freezer to migrate")
return nil
}
isFirstLegacy, firstIdx, err := dbHasLegacyReceipts(db, 0)
if err != nil {
return err
}
if !isFirstLegacy {
log.Info("No legacy receipts to migrate")
return nil
}
log.Info("Starting migration", "ancients", numAncients, "firstLegacy", firstIdx)
start := time.Now()
if err := db.MigrateTable("receipts", types.ConvertLegacyStoredReceipts); err != nil {
return err
}
if err := db.Close(); err != nil {
return err
}
log.Info("Migration finished", "duration", time.Since(start))
return nil
}
// dbHasLegacyReceipts checks freezer entries for legacy receipts. It stops at the first
// non-empty receipt and checks its format. The index of this first non-empty element is
// the second return parameter.
func dbHasLegacyReceipts(db ethdb.Database, firstIdx uint64) (bool, uint64, error) {
// Check first block for legacy receipt format
numAncients, err := db.Ancients()
if err != nil {
return false, 0, err
}
if numAncients < 1 {
return false, 0, nil
}
if firstIdx >= numAncients {
return false, firstIdx, nil
}
var (
legacy bool
blob []byte
emptyRLPList = []byte{192}
)
// Find first block with non-empty receipt, only if
// the index is not already provided.
if firstIdx == 0 {
for i := uint64(0); i < numAncients; i++ {
blob, err = db.Ancient("receipts", i)
if err != nil {
return false, 0, err
}
if len(blob) == 0 {
continue
}
if !bytes.Equal(blob, emptyRLPList) {
firstIdx = i
break
}
}
}
first, err := db.Ancient("receipts", firstIdx)
if err != nil {
return false, 0, err
}
// We looped over all receipts and they were all empty
if bytes.Equal(first, emptyRLPList) {
return false, 0, nil
}
// Is first non-empty receipt legacy?
legacy, err = types.IsLegacyStoredReceipts(first)
return legacy, firstIdx, err
}

1
cmd/geth/main.go
View File

@ -149,7 +149,6 @@ var (
utils.GpoMaxGasPriceFlag,
utils.GpoIgnoreGasPriceFlag,
utils.MinerNotifyFullFlag,
utils.IgnoreLegacyReceiptsFlag,
configFileFlag,
}, utils.NetworkFlags, utils.DatabasePathFlags)

5
cmd/utils/flags.go
View File

@ -669,11 +669,6 @@ var (
}
// MISC settings
IgnoreLegacyReceiptsFlag = &cli.BoolFlag{
Name: "ignore-legacy-receipts",
Usage: "Geth will start up even if there are legacy receipts in freezer",
Category: flags.MiscCategory,
}
SyncTargetFlag = &cli.PathFlag{
Name: "synctarget",
Usage: `File for containing the hex-encoded block-rlp as sync target(dev feature)`,

28
core/rawdb/accessors_chain.go
View File

@ -669,10 +669,11 @@ func DeleteReceipts(db ethdb.KeyValueWriter, hash common.Hash, number uint64) {
// storedReceiptRLP is the storage encoding of a receipt.
// Re-definition in core/types/receipt.go.
// TODO: Re-use the existing definition.
type storedReceiptRLP struct {
PostStateOrStatus []byte
CumulativeGasUsed uint64
Logs []*types.LogForStorage
Logs []*types.Log
}
// ReceiptLogs is a barebone version of ReceiptForStorage which only keeps
@ -688,10 +689,7 @@ func (r *receiptLogs) DecodeRLP(s *rlp.Stream) error {
if err := s.Decode(&stored); err != nil {
return err
}
r.Logs = make([]*types.Log, len(stored.Logs))
for i, log := range stored.Logs {
r.Logs[i] = (*types.Log)(log)
}
r.Logs = stored.Logs
return nil
}
@ -727,11 +725,6 @@ func ReadLogs(db ethdb.Reader, hash common.Hash, number uint64, config *params.C
}
receipts := []*receiptLogs{}
if err := rlp.DecodeBytes(data, &receipts); err != nil {
// Receipts might be in the legacy format, try decoding that.
// TODO: to be removed after users migrated
if logs := readLegacyLogs(db, hash, number, config); logs != nil {
return logs
}
log.Error("Invalid receipt array RLP", "hash", hash, "err", err)
return nil
}
@ -752,21 +745,6 @@ func ReadLogs(db ethdb.Reader, hash common.Hash, number uint64, config *params.C
return logs
}
// readLegacyLogs is a temporary workaround for when trying to read logs
// from a block which has its receipt stored in the legacy format. It'll
// be removed after users have migrated their freezer databases.
func readLegacyLogs(db ethdb.Reader, hash common.Hash, number uint64, config *params.ChainConfig) [][]*types.Log {
receipts := ReadReceipts(db, hash, number, config)
if receipts == nil {
return nil
}
logs := make([][]*types.Log, len(receipts))
for i, receipt := range receipts {
logs[i] = receipt.Logs
}
return logs
}
// ReadBlock retrieves an entire block corresponding to the hash, assembling it
// back from the stored header and body. If either the header or body could not
// be retrieved nil is returned.

53
core/types/legacy.go
View File

@ -1,53 +0,0 @@
// Copyright 2022 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package types
import (
"errors"
"github.com/ethereum/go-ethereum/rlp"
)
// IsLegacyStoredReceipts tries to parse the RLP-encoded blob
// first as an array of v3 stored receipt, then v4 stored receipt and
// returns true if successful.
func IsLegacyStoredReceipts(raw []byte) (bool, error) {
var v3 []v3StoredReceiptRLP
if err := rlp.DecodeBytes(raw, &v3); err == nil {
return true, nil
}
var v4 []v4StoredReceiptRLP
if err := rlp.DecodeBytes(raw, &v4); err == nil {
return true, nil
}
var v5 []storedReceiptRLP
// Check to see valid fresh stored receipt
if err := rlp.DecodeBytes(raw, &v5); err == nil {
return false, nil
}
return false, errors.New("value is not a valid receipt encoding")
}
// ConvertLegacyStoredReceipts takes the RLP encoding of an array of legacy
// stored receipts and returns a fresh RLP-encoded stored receipt.
func ConvertLegacyStoredReceipts(raw []byte) ([]byte, error) {
var receipts []ReceiptForStorage
if err := rlp.DecodeBytes(raw, &receipts); err != nil {
return nil, err
}
return rlp.EncodeToBytes(&receipts)
}

54
core/types/log.go
View File

@ -64,24 +64,13 @@ type logMarshaling struct {
//go:generate go run ../../rlp/rlpgen -type rlpLog -out gen_log_rlp.go
// rlpLog is used to RLP-encode both the consensus and storage formats.
type rlpLog struct {
Address common.Address
Topics []common.Hash
Data []byte
}
// legacyRlpStorageLog is the previous storage encoding of a log including some redundant fields.
type legacyRlpStorageLog struct {
Address common.Address
Topics []common.Hash
Data []byte
BlockNumber uint64
TxHash common.Hash
TxIndex uint
BlockHash common.Hash
Index uint
}
// EncodeRLP implements rlp.Encoder.
func (l *Log) EncodeRLP(w io.Writer) error {
rl := rlpLog{Address: l.Address, Topics: l.Topics, Data: l.Data}
@ -97,44 +86,3 @@ func (l *Log) DecodeRLP(s *rlp.Stream) error {
}
return err
}
// LogForStorage is a wrapper around a Log that handles
// backward compatibility with prior storage formats.
type LogForStorage Log
// EncodeRLP implements rlp.Encoder.
func (l *LogForStorage) EncodeRLP(w io.Writer) error {
rl := rlpLog{Address: l.Address, Topics: l.Topics, Data: l.Data}
return rlp.Encode(w, &rl)
}
// DecodeRLP implements rlp.Decoder.
//
// Note some redundant fields(e.g. block number, tx hash etc) will be assembled later.
func (l *LogForStorage) DecodeRLP(s *rlp.Stream) error {
blob, err := s.Raw()
if err != nil {
return err
}
var dec rlpLog
err = rlp.DecodeBytes(blob, &dec)
if err == nil {
*l = LogForStorage{
Address: dec.Address,
Topics: dec.Topics,
Data: dec.Data,
}
} else {
// Try to decode log with previous definition.
var dec legacyRlpStorageLog
err = rlp.DecodeBytes(blob, &dec)
if err == nil {
*l = LogForStorage{
Address: dec.Address,
Topics: dec.Topics,
Data: dec.Data,
}
}
}
return err
}

89
core/types/receipt.go
View File

@ -93,28 +93,7 @@ type receiptRLP struct {
type storedReceiptRLP struct {
PostStateOrStatus []byte
CumulativeGasUsed uint64
Logs []*LogForStorage
}
// v4StoredReceiptRLP is the storage encoding of a receipt used in database version 4.
type v4StoredReceiptRLP struct {
PostStateOrStatus []byte
CumulativeGasUsed uint64
TxHash common.Hash
ContractAddress common.Address
Logs []*LogForStorage
GasUsed uint64
}
// v3StoredReceiptRLP is the original storage encoding of a receipt including some unnecessary fields.
type v3StoredReceiptRLP struct {
PostStateOrStatus []byte
CumulativeGasUsed uint64
Bloom Bloom
TxHash common.Hash
ContractAddress common.Address
Logs []*LogForStorage
GasUsed uint64
Logs []*Log
}
// NewReceipt creates a barebone transaction receipt, copying the init fields.
@ -292,82 +271,20 @@ func (r *ReceiptForStorage) EncodeRLP(_w io.Writer) error {
// DecodeRLP implements rlp.Decoder, and loads both consensus and implementation
// fields of a receipt from an RLP stream.
func (r *ReceiptForStorage) DecodeRLP(s *rlp.Stream) error {
// Retrieve the entire receipt blob as we need to try multiple decoders
blob, err := s.Raw()
if err != nil {
return err
}
// Try decoding from the newest format for future proofness, then the older one
// for old nodes that just upgraded. V4 was an intermediate unreleased format so
// we do need to decode it, but it's not common (try last).
if err := decodeStoredReceiptRLP(r, blob); err == nil {
return nil
}
if err := decodeV3StoredReceiptRLP(r, blob); err == nil {
return nil
}
return decodeV4StoredReceiptRLP(r, blob)
}
func decodeStoredReceiptRLP(r *ReceiptForStorage, blob []byte) error {
var stored storedReceiptRLP
if err := rlp.DecodeBytes(blob, &stored); err != nil {
if err := s.Decode(&stored); err != nil {
return err
}
if err := (*Receipt)(r).setStatus(stored.PostStateOrStatus); err != nil {
return err
}
r.CumulativeGasUsed = stored.CumulativeGasUsed
r.Logs = make([]*Log, len(stored.Logs))
for i, log := range stored.Logs {
r.Logs[i] = (*Log)(log)
}
r.Logs = stored.Logs
r.Bloom = CreateBloom(Receipts{(*Receipt)(r)})
return nil
}
func decodeV4StoredReceiptRLP(r *ReceiptForStorage, blob []byte) error {
var stored v4StoredReceiptRLP
if err := rlp.DecodeBytes(blob, &stored); err != nil {
return err
}
if err := (*Receipt)(r).setStatus(stored.PostStateOrStatus); err != nil {
return err
}
r.CumulativeGasUsed = stored.CumulativeGasUsed
r.TxHash = stored.TxHash
r.ContractAddress = stored.ContractAddress
r.GasUsed = stored.GasUsed
r.Logs = make([]*Log, len(stored.Logs))
for i, log := range stored.Logs {
r.Logs[i] = (*Log)(log)
}
r.Bloom = CreateBloom(Receipts{(*Receipt)(r)})
return nil
}
func decodeV3StoredReceiptRLP(r *ReceiptForStorage, blob []byte) error {
var stored v3StoredReceiptRLP
if err := rlp.DecodeBytes(blob, &stored); err != nil {
return err
}
if err := (*Receipt)(r).setStatus(stored.PostStateOrStatus); err != nil {
return err
}
r.CumulativeGasUsed = stored.CumulativeGasUsed
r.Bloom = stored.Bloom
r.TxHash = stored.TxHash
r.ContractAddress = stored.ContractAddress
r.GasUsed = stored.GasUsed
r.Logs = make([]*Log, len(stored.Logs))
for i, log := range stored.Logs {
r.Logs[i] = (*Log)(log)
}
return nil
}
// Receipts implements DerivableList for receipts.
type Receipts []*Receipt

130
core/types/receipt_test.go
View File

@ -91,136 +91,6 @@ func TestDecodeEmptyTypedReceipt(t *testing.T) {
}
}
func TestLegacyReceiptDecoding(t *testing.T) {
tests := []struct {
name string
encode func(*Receipt) ([]byte, error)
}{
{
"ReceiptForStorage",
encodeAsReceiptForStorage,
},
{
"StoredReceiptRLP",
encodeAsStoredReceiptRLP,
},
{
"V4StoredReceiptRLP",
encodeAsV4StoredReceiptRLP,
},
{
"V3StoredReceiptRLP",
encodeAsV3StoredReceiptRLP,
},
}
tx := NewTransaction(1, common.HexToAddress("0x1"), big.NewInt(1), 1, big.NewInt(1), nil)
receipt := &Receipt{
Status: ReceiptStatusFailed,
CumulativeGasUsed: 1,
Logs: []*Log{
{
Address: common.BytesToAddress([]byte{0x11}),
Topics: []common.Hash{common.HexToHash("dead"), common.HexToHash("beef")},
Data: []byte{0x01, 0x00, 0xff},
},
{
Address: common.BytesToAddress([]byte{0x01, 0x11}),
Topics: []common.Hash{common.HexToHash("dead"), common.HexToHash("beef")},
Data: []byte{0x01, 0x00, 0xff},
},
},
TxHash: tx.Hash(),
ContractAddress: common.BytesToAddress([]byte{0x01, 0x11, 0x11}),
GasUsed: 111111,
}
receipt.Bloom = CreateBloom(Receipts{receipt})
for _, tc := range tests {
t.Run(tc.name, func(t *testing.T) {
enc, err := tc.encode(receipt)
if err != nil {
t.Fatalf("Error encoding receipt: %v", err)
}
var dec ReceiptForStorage
if err := rlp.DecodeBytes(enc, &dec); err != nil {
t.Fatalf("Error decoding RLP receipt: %v", err)
}
// Check whether all consensus fields are correct.
if dec.Status != receipt.Status {
t.Fatalf("Receipt status mismatch, want %v, have %v", receipt.Status, dec.Status)
}
if dec.CumulativeGasUsed != receipt.CumulativeGasUsed {
t.Fatalf("Receipt CumulativeGasUsed mismatch, want %v, have %v", receipt.CumulativeGasUsed, dec.CumulativeGasUsed)
}
if dec.Bloom != receipt.Bloom {
t.Fatalf("Bloom data mismatch, want %v, have %v", receipt.Bloom, dec.Bloom)
}
if len(dec.Logs) != len(receipt.Logs) {
t.Fatalf("Receipt log number mismatch, want %v, have %v", len(receipt.Logs), len(dec.Logs))
}
for i := 0; i < len(dec.Logs); i++ {
if dec.Logs[i].Address != receipt.Logs[i].Address {
t.Fatalf("Receipt log %d address mismatch, want %v, have %v", i, receipt.Logs[i].Address, dec.Logs[i].Address)
}
if !reflect.DeepEqual(dec.Logs[i].Topics, receipt.Logs[i].Topics) {
t.Fatalf("Receipt log %d topics mismatch, want %v, have %v", i, receipt.Logs[i].Topics, dec.Logs[i].Topics)
}
if !bytes.Equal(dec.Logs[i].Data, receipt.Logs[i].Data) {
t.Fatalf("Receipt log %d data mismatch, want %v, have %v", i, receipt.Logs[i].Data, dec.Logs[i].Data)
}
}
})
}
}
func encodeAsReceiptForStorage(want *Receipt) ([]byte, error) {
return rlp.EncodeToBytes((*ReceiptForStorage)(want))
}
func encodeAsStoredReceiptRLP(want *Receipt) ([]byte, error) {
stored := &storedReceiptRLP{
PostStateOrStatus: want.statusEncoding(),
CumulativeGasUsed: want.CumulativeGasUsed,
Logs: make([]*LogForStorage, len(want.Logs)),
}
for i, log := range want.Logs {
stored.Logs[i] = (*LogForStorage)(log)
}
return rlp.EncodeToBytes(stored)
}
func encodeAsV4StoredReceiptRLP(want *Receipt) ([]byte, error) {
stored := &v4StoredReceiptRLP{
PostStateOrStatus: want.statusEncoding(),
CumulativeGasUsed: want.CumulativeGasUsed,
TxHash: want.TxHash,
ContractAddress: want.ContractAddress,
Logs: make([]*LogForStorage, len(want.Logs)),
GasUsed: want.GasUsed,
}
for i, log := range want.Logs {
stored.Logs[i] = (*LogForStorage)(log)
}
return rlp.EncodeToBytes(stored)
}
func encodeAsV3StoredReceiptRLP(want *Receipt) ([]byte, error) {
stored := &v3StoredReceiptRLP{
PostStateOrStatus: want.statusEncoding(),
CumulativeGasUsed: want.CumulativeGasUsed,
Bloom: want.Bloom,
TxHash: want.TxHash,
ContractAddress: want.ContractAddress,
Logs: make([]*LogForStorage, len(want.Logs)),
GasUsed: want.GasUsed,
}
for i, log := range want.Logs {
stored.Logs[i] = (*LogForStorage)(log)
}
return rlp.EncodeToBytes(stored)
}
// Tests that receipt data can be correctly derived from the contextual infos
func TestDeriveFields(t *testing.T) {
// Create a few transactions to have receipts for