From 54007f5e0aeff30e5a2cb3276cb9bb2eaa55a841 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 27 Jul 2022 13:56:40 +0200 Subject: [PATCH 01/59] params: begin v1.10.22 release cycle --- params/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/params/version.go b/params/version.go index c1dd05955..258be5d5d 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 10 // Minor version component of the current release - VersionPatch = 21 // Patch version component of the current release - VersionMeta = "stable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 10 // Minor version component of the current release + VersionPatch = 22 // Patch version component of the current release + VersionMeta = "unstable" // Version metadata to append to the version string ) // Version holds the textual version string. From 9d76a9b94f1d4f8b8a9c212151d96d6a13cb6ead Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Thu, 28 Jul 2022 02:37:04 +0800 Subject: [PATCH 02/59] core, trie, eth, cmd: rework preimage store (#25287) * core, trie, eth, cmd: rework preimage store * trie: address comment --- cmd/evm/internal/t8ntool/execution.go | 2 +- core/state/state_test.go | 3 +- eth/api_test.go | 3 +- trie/database.go | 91 +++++-------------------- trie/preimages.go | 95 +++++++++++++++++++++++++++ trie/secure_trie.go | 17 +++-- 6 files changed, 127 insertions(+), 84 deletions(-) create mode 100644 trie/preimages.go diff --git a/cmd/evm/internal/t8ntool/execution.go b/cmd/evm/internal/t8ntool/execution.go index 28fe77d2d..77f6ec371 100644 --- a/cmd/evm/internal/t8ntool/execution.go +++ b/cmd/evm/internal/t8ntool/execution.go @@ -268,7 +268,7 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, } func MakePreState(db ethdb.Database, accounts core.GenesisAlloc) *state.StateDB { - sdb := state.NewDatabase(db) + sdb := state.NewDatabaseWithConfig(db, &trie.Config{Preimages: true}) statedb, _ := state.New(common.Hash{}, sdb, nil) for addr, a := range accounts { statedb.SetCode(addr, a.Code) diff --git a/core/state/state_test.go b/core/state/state_test.go index 0a55d7781..b6b46e446 100644 --- a/core/state/state_test.go +++ b/core/state/state_test.go @@ -25,6 +25,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/trie" ) type stateTest struct { @@ -40,7 +41,7 @@ func newStateTest() *stateTest { func TestDump(t *testing.T) { db := rawdb.NewMemoryDatabase() - sdb, _ := New(common.Hash{}, NewDatabaseWithConfig(db, nil), nil) + sdb, _ := New(common.Hash{}, NewDatabaseWithConfig(db, &trie.Config{Preimages: true}), nil) s := &stateTest{db: db, state: sdb} // generate a few entries diff --git a/eth/api_test.go b/eth/api_test.go index aae04eaa9..250591c10 100644 --- a/eth/api_test.go +++ b/eth/api_test.go @@ -29,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/trie" ) var dumper = spew.ConfigState{Indent: " "} @@ -66,7 +67,7 @@ func TestAccountRange(t *testing.T) { t.Parallel() var ( - statedb = state.NewDatabaseWithConfig(rawdb.NewMemoryDatabase(), nil) + statedb = state.NewDatabaseWithConfig(rawdb.NewMemoryDatabase(), &trie.Config{Preimages: true}) state, _ = state.New(common.Hash{}, statedb, nil) addrs = [AccountRangeMaxResults * 2]common.Address{} m = map[common.Address]bool{} diff --git a/trie/database.go b/trie/database.go index 2df2e859d..8e1788a21 100644 --- a/trie/database.go +++ b/trie/database.go @@ -74,8 +74,6 @@ type Database struct { oldest common.Hash // Oldest tracked node, flush-list head newest common.Hash // Newest tracked node, flush-list tail - preimages map[common.Hash][]byte // Preimages of nodes from the secure trie - gctime time.Duration // Time spent on garbage collection since last commit gcnodes uint64 // Nodes garbage collected since last commit gcsize common.StorageSize // Data storage garbage collected since last commit @@ -84,9 +82,9 @@ type Database struct { flushnodes uint64 // Nodes flushed since last commit flushsize common.StorageSize // Data storage flushed since last commit - dirtiesSize common.StorageSize // Storage size of the dirty node cache (exc. metadata) - childrenSize common.StorageSize // Storage size of the external children tracking - preimagesSize common.StorageSize // Storage size of the preimages cache + dirtiesSize common.StorageSize // Storage size of the dirty node cache (exc. metadata) + childrenSize common.StorageSize // Storage size of the external children tracking + preimages *preimageStore // The store for caching preimages lock sync.RWMutex } @@ -287,15 +285,17 @@ func NewDatabaseWithConfig(diskdb ethdb.KeyValueStore, config *Config) *Database cleans = fastcache.LoadFromFileOrNew(config.Journal, config.Cache*1024*1024) } } + var preimage *preimageStore + if config != nil && config.Preimages { + preimage = newPreimageStore(diskdb) + } db := &Database{ diskdb: diskdb, cleans: cleans, dirties: map[common.Hash]*cachedNode{{}: { children: make(map[common.Hash]uint16), }}, - } - if config == nil || config.Preimages { // TODO(karalabe): Flip to default off in the future - db.preimages = make(map[common.Hash][]byte) + preimages: preimage, } return db } @@ -341,24 +341,6 @@ func (db *Database) insert(hash common.Hash, size int, node node) { db.dirtiesSize += common.StorageSize(common.HashLength + entry.size) } -// insertPreimage writes a new trie node pre-image to the memory database if it's -// yet unknown. The method will NOT make a copy of the slice, -// only use if the preimage will NOT be changed later on. -// -// Note, this method assumes that the database's lock is held! -func (db *Database) insertPreimage(hash common.Hash, preimage []byte) { - // Short circuit if preimage collection is disabled - if db.preimages == nil { - return - } - // Track the preimage if a yet unknown one - if _, ok := db.preimages[hash]; ok { - return - } - db.preimages[hash] = preimage - db.preimagesSize += common.StorageSize(common.HashLength + len(preimage)) -} - // node retrieves a cached trie node from memory, or returns nil if none can be // found in the memory cache. func (db *Database) node(hash common.Hash) node { @@ -435,24 +417,6 @@ func (db *Database) Node(hash common.Hash) ([]byte, error) { return nil, errors.New("not found") } -// preimage retrieves a cached trie node pre-image from memory. If it cannot be -// found cached, the method queries the persistent database for the content. -func (db *Database) preimage(hash common.Hash) []byte { - // Short circuit if preimage collection is disabled - if db.preimages == nil { - return nil - } - // Retrieve the node from cache if available - db.lock.RLock() - preimage := db.preimages[hash] - db.lock.RUnlock() - - if preimage != nil { - return preimage - } - return rawdb.ReadPreimage(db.diskdb, hash) -} - // Nodes retrieves the hashes of all the nodes cached within the memory database. // This method is extremely expensive and should only be used to validate internal // states in test code. @@ -597,19 +561,8 @@ func (db *Database) Cap(limit common.StorageSize) error { // If the preimage cache got large enough, push to disk. If it's still small // leave for later to deduplicate writes. - flushPreimages := db.preimagesSize > 4*1024*1024 - if flushPreimages { - if db.preimages == nil { - log.Error("Attempted to write preimages whilst disabled") - } else { - rawdb.WritePreimages(batch, db.preimages) - if batch.ValueSize() > ethdb.IdealBatchSize { - if err := batch.Write(); err != nil { - return err - } - batch.Reset() - } - } + if db.preimages != nil { + db.preimages.commit(false) } // Keep committing nodes from the flush-list until we're below allowance oldest := db.oldest @@ -644,13 +597,6 @@ func (db *Database) Cap(limit common.StorageSize) error { db.lock.Lock() defer db.lock.Unlock() - if flushPreimages { - if db.preimages == nil { - log.Error("Attempted to reset preimage cache whilst disabled") - } else { - db.preimages, db.preimagesSize = make(map[common.Hash][]byte), 0 - } - } for db.oldest != oldest { node := db.dirties[db.oldest] delete(db.dirties, db.oldest) @@ -694,13 +640,7 @@ func (db *Database) Commit(node common.Hash, report bool, callback func(common.H // Move all of the accumulated preimages into a write batch if db.preimages != nil { - rawdb.WritePreimages(batch, db.preimages) - // Since we're going to replay trie node writes into the clean cache, flush out - // any batched pre-images before continuing. - if err := batch.Write(); err != nil { - return err - } - batch.Reset() + db.preimages.commit(true) } // Move the trie itself into the batch, flushing if enough data is accumulated nodes, storage := len(db.dirties), db.dirtiesSize @@ -723,9 +663,6 @@ func (db *Database) Commit(node common.Hash, report bool, callback func(common.H batch.Reset() // Reset the storage counters and bumped metrics - if db.preimages != nil { - db.preimages, db.preimagesSize = make(map[common.Hash][]byte), 0 - } memcacheCommitTimeTimer.Update(time.Since(start)) memcacheCommitSizeMeter.Mark(int64(storage - db.dirtiesSize)) memcacheCommitNodesMeter.Mark(int64(nodes - len(db.dirties))) @@ -837,7 +774,11 @@ func (db *Database) Size() (common.StorageSize, common.StorageSize) { // counted. var metadataSize = common.StorageSize((len(db.dirties) - 1) * cachedNodeSize) var metarootRefs = common.StorageSize(len(db.dirties[common.Hash{}].children) * (common.HashLength + 2)) - return db.dirtiesSize + db.childrenSize + metadataSize - metarootRefs, db.preimagesSize + var preimageSize common.StorageSize + if db.preimages != nil { + preimageSize = db.preimages.size() + } + return db.dirtiesSize + db.childrenSize + metadataSize - metarootRefs, preimageSize } // saveCache saves clean state cache to given directory path diff --git a/trie/preimages.go b/trie/preimages.go new file mode 100644 index 000000000..66f34117c --- /dev/null +++ b/trie/preimages.go @@ -0,0 +1,95 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package trie + +import ( + "sync" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/ethdb" +) + +// preimageStore is the store for caching preimages of node key. +type preimageStore struct { + lock sync.RWMutex + disk ethdb.KeyValueStore + preimages map[common.Hash][]byte // Preimages of nodes from the secure trie + preimagesSize common.StorageSize // Storage size of the preimages cache +} + +// newPreimageStore initializes the store for caching preimages. +func newPreimageStore(disk ethdb.KeyValueStore) *preimageStore { + return &preimageStore{ + disk: disk, + preimages: make(map[common.Hash][]byte), + } +} + +// insertPreimage writes a new trie node pre-image to the memory database if it's +// yet unknown. The method will NOT make a copy of the slice, only use if the +// preimage will NOT be changed later on. +func (store *preimageStore) insertPreimage(preimages map[common.Hash][]byte) { + store.lock.Lock() + defer store.lock.Unlock() + + for hash, preimage := range preimages { + if _, ok := store.preimages[hash]; ok { + continue + } + store.preimages[hash] = preimage + store.preimagesSize += common.StorageSize(common.HashLength + len(preimage)) + } +} + +// preimage retrieves a cached trie node pre-image from memory. If it cannot be +// found cached, the method queries the persistent database for the content. +func (store *preimageStore) preimage(hash common.Hash) []byte { + store.lock.RLock() + preimage := store.preimages[hash] + store.lock.RUnlock() + + if preimage != nil { + return preimage + } + return rawdb.ReadPreimage(store.disk, hash) +} + +// commit flushes the cached preimages into the disk. +func (store *preimageStore) commit(force bool) error { + store.lock.Lock() + defer store.lock.Unlock() + + if store.preimagesSize <= 4*1024*1024 && !force { + return nil + } + batch := store.disk.NewBatch() + rawdb.WritePreimages(batch, store.preimages) + if err := batch.Write(); err != nil { + return err + } + store.preimages, store.preimagesSize = make(map[common.Hash][]byte), 0 + return nil +} + +// size returns the current storage size of accumulated preimages. +func (store *preimageStore) size() common.StorageSize { + store.lock.RLock() + defer store.lock.RUnlock() + + return store.preimagesSize +} diff --git a/trie/secure_trie.go b/trie/secure_trie.go index 6a5cc89c9..967194df9 100644 --- a/trie/secure_trie.go +++ b/trie/secure_trie.go @@ -37,6 +37,7 @@ import ( // SecureTrie is not safe for concurrent use. type SecureTrie struct { trie Trie + preimages *preimageStore hashKeyBuf [common.HashLength]byte secKeyCache map[string][]byte secKeyCacheOwner *SecureTrie // Pointer to self, replace the key cache on mismatch @@ -61,7 +62,7 @@ func NewSecure(owner common.Hash, root common.Hash, db *Database) (*SecureTrie, if err != nil { return nil, err } - return &SecureTrie{trie: *trie}, nil + return &SecureTrie{trie: *trie, preimages: db.preimages}, nil } // Get returns the value for key stored in the trie. @@ -153,7 +154,10 @@ func (t *SecureTrie) GetKey(shaKey []byte) []byte { if key, ok := t.getSecKeyCache()[string(shaKey)]; ok { return key } - return t.trie.db.preimage(common.BytesToHash(shaKey)) + if t.preimages == nil { + return nil + } + return t.preimages.preimage(common.BytesToHash(shaKey)) } // Commit writes all nodes and the secure hash pre-images to the trie's database. @@ -164,12 +168,12 @@ func (t *SecureTrie) GetKey(shaKey []byte) []byte { func (t *SecureTrie) Commit(onleaf LeafCallback) (common.Hash, int, error) { // Write all the pre-images to the actual disk database if len(t.getSecKeyCache()) > 0 { - if t.trie.db.preimages != nil { // Ugly direct check but avoids the below write lock - t.trie.db.lock.Lock() + if t.preimages != nil { + preimages := make(map[common.Hash][]byte) for hk, key := range t.secKeyCache { - t.trie.db.insertPreimage(common.BytesToHash([]byte(hk)), key) + preimages[common.BytesToHash([]byte(hk))] = key } - t.trie.db.lock.Unlock() + t.preimages.insertPreimage(preimages) } t.secKeyCache = make(map[string][]byte) } @@ -187,6 +191,7 @@ func (t *SecureTrie) Hash() common.Hash { func (t *SecureTrie) Copy() *SecureTrie { return &SecureTrie{ trie: *t.trie.Copy(), + preimages: t.preimages, secKeyCache: t.secKeyCache, } } From ad7106dfc4c7affd5e64369c4ec266b51ceae224 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Thu, 28 Jul 2022 08:46:13 +0300 Subject: [PATCH 03/59] eth/catalyst: fix NewPayload warn log when dropping due to snap sync --- eth/catalyst/api.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/eth/catalyst/api.go b/eth/catalyst/api.go index beffbb972..7903a7e88 100644 --- a/eth/catalyst/api.go +++ b/eth/catalyst/api.go @@ -417,7 +417,17 @@ func (api *ConsensusAPI) delayPayloadImport(block *types.Block) (beacon.PayloadS // payload as non-integratable on top of the existing sync. We'll just // have to rely on the beacon client to forcefully update the head with // a forkchoice update request. - log.Warn("Ignoring payload with missing parent", "number", block.NumberU64(), "hash", block.Hash(), "parent", block.ParentHash()) + if api.eth.SyncMode() == downloader.FullSync { + // In full sync mode, failure to import a well-formed block can only mean + // that the parent state is missing and the syncer rejected extending the + // current cycle with the new payload. + log.Warn("Ignoring payload with missing parent", "number", block.NumberU64(), "hash", block.Hash(), "parent", block.ParentHash()) + } else { + // In non-full sync mode (i.e. snap sync) all payloads are rejected until + // snap sync terminates as snap sync relies on direct database injections + // and cannot afford concurrent out-if-band modifications via imports. + log.Warn("Ignoring payload while snap syncing", "number", block.NumberU64(), "hash", block.Hash()) + } return beacon.PayloadStatusV1{Status: beacon.ACCEPTED}, nil } From f26b63089abaa127625e99fe79240e8da533c63e Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Thu, 28 Jul 2022 08:30:50 +0200 Subject: [PATCH 04/59] node: set JWT expiry to 60 seconds (#25416) * node: set JWT expiry to 60 seconds * node: rename var --- node/jwt_handler.go | 6 ++++-- node/rpcstack_test.go | 4 ++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/node/jwt_handler.go b/node/jwt_handler.go index 28d5b87c6..363f6b3aa 100644 --- a/node/jwt_handler.go +++ b/node/jwt_handler.go @@ -24,6 +24,8 @@ import ( "github.com/golang-jwt/jwt/v4" ) +const jwtExpiryTimeout = 60 * time.Second + type jwtHandler struct { keyFunc func(token *jwt.Token) (interface{}, error) next http.Handler @@ -68,9 +70,9 @@ func (handler *jwtHandler) ServeHTTP(out http.ResponseWriter, r *http.Request) { http.Error(out, "token is expired", http.StatusForbidden) case claims.IssuedAt == nil: http.Error(out, "missing issued-at", http.StatusForbidden) - case time.Since(claims.IssuedAt.Time) > 5*time.Second: + case time.Since(claims.IssuedAt.Time) > jwtExpiryTimeout: http.Error(out, "stale token", http.StatusForbidden) - case time.Until(claims.IssuedAt.Time) > 5*time.Second: + case time.Until(claims.IssuedAt.Time) > jwtExpiryTimeout: http.Error(out, "future token", http.StatusForbidden) default: handler.next.ServeHTTP(out, r) diff --git a/node/rpcstack_test.go b/node/rpcstack_test.go index 58a022340..6fb16c504 100644 --- a/node/rpcstack_test.go +++ b/node/rpcstack_test.go @@ -356,11 +356,11 @@ func TestJWT(t *testing.T) { expFail := []func() string{ // future func() string { - return fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix() + 6})) + return fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix() + int64(jwtExpiryTimeout.Seconds()) + 1})) }, // stale func() string { - return fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix() - 6})) + return fmt.Sprintf("Bearer %v", issueToken(secret, nil, testClaim{"iat": time.Now().Unix() - int64(jwtExpiryTimeout.Seconds()) - 1})) }, // wrong algo func() string { From f3549814a9f3198a700211c31a59a081d9c4bcf9 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Thu, 28 Jul 2022 15:01:35 +0200 Subject: [PATCH 05/59] eth/catalyst: return syncing not accepted (#25414) * eth/catalyst: return syncing not accepted * eth/catalyst: fix test --- eth/catalyst/api.go | 2 +- eth/catalyst/api_test.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/eth/catalyst/api.go b/eth/catalyst/api.go index 7903a7e88..6287bdab4 100644 --- a/eth/catalyst/api.go +++ b/eth/catalyst/api.go @@ -428,7 +428,7 @@ func (api *ConsensusAPI) delayPayloadImport(block *types.Block) (beacon.PayloadS // and cannot afford concurrent out-if-band modifications via imports. log.Warn("Ignoring payload while snap syncing", "number", block.NumberU64(), "hash", block.Hash()) } - return beacon.PayloadStatusV1{Status: beacon.ACCEPTED}, nil + return beacon.PayloadStatusV1{Status: beacon.SYNCING}, nil } // setInvalidAncestor is a callback for the downloader to notify us if a bad block diff --git a/eth/catalyst/api_test.go b/eth/catalyst/api_test.go index 0372aad6b..df302f821 100644 --- a/eth/catalyst/api_test.go +++ b/eth/catalyst/api_test.go @@ -662,8 +662,8 @@ func TestEmptyBlocks(t *testing.T) { if err != nil { t.Fatal(err) } - if status.Status != beacon.ACCEPTED { - t.Errorf("invalid status: expected ACCEPTED got: %v", status.Status) + if status.Status != beacon.SYNCING { + t.Errorf("invalid status: expected SYNCING got: %v", status.Status) } if status.LatestValidHash != nil { t.Fatalf("invalid LVH: got %v wanted nil", status.LatestValidHash) From 377c7d799ff62c6060939a4f95532df93a345f63 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Thu, 28 Jul 2022 15:16:15 +0200 Subject: [PATCH 06/59] eth/catalyst: return 0x0 if latestvalid is pow block (#25423) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * eth/catalyst: return 0x0 if latestvalid is pow block * eth/catalyst: return 0x0 if latestvalid is pow block * eth/catalyst: fix header retrieval, fix sign check Co-authored-by: Péter Szilágyi --- eth/catalyst/api.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/eth/catalyst/api.go b/eth/catalyst/api.go index 6287bdab4..c5f2313ca 100644 --- a/eth/catalyst/api.go +++ b/eth/catalyst/api.go @@ -479,10 +479,15 @@ func (api *ConsensusAPI) checkInvalidAncestor(check common.Hash, head common.Has } api.invalidTipsets[head] = invalid } + // If the last valid hash is the terminal pow block, return 0x0 for latest valid hash + lastValid := &invalid.ParentHash + if header := api.eth.BlockChain().GetHeader(invalid.ParentHash, invalid.Number.Uint64()-1); header != nil && header.Difficulty.Sign() != 0 { + lastValid = &common.Hash{} + } failure := "links to previously rejected block" return &beacon.PayloadStatusV1{ Status: beacon.INVALID, - LatestValidHash: &invalid.ParentHash, + LatestValidHash: lastValid, ValidationError: &failure, } } From 9ad508018e4790da0c1c00ac355f206fca12ab7c Mon Sep 17 00:00:00 2001 From: lightclient <14004106+lightclient@users.noreply.github.com> Date: Fri, 29 Jul 2022 10:22:04 -0600 Subject: [PATCH 07/59] ethereum, ethclient: add FeeHistory support (#25403) Co-authored-by: Felix Lange --- ethclient/ethclient.go | 32 ++++++++++++++++++++++++++++++++ ethclient/ethclient_test.go | 23 +++++++++++++++++++++++ interfaces.go | 9 +++++++++ internal/ethapi/api.go | 1 + 4 files changed, 65 insertions(+) diff --git a/ethclient/ethclient.go b/ethclient/ethclient.go index 24edd8648..8a0018431 100644 --- a/ethclient/ethclient.go +++ b/ethclient/ethclient.go @@ -505,6 +505,38 @@ func (ec *Client) SuggestGasTipCap(ctx context.Context) (*big.Int, error) { return (*big.Int)(&hex), nil } +type feeHistoryResultMarshaling struct { + OldestBlock *hexutil.Big `json:"oldestBlock"` + Reward [][]*hexutil.Big `json:"reward,omitempty"` + BaseFee []*hexutil.Big `json:"baseFeePerGas,omitempty"` + GasUsedRatio []float64 `json:"gasUsedRatio"` +} + +// FeeHistory retrieves the fee market history. +func (ec *Client) FeeHistory(ctx context.Context, blockCount uint64, lastBlock *big.Int, rewardPercentiles []float64) (*ethereum.FeeHistory, error) { + var res feeHistoryResultMarshaling + if err := ec.c.CallContext(ctx, &res, "eth_feeHistory", hexutil.Uint(blockCount), toBlockNumArg(lastBlock), rewardPercentiles); err != nil { + return nil, err + } + reward := make([][]*big.Int, len(res.Reward)) + for i, r := range res.Reward { + reward[i] = make([]*big.Int, len(r)) + for j, r := range r { + reward[i][j] = (*big.Int)(r) + } + } + baseFee := make([]*big.Int, len(res.BaseFee)) + for i, b := range res.BaseFee { + baseFee[i] = (*big.Int)(b) + } + return ðereum.FeeHistory{ + OldestBlock: (*big.Int)(res.OldestBlock), + Reward: reward, + BaseFee: baseFee, + GasUsedRatio: res.GasUsedRatio, + }, nil +} + // EstimateGas tries to estimate the gas needed to execute a specific transaction based on // the current pending state of the backend blockchain. There is no guarantee that this is // the true gas limit requirement as other transactions may be added or removed by miners, diff --git a/ethclient/ethclient_test.go b/ethclient/ethclient_test.go index 4a8727b37..37add1bd3 100644 --- a/ethclient/ethclient_test.go +++ b/ethclient/ethclient_test.go @@ -508,6 +508,29 @@ func testStatusFunctions(t *testing.T, client *rpc.Client) { if gasTipCap.Cmp(big.NewInt(234375000)) != 0 { t.Fatalf("unexpected gas tip cap: %v", gasTipCap) } + + // FeeHistory + history, err := ec.FeeHistory(context.Background(), 1, big.NewInt(2), []float64{95, 99}) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + want := ðereum.FeeHistory{ + OldestBlock: big.NewInt(2), + Reward: [][]*big.Int{ + { + big.NewInt(234375000), + big.NewInt(234375000), + }, + }, + BaseFee: []*big.Int{ + big.NewInt(765625000), + big.NewInt(671627818), + }, + GasUsedRatio: []float64{0.008912678667376286}, + } + if !reflect.DeepEqual(history, want) { + t.Fatalf("FeeHistory result doesn't match expected: (got: %v, want: %v)", history, want) + } } func testCallContractAtHash(t *testing.T, client *rpc.Client) { diff --git a/interfaces.go b/interfaces.go index 76c1ef690..e8d24a57c 100644 --- a/interfaces.go +++ b/interfaces.go @@ -201,6 +201,15 @@ type GasPricer interface { SuggestGasPrice(ctx context.Context) (*big.Int, error) } +// FeeHistory provides recent fee market data that consumers can use to determine +// a reasonable maxPriorityFeePerGas value. +type FeeHistory struct { + OldestBlock *big.Int // block coresponding to first response value + Reward [][]*big.Int // list every txs priority fee per block + BaseFee []*big.Int // list of each block's base fee + GasUsedRatio []float64 // ratio of gas used out of the total available limit +} + // A PendingStateReader provides access to the pending state, which is the result of all // known executable transactions which have not yet been included in the blockchain. It is // commonly used to display the result of ’unconfirmed’ actions (e.g. wallet value diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index a3637969e..90322033b 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -86,6 +86,7 @@ type feeHistoryResult struct { GasUsedRatio []float64 `json:"gasUsedRatio"` } +// FeeHistory returns the fee market history. func (s *EthereumAPI) FeeHistory(ctx context.Context, blockCount rpc.DecimalOrHex, lastBlock rpc.BlockNumber, rewardPercentiles []float64) (*feeHistoryResult, error) { oldest, reward, baseFee, gasUsed, err := s.b.FeeHistory(ctx, int(blockCount), lastBlock, rewardPercentiles) if err != nil { From 029059947a9570e3b601686ae818c92d5f8f1b85 Mon Sep 17 00:00:00 2001 From: Seungbae Yu <72970043+dbadoy@users.noreply.github.com> Date: Sat, 30 Jul 2022 01:23:30 +0900 Subject: [PATCH 08/59] all: use AbsTime.Add instead of conversion (#25417) --- common/mclock/simclock.go | 4 ++-- common/prque/lazyqueue.go | 4 ++-- les/distributor.go | 2 +- les/flowcontrol/control.go | 2 +- les/utils/timeutils_test.go | 2 +- les/vflux/server/balance.go | 8 ++++---- 6 files changed, 11 insertions(+), 11 deletions(-) diff --git a/common/mclock/simclock.go b/common/mclock/simclock.go index 766ca0f87..f5ad3f8bc 100644 --- a/common/mclock/simclock.go +++ b/common/mclock/simclock.go @@ -58,7 +58,7 @@ func (s *Simulated) Run(d time.Duration) { s.mu.Lock() s.init() - end := s.now + AbsTime(d) + end := s.now.Add(d) var do []func() for len(s.scheduled) > 0 && s.scheduled[0].at <= end { ev := heap.Pop(&s.scheduled).(*simTimer) @@ -134,7 +134,7 @@ func (s *Simulated) AfterFunc(d time.Duration, fn func()) Timer { func (s *Simulated) schedule(d time.Duration, fn func()) *simTimer { s.init() - at := s.now + AbsTime(d) + at := s.now.Add(d) ev := &simTimer{do: fn, at: at, s: s} heap.Push(&s.scheduled, ev) s.cond.Broadcast() diff --git a/common/prque/lazyqueue.go b/common/prque/lazyqueue.go index 2fd2300d7..6fdb6cc1b 100644 --- a/common/prque/lazyqueue.go +++ b/common/prque/lazyqueue.go @@ -87,13 +87,13 @@ func (q *LazyQueue) Refresh() { // refresh re-evaluates items in the older queue and swaps the two queues func (q *LazyQueue) refresh(now mclock.AbsTime) { - q.maxUntil = now + mclock.AbsTime(q.period) + q.maxUntil = now.Add(q.period) for q.queue[0].Len() != 0 { q.Push(heap.Pop(q.queue[0]).(*item).value) } q.queue[0], q.queue[1] = q.queue[1], q.queue[0] q.indexOffset = 1 - q.indexOffset - q.maxUntil += mclock.AbsTime(q.period) + q.maxUntil = q.maxUntil.Add(q.period) } // Push adds an item to the queue diff --git a/les/distributor.go b/les/distributor.go index 31150e4d7..a0319c67f 100644 --- a/les/distributor.go +++ b/les/distributor.go @@ -256,7 +256,7 @@ func (d *requestDistributor) queue(r *distReq) chan distPeer { if r.reqOrder == 0 { d.lastReqOrder++ r.reqOrder = d.lastReqOrder - r.waitForPeers = d.clock.Now() + mclock.AbsTime(waitForPeers) + r.waitForPeers = d.clock.Now().Add(waitForPeers) } // Assign the timestamp when the request is queued no matter it's // a new one or re-queued one. diff --git a/les/flowcontrol/control.go b/les/flowcontrol/control.go index 4f0de8231..76a241fa5 100644 --- a/les/flowcontrol/control.go +++ b/les/flowcontrol/control.go @@ -182,7 +182,7 @@ func (node *ClientNode) UpdateParams(params ServerParams) { return } } - node.updateSchedule = append(node.updateSchedule, scheduledUpdate{time: now + mclock.AbsTime(DecParamDelay), params: params}) + node.updateSchedule = append(node.updateSchedule, scheduledUpdate{time: now.Add(DecParamDelay), params: params}) } } diff --git a/les/utils/timeutils_test.go b/les/utils/timeutils_test.go index 9f9e1c2dc..b219d0439 100644 --- a/les/utils/timeutils_test.go +++ b/les/utils/timeutils_test.go @@ -37,7 +37,7 @@ func TestUpdateTimer(t *testing.T) { if updated := timer.Update(func(diff time.Duration) bool { return true }); !updated { t.Fatalf("Doesn't update the clock when reaching the threshold") } - if updated := timer.UpdateAt(sim.Now()+mclock.AbsTime(time.Second), func(diff time.Duration) bool { return true }); !updated { + if updated := timer.UpdateAt(sim.Now().Add(time.Second), func(diff time.Duration) bool { return true }); !updated { t.Fatalf("Doesn't update the clock when reaching the threshold") } timer = NewUpdateTimer(sim, 0) diff --git a/les/vflux/server/balance.go b/les/vflux/server/balance.go index 727ce09a4..550c6d70c 100644 --- a/les/vflux/server/balance.go +++ b/les/vflux/server/balance.go @@ -356,7 +356,7 @@ func (n *nodeBalance) estimatePriority(capacity uint64, addBalance int64, future b = n.reducedBalance(b, now, future, capacity, avgReqCost) } if bias > 0 { - b = n.reducedBalance(b, now+mclock.AbsTime(future), bias, capacity, 0) + b = n.reducedBalance(b, now.Add(future), bias, capacity, 0) } pri := n.balanceToPriority(now, b, capacity) // Ensure that biased estimates are always lower than actual priorities, even if @@ -512,7 +512,7 @@ func (n *nodeBalance) scheduleCheck(now mclock.AbsTime) { n.updateAfter(0) return } - if n.nextUpdate == 0 || n.nextUpdate > now+mclock.AbsTime(d) { + if n.nextUpdate == 0 || n.nextUpdate > now.Add(d) { if d > time.Second { // Note: if the scheduled update is not in the very near future then we // schedule the update a bit earlier. This way we do need to update a few @@ -520,7 +520,7 @@ func (n *nodeBalance) scheduleCheck(now mclock.AbsTime) { // brings the expected firing time a little bit closer. d = ((d - time.Second) * 7 / 8) + time.Second } - n.nextUpdate = now + mclock.AbsTime(d) + n.nextUpdate = now.Add(d) n.updateAfter(d) } } else { @@ -629,7 +629,7 @@ func (n *nodeBalance) reducedBalance(b balance, start mclock.AbsTime, dt time.Du // since the costs are applied continuously during the dt time period we calculate // the expiration offset at the middle of the period var ( - at = start + mclock.AbsTime(dt/2) + at = start.Add(dt / 2) dtf = float64(dt) ) if !b.pos.IsZero() { From 1af9e4f34caccc6bdc30c151b74f9689f5627c6d Mon Sep 17 00:00:00 2001 From: Delweng Date: Sat, 30 Jul 2022 00:28:14 +0800 Subject: [PATCH 09/59] cm/puppeth: fix crash when of ethstats specifier doesn't contain `:` (#25405) Signed-off-by: Delweng --- cmd/puppeth/module.go | 9 +++++++++ cmd/puppeth/module_explorer.go | 2 +- cmd/puppeth/module_faucet.go | 2 +- cmd/puppeth/module_node.go | 2 +- 4 files changed, 12 insertions(+), 3 deletions(-) diff --git a/cmd/puppeth/module.go b/cmd/puppeth/module.go index b6a029a01..771ae3805 100644 --- a/cmd/puppeth/module.go +++ b/cmd/puppeth/module.go @@ -150,3 +150,12 @@ func checkPort(host string, port int) error { conn.Close() return nil } + +// getEthName gets the Ethereum Name from ethstats +func getEthName(s string) string { + n := strings.Index(s, ":") + if n >= 0 { + return s[:n] + } + return s +} diff --git a/cmd/puppeth/module_explorer.go b/cmd/puppeth/module_explorer.go index 1165f70fc..3812f9fdb 100644 --- a/cmd/puppeth/module_explorer.go +++ b/cmd/puppeth/module_explorer.go @@ -104,7 +104,7 @@ func deployExplorer(client *sshClient, network string, bootnodes []string, confi "Datadir": config.node.datadir, "DBDir": config.dbdir, "EthPort": config.node.port, - "EthName": config.node.ethstats[:strings.Index(config.node.ethstats, ":")], + "EthName": getEthName(config.node.ethstats), "WebPort": config.port, "Transformer": transformer, }) diff --git a/cmd/puppeth/module_faucet.go b/cmd/puppeth/module_faucet.go index 88cb80ae4..a4f6e6569 100644 --- a/cmd/puppeth/module_faucet.go +++ b/cmd/puppeth/module_faucet.go @@ -116,7 +116,7 @@ func deployFaucet(client *sshClient, network string, bootnodes []string, config "VHost": config.host, "ApiPort": config.port, "EthPort": config.node.port, - "EthName": config.node.ethstats[:strings.Index(config.node.ethstats, ":")], + "EthName": getEthName(config.node.ethstats), "CaptchaToken": config.captchaToken, "CaptchaSecret": config.captchaSecret, "FaucetAmount": config.amount, diff --git a/cmd/puppeth/module_node.go b/cmd/puppeth/module_node.go index 3ea96870d..b8aa30db3 100644 --- a/cmd/puppeth/module_node.go +++ b/cmd/puppeth/module_node.go @@ -123,7 +123,7 @@ func deployNode(client *sshClient, network string, bootnodes []string, config *n "TotalPeers": config.peersTotal, "Light": config.peersLight > 0, "LightPeers": config.peersLight, - "Ethstats": config.ethstats[:strings.Index(config.ethstats, ":")], + "Ethstats": getEthName(config.ethstats), "Etherbase": config.etherbase, "GasTarget": config.gasTarget, "GasLimit": config.gasLimit, From fea569f90aa608abeb13cea6c9fd55f1d0a77f40 Mon Sep 17 00:00:00 2001 From: Rithwik Babu Date: Fri, 29 Jul 2022 11:29:01 -0500 Subject: [PATCH 10/59] eth: fix typo in comment (#25327) --- eth/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth/api.go b/eth/api.go index ad8566dae..5d9a3cc3a 100644 --- a/eth/api.go +++ b/eth/api.go @@ -157,7 +157,7 @@ func (api *AdminAPI) ExportChain(file string, first *uint64, last *uint64) (bool } if _, err := os.Stat(file); err == nil { // File already exists. Allowing overwrite could be a DoS vector, - // since the 'file' may point to arbitrary paths on the drive + // since the 'file' may point to arbitrary paths on the drive. return false, errors.New("location would overwrite an existing file") } // Make sure we can create the file to export into From 49aa8a633b7333d16ff68777c95d45e897e1903f Mon Sep 17 00:00:00 2001 From: Henry <101552941+henry-0@users.noreply.github.com> Date: Mon, 1 Aug 2022 19:47:21 +0800 Subject: [PATCH 11/59] common/compiler: json unmarshalling error checks (#25449) complier/solidity:add json.Unmarshal err check --- common/compiler/solidity.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/common/compiler/solidity.go b/common/compiler/solidity.go index ad8a44aa0..9de94017c 100644 --- a/common/compiler/solidity.go +++ b/common/compiler/solidity.go @@ -66,13 +66,16 @@ func ParseCombinedJSON(combinedJSON []byte, source string, languageVersion strin contracts := make(map[string]*Contract) for name, info := range output.Contracts { // Parse the individual compilation results. - var abi interface{} + var abi, userdoc, devdoc interface{} if err := json.Unmarshal([]byte(info.Abi), &abi); err != nil { return nil, fmt.Errorf("solc: error reading abi definition (%v)", err) } - var userdoc, devdoc interface{} - json.Unmarshal([]byte(info.Userdoc), &userdoc) - json.Unmarshal([]byte(info.Devdoc), &devdoc) + if err := json.Unmarshal([]byte(info.Userdoc), &userdoc); err != nil { + return nil, fmt.Errorf("solc: error reading userdoc definition (%v)", err) + } + if err := json.Unmarshal([]byte(info.Devdoc), &devdoc); err != nil { + return nil, fmt.Errorf("solc: error reading devdoc definition (%v)", err) + } contracts[name] = &Contract{ Code: "0x" + info.Bin, From 6fd06ab0751f1312d48bf593c7f356dcd6f51ade Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 1 Aug 2022 15:13:25 +0300 Subject: [PATCH 12/59] cmd, core, eth, les, params: add merge-passed chain config (#24538) * cmd, core, eth, les, params: add merge-passed chain config * eth/catalyst, params: add various warning on malfunctioning beacons * eth/catalyst: fix warning for beacons without transition exchanges --- cmd/geth/config.go | 9 +- cmd/geth/main.go | 2 +- cmd/utils/flags.go | 11 +-- core/genesis.go | 6 +- core/rawdb/chain_freezer.go | 2 +- eth/backend.go | 2 +- eth/catalyst/api.go | 159 ++++++++++++++++++++++++++++++- eth/ethconfig/config.go | 6 +- eth/ethconfig/gen_config.go | 180 ++++++++++++++++++------------------ eth/handler.go | 19 +++- eth/sync.go | 2 +- les/client.go | 2 +- params/config.go | 92 ++++++++++-------- 13 files changed, 332 insertions(+), 160 deletions(-) diff --git a/cmd/geth/config.go b/cmd/geth/config.go index 710e71836..2562de8ae 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -20,7 +20,6 @@ import ( "bufio" "errors" "fmt" - "math/big" "os" "reflect" "unicode" @@ -157,12 +156,13 @@ func makeConfigNode(ctx *cli.Context) (*node.Node, gethConfig) { // makeFullNode loads geth configuration and creates the Ethereum backend. func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) { stack, cfg := makeConfigNode(ctx) - if ctx.IsSet(utils.OverrideGrayGlacierFlag.Name) { - cfg.Eth.OverrideGrayGlacier = new(big.Int).SetUint64(ctx.Uint64(utils.OverrideGrayGlacierFlag.Name)) - } if ctx.IsSet(utils.OverrideTerminalTotalDifficulty.Name) { cfg.Eth.OverrideTerminalTotalDifficulty = flags.GlobalBig(ctx, utils.OverrideTerminalTotalDifficulty.Name) } + if ctx.IsSet(utils.OverrideTerminalTotalDifficultyPassed.Name) { + override := ctx.Bool(utils.OverrideTerminalTotalDifficultyPassed.Name) + cfg.Eth.OverrideTerminalTotalDifficultyPassed = &override + } backend, eth := utils.RegisterEthService(stack, &cfg.Eth) // Warn users to migrate if they have a legacy freezer format. if eth != nil && !ctx.IsSet(utils.IgnoreLegacyReceiptsFlag.Name) { @@ -181,7 +181,6 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) { utils.Fatalf("Database has receipts with a legacy format. Please run `geth db freezer-migrate`.") } } - // Configure GraphQL if requested if ctx.IsSet(utils.GraphQLEnabledFlag.Name) { utils.RegisterGraphQLService(stack, backend, cfg.Node) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 9aae75974..c0f636fb2 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -69,8 +69,8 @@ var ( utils.NoUSBFlag, utils.USBFlag, utils.SmartCardDaemonPathFlag, - utils.OverrideGrayGlacierFlag, utils.OverrideTerminalTotalDifficulty, + utils.OverrideTerminalTotalDifficultyPassed, utils.EthashCacheDirFlag, utils.EthashCachesInMemoryFlag, utils.EthashCachesOnDiskFlag, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index f55719af4..ff85e259a 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -262,17 +262,16 @@ var ( Value: 2048, Category: flags.EthCategory, } - OverrideGrayGlacierFlag = &cli.Uint64Flag{ - Name: "override.grayglacier", - Usage: "Manually specify Gray Glacier fork-block, overriding the bundled setting", - Category: flags.EthCategory, - } OverrideTerminalTotalDifficulty = &flags.BigFlag{ Name: "override.terminaltotaldifficulty", Usage: "Manually specify TerminalTotalDifficulty, overriding the bundled setting", Category: flags.EthCategory, } - + OverrideTerminalTotalDifficultyPassed = &cli.BoolFlag{ + Name: "override.terminaltotaldifficultypassed", + Usage: "Manually specify TerminalTotalDifficultyPassed, overriding the bundled setting", + Category: flags.EthCategory, + } // Light server and client settings LightServeFlag = &cli.IntFlag{ Name: "light.serve", diff --git a/core/genesis.go b/core/genesis.go index 7dcc7cfc3..45284f3de 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -233,7 +233,7 @@ func SetupGenesisBlock(db ethdb.Database, genesis *Genesis) (*params.ChainConfig return SetupGenesisBlockWithOverride(db, genesis, nil, nil) } -func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, overrideGrayGlacier, overrideTerminalTotalDifficulty *big.Int) (*params.ChainConfig, common.Hash, error) { +func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, overrideTerminalTotalDifficulty *big.Int, overrideTerminalTotalDifficultyPassed *bool) (*params.ChainConfig, common.Hash, error) { if genesis != nil && genesis.Config == nil { return params.AllEthashProtocolChanges, common.Hash{}, errGenesisNoConfig } @@ -243,8 +243,8 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, override if overrideTerminalTotalDifficulty != nil { config.TerminalTotalDifficulty = overrideTerminalTotalDifficulty } - if overrideGrayGlacier != nil { - config.GrayGlacierBlock = overrideGrayGlacier + if overrideTerminalTotalDifficultyPassed != nil { + config.TerminalTotalDifficultyPassed = *overrideTerminalTotalDifficultyPassed } } } diff --git a/core/rawdb/chain_freezer.go b/core/rawdb/chain_freezer.go index 4c49db274..ec39b7b59 100644 --- a/core/rawdb/chain_freezer.go +++ b/core/rawdb/chain_freezer.go @@ -241,7 +241,7 @@ func (f *chainFreezer) freeze(db ethdb.KeyValueStore) { if n := len(ancients); n > 0 { context = append(context, []interface{}{"hash", ancients[n-1]}...) } - log.Info("Deep froze chain segment", context...) + log.Debug("Deep froze chain segment", context...) // Avoid database thrashing with tiny writes if frozen-first < freezerBatchLimit { diff --git a/eth/backend.go b/eth/backend.go index b16ce4b54..ebe7001c7 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -137,7 +137,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*Ethereum, error) { if err != nil { return nil, err } - chainConfig, genesisHash, genesisErr := core.SetupGenesisBlockWithOverride(chainDb, config.Genesis, config.OverrideGrayGlacier, config.OverrideTerminalTotalDifficulty) + chainConfig, genesisHash, genesisErr := core.SetupGenesisBlockWithOverride(chainDb, config.Genesis, config.OverrideTerminalTotalDifficulty, config.OverrideTerminalTotalDifficultyPassed) if _, ok := genesisErr.(*params.ConfigCompatError); genesisErr != nil && !ok { return nil, genesisErr } diff --git a/eth/catalyst/api.go b/eth/catalyst/api.go index c5f2313ca..93b78a386 100644 --- a/eth/catalyst/api.go +++ b/eth/catalyst/api.go @@ -22,6 +22,7 @@ import ( "encoding/binary" "errors" "fmt" + "math/big" "sync" "time" @@ -59,6 +60,11 @@ const ( // invalidTipsetsCap is the max number of recent block hashes tracked that // have lead to some bad ancestor block. It's just an OOM protection. invalidTipsetsCap = 512 + + // beaconUpdateTimeout is the max time allowed for a beacon client to signal + // use (from the last heartbeat) before it's consifered offline and the user + // is warned. + beaconUpdateTimeout = 30 * time.Second ) type ConsensusAPI struct { @@ -90,7 +96,17 @@ type ConsensusAPI struct { invalidTipsets map[common.Hash]*types.Header // Ephemeral cache to track invalid tipsets and their bad ancestor invalidLock sync.Mutex // Protects the invalid maps from concurrent access - forkChoiceLock sync.Mutex // Lock for the forkChoiceUpdated method + // Geth can appear to be stuck or do strange things if the beacon client is + // offline or is sending us strange data. Stash some update stats away so + // that we can warn the user and not have them open issues on our tracker. + lastTransitionUpdate time.Time + lastTransitionLock sync.Mutex + lastForkchoiceUpdate time.Time + lastForkchoiceLock sync.Mutex + lastNewPayloadUpdate time.Time + lastNewPayloadLock sync.Mutex + + forkchoiceLock sync.Mutex // Lock for the forkChoiceUpdated method } // NewConsensusAPI creates a new consensus api for the given backend. @@ -107,6 +123,7 @@ func NewConsensusAPI(eth *eth.Ethereum) *ConsensusAPI { invalidTipsets: make(map[common.Hash]*types.Header), } eth.Downloader().SetBadBlockCallback(api.setInvalidAncestor) + go api.heartbeat() return api } @@ -122,14 +139,18 @@ func NewConsensusAPI(eth *eth.Ethereum) *ConsensusAPI { // If there are payloadAttributes: // we try to assemble a block with the payloadAttributes and return its payloadID func (api *ConsensusAPI) ForkchoiceUpdatedV1(update beacon.ForkchoiceStateV1, payloadAttributes *beacon.PayloadAttributesV1) (beacon.ForkChoiceResponse, error) { - api.forkChoiceLock.Lock() - defer api.forkChoiceLock.Unlock() + api.forkchoiceLock.Lock() + defer api.forkchoiceLock.Unlock() log.Trace("Engine API request received", "method", "ForkchoiceUpdated", "head", update.HeadBlockHash, "finalized", update.FinalizedBlockHash, "safe", update.SafeBlockHash) if update.HeadBlockHash == (common.Hash{}) { log.Warn("Forkchoice requested update to zero hash") return beacon.STATUS_INVALID, nil // TODO(karalabe): Why does someone send us this? } + // Stash away the last update to warn the user if the beacon client goes offline + api.lastForkchoiceLock.Lock() + api.lastForkchoiceUpdate = time.Now() + api.lastForkchoiceLock.Unlock() // Check whether we have the block yet in our database or not. If not, we'll // need to either trigger a sync, or to reject this forkchoice update for a @@ -265,15 +286,20 @@ func (api *ConsensusAPI) ForkchoiceUpdatedV1(update beacon.ForkchoiceStateV1, pa // ExchangeTransitionConfigurationV1 checks the given configuration against // the configuration of the node. func (api *ConsensusAPI) ExchangeTransitionConfigurationV1(config beacon.TransitionConfigurationV1) (*beacon.TransitionConfigurationV1, error) { + log.Trace("Engine API request received", "method", "ExchangeTransitionConfiguration", "ttd", config.TerminalTotalDifficulty) if config.TerminalTotalDifficulty == nil { return nil, errors.New("invalid terminal total difficulty") } + // Stash away the last update to warn the user if the beacon client goes offline + api.lastTransitionLock.Lock() + api.lastTransitionUpdate = time.Now() + api.lastTransitionLock.Unlock() + ttd := api.eth.BlockChain().Config().TerminalTotalDifficulty if ttd == nil || ttd.Cmp(config.TerminalTotalDifficulty.ToInt()) != 0 { log.Warn("Invalid TTD configured", "geth", ttd, "beacon", config.TerminalTotalDifficulty) return nil, fmt.Errorf("invalid ttd: execution %v consensus %v", ttd, config.TerminalTotalDifficulty) } - if config.TerminalBlockHash != (common.Hash{}) { if hash := api.eth.BlockChain().GetCanonicalHash(uint64(config.TerminalBlockNumber)); hash == config.TerminalBlockHash { return &beacon.TransitionConfigurationV1{ @@ -305,6 +331,11 @@ func (api *ConsensusAPI) NewPayloadV1(params beacon.ExecutableDataV1) (beacon.Pa log.Debug("Invalid NewPayload params", "params", params, "error", err) return beacon.PayloadStatusV1{Status: beacon.INVALIDBLOCKHASH}, nil } + // Stash away the last update to warn the user if the beacon client goes offline + api.lastNewPayloadLock.Lock() + api.lastNewPayloadUpdate = time.Now() + api.lastNewPayloadLock.Unlock() + // If we already have the block locally, ignore the entire execution and just // return a fake success. if block := api.eth.BlockChain().GetBlockByHash(params.BlockHash); block != nil { @@ -507,3 +538,123 @@ func (api *ConsensusAPI) invalid(err error, latestValid *types.Header) beacon.Pa errorMsg := err.Error() return beacon.PayloadStatusV1{Status: beacon.INVALID, LatestValidHash: ¤tHash, ValidationError: &errorMsg} } + +// heatbeat loops indefinitely, and checks if there have been beacon client updates +// received in the last while. If not - or if they but strange ones - it warns the +// user that something might be off with their consensus node. +// +// TODO(karalabe): Spin this goroutine down somehow +func (api *ConsensusAPI) heartbeat() { + // Sleep a bit more on startup since there's obviously no beacon client yet + // attached, so no need to print scary warnings to the user. + time.Sleep(beaconUpdateTimeout) + + var ( + offlineLogged time.Time + ) + for { + // Sleep a bit and retrieve the last known consensus updates + time.Sleep(5 * time.Second) + + // If the network is not yet merged/merging, don't bother scaring the user + ttd := api.eth.BlockChain().Config().TerminalTotalDifficulty + if ttd == nil { + continue + } + api.lastTransitionLock.Lock() + lastTransitionUpdate := api.lastTransitionUpdate + api.lastTransitionLock.Unlock() + + api.lastForkchoiceLock.Lock() + lastForkchoiceUpdate := api.lastForkchoiceUpdate + api.lastForkchoiceLock.Unlock() + + api.lastNewPayloadLock.Lock() + lastNewPayloadUpdate := api.lastNewPayloadUpdate + api.lastNewPayloadLock.Unlock() + + // If there have been no updates for the past while, warn the user + // that the beacon client is probably offline + if api.eth.BlockChain().Config().TerminalTotalDifficultyPassed || api.eth.Merger().TDDReached() { + if time.Since(lastForkchoiceUpdate) > beaconUpdateTimeout && time.Since(lastNewPayloadUpdate) > beaconUpdateTimeout { + if time.Since(lastTransitionUpdate) > beaconUpdateTimeout { + if time.Since(offlineLogged) > beaconUpdateTimeout { + if lastTransitionUpdate.IsZero() { + log.Warn("Post-merge network, but no beacon client seen. Please launch one to follow the chain!") + } else { + log.Warn("Previously seen beacon client is offline. Please ensure it is operational to follow the chain!") + } + offlineLogged = time.Now() + } + continue + } + if time.Since(offlineLogged) > beaconUpdateTimeout { + if lastForkchoiceUpdate.IsZero() && lastNewPayloadUpdate.IsZero() { + log.Warn("Beacon client online, but never received consensus updates. Please ensure your beacon client is operational to follow the chain!") + } else { + log.Warn("Beacon client online, but no consensus updates received in a while. Please fix your beacon client to follow the chain!") + } + offlineLogged = time.Now() + } + continue + } + } else { + if time.Since(lastTransitionUpdate) > beaconUpdateTimeout { + if time.Since(offlineLogged) > beaconUpdateTimeout { + // Retrieve the last few blocks and make a rough estimate as + // to when the merge transition should happen + var ( + chain = api.eth.BlockChain() + head = chain.CurrentBlock() + htd = chain.GetTd(head.Hash(), head.NumberU64()) + eta time.Duration + ) + if head.NumberU64() > 0 && htd.Cmp(ttd) < 0 { + // Accumulate the last 64 difficulties to estimate the growth + var diff float64 + + block := head + for i := 0; i < 64; i++ { + diff += float64(block.Difficulty().Uint64()) + if parent := chain.GetBlock(block.ParentHash(), block.NumberU64()-1); parent == nil { + break + } else { + block = parent + } + } + // Estimate an ETA based on the block times and the difficulty growth + growth := diff / float64(head.Time()-block.Time()+1) // +1 to avoid div by zero + if growth > 0 { + if left := new(big.Int).Sub(ttd, htd); left.IsUint64() { + eta = time.Duration(float64(left.Uint64())/growth) * time.Second + } else { + eta = time.Duration(new(big.Int).Div(left, big.NewInt(int64(growth))).Uint64()) * time.Second + } + } + } + var message string + if htd.Cmp(ttd) > 0 { + if lastTransitionUpdate.IsZero() { + message = "Merge already reached, but no beacon client seen. Please launch one to follow the chain!" + } else { + message = "Merge already reached, but previously seen beacon client is offline. Please ensure it is operational to follow the chain!" + } + } else { + if lastTransitionUpdate.IsZero() { + message = "Merge is configured, but no beacon client seen. Please ensure you have one available before the transision arrives!" + } else { + message = "Merge is configured, but previously seen beacon client is offline. Please ensure it is operational before the transision arrives!" + } + } + if eta == 0 { + log.Warn(message) + } else { + log.Warn(message, "eta", common.PrettyAge(time.Now().Add(-eta))) // weird hack, but duration formatted doens't handle days + } + offlineLogged = time.Now() + } + continue + } + } + } +} diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index f94963595..7ba2faf79 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -205,11 +205,11 @@ type Config struct { // CheckpointOracle is the configuration for checkpoint oracle. CheckpointOracle *params.CheckpointOracleConfig `toml:",omitempty"` - // Gray Glacier block override (TODO: remove after the fork) - OverrideGrayGlacier *big.Int `toml:",omitempty"` - // OverrideTerminalTotalDifficulty (TODO: remove after the fork) OverrideTerminalTotalDifficulty *big.Int `toml:",omitempty"` + + // OverrideTerminalTotalDifficultyPassed (TODO: remove after the fork) + OverrideTerminalTotalDifficultyPassed *bool `toml:",omitempty"` } // CreateConsensusEngine creates a consensus engine for the given chain configuration. diff --git a/eth/ethconfig/gen_config.go b/eth/ethconfig/gen_config.go index e714dd97a..a6528c8df 100644 --- a/eth/ethconfig/gen_config.go +++ b/eth/ethconfig/gen_config.go @@ -18,49 +18,49 @@ import ( // MarshalTOML marshals as TOML. func (c Config) MarshalTOML() (interface{}, error) { type Config struct { - Genesis *core.Genesis `toml:",omitempty"` - NetworkId uint64 - SyncMode downloader.SyncMode - EthDiscoveryURLs []string - SnapDiscoveryURLs []string - NoPruning bool - NoPrefetch bool - TxLookupLimit uint64 `toml:",omitempty"` - RequiredBlocks map[uint64]common.Hash `toml:"-"` - LightServ int `toml:",omitempty"` - LightIngress int `toml:",omitempty"` - LightEgress int `toml:",omitempty"` - LightPeers int `toml:",omitempty"` - LightNoPrune bool `toml:",omitempty"` - LightNoSyncServe bool `toml:",omitempty"` - SyncFromCheckpoint bool `toml:",omitempty"` - UltraLightServers []string `toml:",omitempty"` - UltraLightFraction int `toml:",omitempty"` - UltraLightOnlyAnnounce bool `toml:",omitempty"` - SkipBcVersionCheck bool `toml:"-"` - DatabaseHandles int `toml:"-"` - DatabaseCache int - DatabaseFreezer string - TrieCleanCache int - TrieCleanCacheJournal string `toml:",omitempty"` - TrieCleanCacheRejournal time.Duration `toml:",omitempty"` - TrieDirtyCache int - TrieTimeout time.Duration - SnapshotCache int - Preimages bool - Miner miner.Config - Ethash ethash.Config - TxPool core.TxPoolConfig - GPO gasprice.Config - EnablePreimageRecording bool - DocRoot string `toml:"-"` - RPCGasCap uint64 - RPCEVMTimeout time.Duration - RPCTxFeeCap float64 - Checkpoint *params.TrustedCheckpoint `toml:",omitempty"` - CheckpointOracle *params.CheckpointOracleConfig `toml:",omitempty"` - OverrideGrayGlacier *big.Int `toml:",omitempty"` - OverrideTerminalTotalDifficulty *big.Int `toml:",omitempty"` + Genesis *core.Genesis `toml:",omitempty"` + NetworkId uint64 + SyncMode downloader.SyncMode + EthDiscoveryURLs []string + SnapDiscoveryURLs []string + NoPruning bool + NoPrefetch bool + TxLookupLimit uint64 `toml:",omitempty"` + RequiredBlocks map[uint64]common.Hash `toml:"-"` + LightServ int `toml:",omitempty"` + LightIngress int `toml:",omitempty"` + LightEgress int `toml:",omitempty"` + LightPeers int `toml:",omitempty"` + LightNoPrune bool `toml:",omitempty"` + LightNoSyncServe bool `toml:",omitempty"` + SyncFromCheckpoint bool `toml:",omitempty"` + UltraLightServers []string `toml:",omitempty"` + UltraLightFraction int `toml:",omitempty"` + UltraLightOnlyAnnounce bool `toml:",omitempty"` + SkipBcVersionCheck bool `toml:"-"` + DatabaseHandles int `toml:"-"` + DatabaseCache int + DatabaseFreezer string + TrieCleanCache int + TrieCleanCacheJournal string `toml:",omitempty"` + TrieCleanCacheRejournal time.Duration `toml:",omitempty"` + TrieDirtyCache int + TrieTimeout time.Duration + SnapshotCache int + Preimages bool + Miner miner.Config + Ethash ethash.Config + TxPool core.TxPoolConfig + GPO gasprice.Config + EnablePreimageRecording bool + DocRoot string `toml:"-"` + RPCGasCap uint64 + RPCEVMTimeout time.Duration + RPCTxFeeCap float64 + Checkpoint *params.TrustedCheckpoint `toml:",omitempty"` + CheckpointOracle *params.CheckpointOracleConfig `toml:",omitempty"` + OverrideTerminalTotalDifficulty *big.Int `toml:",omitempty"` + OverrideTerminalTotalDifficultyPassed *bool `toml:",omitempty"` } var enc Config enc.Genesis = c.Genesis @@ -104,57 +104,57 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.RPCTxFeeCap = c.RPCTxFeeCap enc.Checkpoint = c.Checkpoint enc.CheckpointOracle = c.CheckpointOracle - enc.OverrideGrayGlacier = c.OverrideGrayGlacier enc.OverrideTerminalTotalDifficulty = c.OverrideTerminalTotalDifficulty + enc.OverrideTerminalTotalDifficultyPassed = c.OverrideTerminalTotalDifficultyPassed return &enc, nil } // UnmarshalTOML unmarshals from TOML. func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { type Config struct { - Genesis *core.Genesis `toml:",omitempty"` - NetworkId *uint64 - SyncMode *downloader.SyncMode - EthDiscoveryURLs []string - SnapDiscoveryURLs []string - NoPruning *bool - NoPrefetch *bool - TxLookupLimit *uint64 `toml:",omitempty"` - RequiredBlocks map[uint64]common.Hash `toml:"-"` - LightServ *int `toml:",omitempty"` - LightIngress *int `toml:",omitempty"` - LightEgress *int `toml:",omitempty"` - LightPeers *int `toml:",omitempty"` - LightNoPrune *bool `toml:",omitempty"` - LightNoSyncServe *bool `toml:",omitempty"` - SyncFromCheckpoint *bool `toml:",omitempty"` - UltraLightServers []string `toml:",omitempty"` - UltraLightFraction *int `toml:",omitempty"` - UltraLightOnlyAnnounce *bool `toml:",omitempty"` - SkipBcVersionCheck *bool `toml:"-"` - DatabaseHandles *int `toml:"-"` - DatabaseCache *int - DatabaseFreezer *string - TrieCleanCache *int - TrieCleanCacheJournal *string `toml:",omitempty"` - TrieCleanCacheRejournal *time.Duration `toml:",omitempty"` - TrieDirtyCache *int - TrieTimeout *time.Duration - SnapshotCache *int - Preimages *bool - Miner *miner.Config - Ethash *ethash.Config - TxPool *core.TxPoolConfig - GPO *gasprice.Config - EnablePreimageRecording *bool - DocRoot *string `toml:"-"` - RPCGasCap *uint64 - RPCEVMTimeout *time.Duration - RPCTxFeeCap *float64 - Checkpoint *params.TrustedCheckpoint `toml:",omitempty"` - CheckpointOracle *params.CheckpointOracleConfig `toml:",omitempty"` - OverrideGrayGlacier *big.Int `toml:",omitempty"` - OverrideTerminalTotalDifficulty *big.Int `toml:",omitempty"` + Genesis *core.Genesis `toml:",omitempty"` + NetworkId *uint64 + SyncMode *downloader.SyncMode + EthDiscoveryURLs []string + SnapDiscoveryURLs []string + NoPruning *bool + NoPrefetch *bool + TxLookupLimit *uint64 `toml:",omitempty"` + RequiredBlocks map[uint64]common.Hash `toml:"-"` + LightServ *int `toml:",omitempty"` + LightIngress *int `toml:",omitempty"` + LightEgress *int `toml:",omitempty"` + LightPeers *int `toml:",omitempty"` + LightNoPrune *bool `toml:",omitempty"` + LightNoSyncServe *bool `toml:",omitempty"` + SyncFromCheckpoint *bool `toml:",omitempty"` + UltraLightServers []string `toml:",omitempty"` + UltraLightFraction *int `toml:",omitempty"` + UltraLightOnlyAnnounce *bool `toml:",omitempty"` + SkipBcVersionCheck *bool `toml:"-"` + DatabaseHandles *int `toml:"-"` + DatabaseCache *int + DatabaseFreezer *string + TrieCleanCache *int + TrieCleanCacheJournal *string `toml:",omitempty"` + TrieCleanCacheRejournal *time.Duration `toml:",omitempty"` + TrieDirtyCache *int + TrieTimeout *time.Duration + SnapshotCache *int + Preimages *bool + Miner *miner.Config + Ethash *ethash.Config + TxPool *core.TxPoolConfig + GPO *gasprice.Config + EnablePreimageRecording *bool + DocRoot *string `toml:"-"` + RPCGasCap *uint64 + RPCEVMTimeout *time.Duration + RPCTxFeeCap *float64 + Checkpoint *params.TrustedCheckpoint `toml:",omitempty"` + CheckpointOracle *params.CheckpointOracleConfig `toml:",omitempty"` + OverrideTerminalTotalDifficulty *big.Int `toml:",omitempty"` + OverrideTerminalTotalDifficultyPassed *bool `toml:",omitempty"` } var dec Config if err := unmarshal(&dec); err != nil { @@ -283,11 +283,11 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.CheckpointOracle != nil { c.CheckpointOracle = dec.CheckpointOracle } - if dec.OverrideGrayGlacier != nil { - c.OverrideGrayGlacier = dec.OverrideGrayGlacier - } if dec.OverrideTerminalTotalDifficulty != nil { c.OverrideTerminalTotalDifficulty = dec.OverrideTerminalTotalDifficulty } + if dec.OverrideTerminalTotalDifficultyPassed != nil { + c.OverrideTerminalTotalDifficultyPassed = dec.OverrideTerminalTotalDifficultyPassed + } return nil } diff --git a/eth/handler.go b/eth/handler.go index 1418c7389..4224a9f33 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -191,11 +191,22 @@ func newHandler(config *handlerConfig) (*handler, error) { } } } - // Construct the downloader (long sync) and its backing state bloom if snap - // sync is requested. The downloader is responsible for deallocating the state - // bloom when it's done. + // Construct the downloader (long sync) h.downloader = downloader.New(h.checkpointNumber, config.Database, h.eventMux, h.chain, nil, h.removePeer, success) - + if ttd := h.chain.Config().TerminalTotalDifficulty; ttd != nil { + if h.chain.Config().TerminalTotalDifficultyPassed { + log.Info("Chain post-merge, sync via beacon client") + } else { + head := h.chain.CurrentBlock() + if td := h.chain.GetTd(head.Hash(), head.NumberU64()); td.Cmp(ttd) >= 0 { + log.Info("Chain post-TTD, sync via beacon client") + } else { + log.Warn("Chain pre-merge, sync via PoW (ensure beacon client is ready)") + } + } + } else if h.chain.Config().TerminalTotalDifficultyPassed { + log.Error("Chain configured post-merge, but without TTD. Are you debugging sync?") + } // Construct the fetcher (short sync) validator := func(header *types.Header) error { // All the block fetcher activities should be disabled diff --git a/eth/sync.go b/eth/sync.go index d67d2311d..8fd86b578 100644 --- a/eth/sync.go +++ b/eth/sync.go @@ -163,7 +163,7 @@ func (cs *chainSyncer) nextSyncOp() *chainSyncOp { // An alternative would be to check the local chain for exceeding the TTD and // avoid triggering a sync in that case, but that could also miss sibling or // other family TTD block being accepted. - if cs.handler.merger.TDDReached() { + if cs.handler.chain.Config().TerminalTotalDifficultyPassed || cs.handler.merger.TDDReached() { return nil } // Ensure we're at minimum peer count. diff --git a/les/client.go b/les/client.go index 44eaffec2..7caaf2c18 100644 --- a/les/client.go +++ b/les/client.go @@ -93,7 +93,7 @@ func New(stack *node.Node, config *ethconfig.Config) (*LightEthereum, error) { if err != nil { return nil, err } - chainConfig, genesisHash, genesisErr := core.SetupGenesisBlockWithOverride(chainDb, config.Genesis, config.OverrideGrayGlacier, config.OverrideTerminalTotalDifficulty) + chainConfig, genesisHash, genesisErr := core.SetupGenesisBlockWithOverride(chainDb, config.Genesis, config.OverrideTerminalTotalDifficulty, config.OverrideTerminalTotalDifficultyPassed) if _, isCompat := genesisErr.(*params.ConfigCompatError); genesisErr != nil && !isCompat { return nil, genesisErr } diff --git a/params/config.go b/params/config.go index 47592e816..fda4c1aec 100644 --- a/params/config.go +++ b/params/config.go @@ -100,23 +100,24 @@ var ( // RopstenChainConfig contains the chain parameters to run a node on the Ropsten test network. RopstenChainConfig = &ChainConfig{ - ChainID: big.NewInt(3), - HomesteadBlock: big.NewInt(0), - DAOForkBlock: nil, - DAOForkSupport: true, - EIP150Block: big.NewInt(0), - EIP150Hash: common.HexToHash("0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d"), - EIP155Block: big.NewInt(10), - EIP158Block: big.NewInt(10), - ByzantiumBlock: big.NewInt(1_700_000), - ConstantinopleBlock: big.NewInt(4_230_000), - PetersburgBlock: big.NewInt(4_939_394), - IstanbulBlock: big.NewInt(6_485_846), - MuirGlacierBlock: big.NewInt(7_117_117), - BerlinBlock: big.NewInt(9_812_189), - LondonBlock: big.NewInt(10_499_401), - TerminalTotalDifficulty: new(big.Int).SetUint64(50000000000000000), - Ethash: new(EthashConfig), + ChainID: big.NewInt(3), + HomesteadBlock: big.NewInt(0), + DAOForkBlock: nil, + DAOForkSupport: true, + EIP150Block: big.NewInt(0), + EIP150Hash: common.HexToHash("0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d"), + EIP155Block: big.NewInt(10), + EIP158Block: big.NewInt(10), + ByzantiumBlock: big.NewInt(1_700_000), + ConstantinopleBlock: big.NewInt(4_230_000), + PetersburgBlock: big.NewInt(4_939_394), + IstanbulBlock: big.NewInt(6_485_846), + MuirGlacierBlock: big.NewInt(7_117_117), + BerlinBlock: big.NewInt(9_812_189), + LondonBlock: big.NewInt(10_499_401), + TerminalTotalDifficulty: new(big.Int).SetUint64(50_000_000_000_000_000), + TerminalTotalDifficultyPassed: true, + Ethash: new(EthashConfig), } // RopstenTrustedCheckpoint contains the light client trusted checkpoint for the Ropsten test network. @@ -142,23 +143,24 @@ var ( // SepoliaChainConfig contains the chain parameters to run a node on the Sepolia test network. SepoliaChainConfig = &ChainConfig{ - ChainID: big.NewInt(11155111), - HomesteadBlock: big.NewInt(0), - DAOForkBlock: nil, - DAOForkSupport: true, - EIP150Block: big.NewInt(0), - EIP155Block: big.NewInt(0), - EIP158Block: big.NewInt(0), - ByzantiumBlock: big.NewInt(0), - ConstantinopleBlock: big.NewInt(0), - PetersburgBlock: big.NewInt(0), - IstanbulBlock: big.NewInt(0), - MuirGlacierBlock: big.NewInt(0), - BerlinBlock: big.NewInt(0), - LondonBlock: big.NewInt(0), - TerminalTotalDifficulty: big.NewInt(17_000_000_000_000_000), - MergeNetsplitBlock: big.NewInt(1735371), - Ethash: new(EthashConfig), + ChainID: big.NewInt(11155111), + HomesteadBlock: big.NewInt(0), + DAOForkBlock: nil, + DAOForkSupport: true, + EIP150Block: big.NewInt(0), + EIP155Block: big.NewInt(0), + EIP158Block: big.NewInt(0), + ByzantiumBlock: big.NewInt(0), + ConstantinopleBlock: big.NewInt(0), + PetersburgBlock: big.NewInt(0), + IstanbulBlock: big.NewInt(0), + MuirGlacierBlock: big.NewInt(0), + BerlinBlock: big.NewInt(0), + LondonBlock: big.NewInt(0), + TerminalTotalDifficulty: big.NewInt(17_000_000_000_000_000), + TerminalTotalDifficultyPassed: true, + MergeNetsplitBlock: big.NewInt(1735371), + Ethash: new(EthashConfig), } // SepoliaTrustedCheckpoint contains the light client trusted checkpoint for the Sepolia test network. @@ -263,16 +265,16 @@ var ( // // This configuration is intentionally not using keyed fields to force anyone // adding flags to the config to also have to set these fields. - AllEthashProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, nil, nil, new(EthashConfig), nil} + AllEthashProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, nil, nil, false, new(EthashConfig), nil} // AllCliqueProtocolChanges contains every protocol change (EIPs) introduced // and accepted by the Ethereum core developers into the Clique consensus. // // This configuration is intentionally not using keyed fields to force anyone // adding flags to the config to also have to set these fields. - AllCliqueProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, nil, nil, nil, nil, nil, &CliqueConfig{Period: 0, Epoch: 30000}} + AllCliqueProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, nil, nil, nil, nil, false, nil, &CliqueConfig{Period: 0, Epoch: 30000}} - TestChainConfig = &ChainConfig{big.NewInt(1), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, nil, nil, new(EthashConfig), nil} + TestChainConfig = &ChainConfig{big.NewInt(1), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, nil, nil, false, new(EthashConfig), nil} TestRules = TestChainConfig.Rules(new(big.Int), false) ) @@ -370,6 +372,11 @@ type ChainConfig struct { // the network that triggers the consensus upgrade. TerminalTotalDifficulty *big.Int `json:"terminalTotalDifficulty,omitempty"` + // TerminalTotalDifficultyPassed is a flag specifying that the network already + // passed the terminal total difficulty. Its purpose is to disable legacy sync + // even without having seen the TTD locally (safer long term). + TerminalTotalDifficultyPassed bool `json:"terminalTotalDifficultyPassed,omitempty"` + // Various consensus engines Ethash *EthashConfig `json:"ethash,omitempty"` Clique *CliqueConfig `json:"clique,omitempty"` @@ -408,12 +415,16 @@ func (c *ChainConfig) String() string { case c.Ethash != nil: if c.TerminalTotalDifficulty == nil { banner += "Consensus: Ethash (proof-of-work)\n" + } else if !c.TerminalTotalDifficultyPassed { + banner += "Consensus: Beacon (proof-of-stake), merging from Ethash (proof-of-work)\n" } else { banner += "Consensus: Beacon (proof-of-stake), merged from Ethash (proof-of-work)\n" } case c.Clique != nil: if c.TerminalTotalDifficulty == nil { banner += "Consensus: Clique (proof-of-authority)\n" + } else if !c.TerminalTotalDifficultyPassed { + banner += "Consensus: Beacon (proof-of-stake), merging from Clique (proof-of-authority)\n" } else { banner += "Consensus: Beacon (proof-of-stake), merged from Clique (proof-of-authority)\n" } @@ -462,9 +473,10 @@ func (c *ChainConfig) String() string { banner += " - Hard-fork specification: https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/paris.md)" } else { banner += "Merge configured:\n" - banner += " - Hard-fork specification: https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/paris.md)\n" - banner += fmt.Sprintf(" - Total terminal difficulty: %v\n", c.TerminalTotalDifficulty) - banner += fmt.Sprintf(" - Merge netsplit block: %-8v", c.MergeNetsplitBlock) + banner += " - Hard-fork specification: https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/paris.md)\n" + banner += fmt.Sprintf(" - Network known to be merged: %v\n", c.TerminalTotalDifficultyPassed) + banner += fmt.Sprintf(" - Total terminal difficulty: %v\n", c.TerminalTotalDifficulty) + banner += fmt.Sprintf(" - Merge netsplit block: %-8v", c.MergeNetsplitBlock) } return banner } From 1b34ed2ed6c1d0b56891b5bd9b1850d9d31e1038 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi <1591639+s1na@users.noreply.github.com> Date: Mon, 1 Aug 2022 16:33:39 +0200 Subject: [PATCH 13/59] eth: fix typo in catalyst api (#25460) eth: fix typo --- eth/catalyst/api.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eth/catalyst/api.go b/eth/catalyst/api.go index 93b78a386..6b5f94646 100644 --- a/eth/catalyst/api.go +++ b/eth/catalyst/api.go @@ -641,9 +641,9 @@ func (api *ConsensusAPI) heartbeat() { } } else { if lastTransitionUpdate.IsZero() { - message = "Merge is configured, but no beacon client seen. Please ensure you have one available before the transision arrives!" + message = "Merge is configured, but no beacon client seen. Please ensure you have one available before the transition arrives!" } else { - message = "Merge is configured, but previously seen beacon client is offline. Please ensure it is operational before the transision arrives!" + message = "Merge is configured, but previously seen beacon client is offline. Please ensure it is operational before the transition arrives!" } } if eta == 0 { From 93eabcaa4ea93fb3a01bfa69ef54a772616d89ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marius=20Kj=C3=A6rstad?= Date: Mon, 1 Aug 2022 18:21:58 +0200 Subject: [PATCH 14/59] build: upgrade -dlgo version to Go 1.18.5 --- build/checksums.txt | 28 ++++++++++++++-------------- build/ci.go | 2 +- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/build/checksums.txt b/build/checksums.txt index 1a8f3cd76..f7b13a033 100644 --- a/build/checksums.txt +++ b/build/checksums.txt @@ -1,19 +1,19 @@ # This file contains sha256 checksums of optional build dependencies. -4525aa6b0e3cecb57845f4060a7075aafc9ab752bb7b6b4cf8a212d43078e1e4 go1.18.4.src.tar.gz -315e1a2b21a827c68da1b7f492b5dcbe81d8df8a79ebe50922df9588893f87f0 go1.18.4.darwin-amd64.tar.gz -04eed623d5143ffa44965b618b509e0beccccfd3a4a1bfebc0cdbcf906046769 go1.18.4.darwin-arm64.tar.gz -e5244fdcd6b6eaf785dbd8c6e02b4804a4d00409e7edecc63cd59fc8f37c34c5 go1.18.4.freebsd-386.tar.gz -fb00f8aaffcc80e0a2bd39db1d8e8e21ef0a691c564f7b7601383dd6adad4042 go1.18.4.freebsd-amd64.tar.gz -418232d905e18ece6cb13c4884bb1c68963d7d3b4d889671b3e5be8bd4059862 go1.18.4.linux-386.tar.gz -c9b099b68d93f5c5c8a8844a89f8db07eaa58270e3a1e01804f17f4cf8df02f5 go1.18.4.linux-amd64.tar.gz -35014d92b50d97da41dade965df7ebeb9a715da600206aa59ce1b2d05527421f go1.18.4.linux-arm64.tar.gz -7dfeab572e49638b0f3d9901457f0622c27b73301c2b99db9f5e9568ff40460c go1.18.4.linux-armv6l.tar.gz -f80acc4dc054ddc89ccc4869664e331bf16e0ac6e07830e94554162e66f66961 go1.18.4.linux-ppc64le.tar.gz -7e932f36e8f347feea2e706dcd32c1a464b1e5767ab2928ae460a37a975fe4a3 go1.18.4.linux-s390x.tar.gz -6343010a13ab783e553786b3cc3b4d63080128f61cf1e963505139c71ca66a0d go1.18.4.windows-386.zip -dfb93c517e050ba0cfc066802b38a8e7cda2ef666efd634859356b33f543cc49 go1.18.4.windows-amd64.zip -7d0d7b73592019d276f2bd44ee3cda0d8bd99356fdbf04fdb40c263518108ae4 go1.18.4.windows-arm64.zip +9920d3306a1ac536cdd2c796d6cb3c54bc559c226fc3cc39c32f1e0bd7f50d2a go1.18.5.src.tar.gz +828eeca8b5abea3e56921df8fa4b1101380a5ebcfee10acbc8ffe7ec0bf5876b go1.18.5.darwin-amd64.tar.gz +923a377c6fc9a2c789f5db61c24b8f64133f7889056897449891f256af34065f go1.18.5.darwin-arm64.tar.gz +c3d90264a706e2d88cfb44126dc6f0d008a48f00732e04bc377cea1a2b716a7c go1.18.5.freebsd-386.tar.gz +0de23843c568d388bc0f0e390a8966938cccaae0d74b698325f7175bac04e0c6 go1.18.5.freebsd-amd64.tar.gz +0c44f85d146c6f98c34e8ff436a42af22e90e36fe232d3d9d3101f23fd61362b go1.18.5.linux-386.tar.gz +9e5de37f9c49942c601b191ac5fba404b868bfc21d446d6960acc12283d6e5f2 go1.18.5.linux-amd64.tar.gz +006f6622718212363fa1ff004a6ab4d87bbbe772ec5631bab7cac10be346e4f1 go1.18.5.linux-arm64.tar.gz +d5ac34ac5f060a5274319aa04b7b11e41b123bd7887d64efb5f44ead236957af go1.18.5.linux-armv6l.tar.gz +2e37fb9c7cbaedd4e729492d658aa4cde821fc94117391a8105c13b25ca1c84b go1.18.5.linux-ppc64le.tar.gz +e3d536e7873639f85353e892444f83b14cb6670603961f215986ae8e28e8e07a go1.18.5.linux-s390x.tar.gz +7b3142ec0c5db991e7f73a231662a92429b90ee151fe47557acb566d8d9ae4d3 go1.18.5.windows-386.zip +73753620602d4b4469770040c53db55e5dd6af2ad07ecc18f71f164c3224eaad go1.18.5.windows-amd64.zip +4d154626affff12ef73ea1017af0e5b52dbc839ef92f6f9e76cf4f71278a5744 go1.18.5.windows-arm64.zip 658078aaaf7608693f37c4cf1380b2af418ab8b2d23fdb33e7e2d4339328590e golangci-lint-1.46.2-darwin-amd64.tar.gz 81f9b4afd62ec5e612ef8bc3b1d612a88b56ff289874831845cdad394427385f golangci-lint-1.46.2-darwin-arm64.tar.gz diff --git a/build/ci.go b/build/ci.go index edf97cdd0..526e76f38 100644 --- a/build/ci.go +++ b/build/ci.go @@ -149,7 +149,7 @@ var ( // This is the version of go that will be downloaded by // // go run ci.go install -dlgo - dlgoVersion = "1.18.4" + dlgoVersion = "1.18.5" ) var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin")) From a0b88ce869d6be243c6b1900d69b9401f2d2c092 Mon Sep 17 00:00:00 2001 From: lightclient <14004106+lightclient@users.noreply.github.com> Date: Mon, 1 Aug 2022 10:13:30 -0700 Subject: [PATCH 15/59] eth/gasprice/feehistory: support finalized block (#25442) --- eth/gasprice/feehistory.go | 80 +++++++++++++++++++++------------ eth/gasprice/feehistory_test.go | 2 + eth/gasprice/gasprice_test.go | 21 +++++++++ 3 files changed, 75 insertions(+), 28 deletions(-) diff --git a/eth/gasprice/feehistory.go b/eth/gasprice/feehistory.go index 4113089af..91835c164 100644 --- a/eth/gasprice/feehistory.go +++ b/eth/gasprice/feehistory.go @@ -137,44 +137,68 @@ func (oracle *Oracle) processBlock(bf *blockFees, percentiles []float64) { // also returned if requested and available. // Note: an error is only returned if retrieving the head header has failed. If there are no // retrievable blocks in the specified range then zero block count is returned with no error. -func (oracle *Oracle) resolveBlockRange(ctx context.Context, lastBlock rpc.BlockNumber, blocks int) (*types.Block, []*types.Receipt, uint64, int, error) { +func (oracle *Oracle) resolveBlockRange(ctx context.Context, reqEnd rpc.BlockNumber, blocks int) (*types.Block, []*types.Receipt, uint64, int, error) { var ( - headBlock rpc.BlockNumber + headBlock *types.Header pendingBlock *types.Block pendingReceipts types.Receipts + err error ) - // query either pending block or head header and set headBlock - if lastBlock == rpc.PendingBlockNumber { - if pendingBlock, pendingReceipts = oracle.backend.PendingBlockAndReceipts(); pendingBlock != nil { - lastBlock = rpc.BlockNumber(pendingBlock.NumberU64()) - headBlock = lastBlock - 1 - } else { - // pending block not supported by backend, process until latest block - lastBlock = rpc.LatestBlockNumber - blocks-- - if blocks == 0 { - return nil, nil, 0, 0, nil - } - } + + // Get the chain's current head. + if headBlock, err = oracle.backend.HeaderByNumber(ctx, rpc.LatestBlockNumber); err != nil { + return nil, nil, 0, 0, err } - if pendingBlock == nil { - // if pending block is not fetched then we retrieve the head header to get the head block number - if latestHeader, err := oracle.backend.HeaderByNumber(ctx, rpc.LatestBlockNumber); err == nil { - headBlock = rpc.BlockNumber(latestHeader.Number.Uint64()) - } else { + head := rpc.BlockNumber(headBlock.Number.Uint64()) + + // Fail if request block is beyond the chain's current head. + if head < reqEnd { + return nil, nil, 0, 0, fmt.Errorf("%w: requested %d, head %d", errRequestBeyondHead, reqEnd, head) + } + + // Resolve block tag. + if reqEnd < 0 { + var ( + resolved *types.Header + err error + ) + switch reqEnd { + case rpc.PendingBlockNumber: + if pendingBlock, pendingReceipts = oracle.backend.PendingBlockAndReceipts(); pendingBlock != nil { + resolved = pendingBlock.Header() + } else { + // Pending block not supported by backend, process only until latest block. + resolved = headBlock + + // Update total blocks to return to account for this. + blocks-- + } + case rpc.LatestBlockNumber: + // Retrieved above. + resolved = headBlock + case rpc.SafeBlockNumber: + resolved, err = oracle.backend.HeaderByNumber(ctx, rpc.SafeBlockNumber) + case rpc.FinalizedBlockNumber: + resolved, err = oracle.backend.HeaderByNumber(ctx, rpc.FinalizedBlockNumber) + case rpc.EarliestBlockNumber: + resolved, err = oracle.backend.HeaderByNumber(ctx, rpc.EarliestBlockNumber) + } + if resolved == nil || err != nil { return nil, nil, 0, 0, err } + // Absolute number resolved. + reqEnd = rpc.BlockNumber(resolved.Number.Uint64()) } - if lastBlock == rpc.LatestBlockNumber { - lastBlock = headBlock - } else if pendingBlock == nil && lastBlock > headBlock { - return nil, nil, 0, 0, fmt.Errorf("%w: requested %d, head %d", errRequestBeyondHead, lastBlock, headBlock) + + // If there are no blocks to return, short circuit. + if blocks == 0 { + return nil, nil, 0, 0, nil } - // ensure not trying to retrieve before genesis - if rpc.BlockNumber(blocks) > lastBlock+1 { - blocks = int(lastBlock + 1) + // Ensure not trying to retrieve before genesis. + if int(reqEnd+1) < blocks { + blocks = int(reqEnd + 1) } - return pendingBlock, pendingReceipts, uint64(lastBlock), blocks, nil + return pendingBlock, pendingReceipts, uint64(reqEnd), blocks, nil } // FeeHistory returns data relevant for fee estimation based on the specified range of blocks. diff --git a/eth/gasprice/feehistory_test.go b/eth/gasprice/feehistory_test.go index c259eb0ac..deece7f46 100644 --- a/eth/gasprice/feehistory_test.go +++ b/eth/gasprice/feehistory_test.go @@ -50,6 +50,8 @@ func TestFeeHistory(t *testing.T) { {false, 1000, 1000, 2, rpc.PendingBlockNumber, nil, 32, 1, nil}, {true, 1000, 1000, 2, rpc.PendingBlockNumber, nil, 32, 2, nil}, {true, 1000, 1000, 2, rpc.PendingBlockNumber, []float64{0, 10}, 32, 2, nil}, + {false, 1000, 1000, 2, rpc.FinalizedBlockNumber, []float64{0, 10}, 24, 2, nil}, + {false, 1000, 1000, 2, rpc.SafeBlockNumber, []float64{0, 10}, 24, 2, nil}, } for i, c := range cases { config := Config{ diff --git a/eth/gasprice/gasprice_test.go b/eth/gasprice/gasprice_test.go index 95a908fc1..3806e993c 100644 --- a/eth/gasprice/gasprice_test.go +++ b/eth/gasprice/gasprice_test.go @@ -45,6 +45,15 @@ func (b *testBackend) HeaderByNumber(ctx context.Context, number rpc.BlockNumber if number > testHead { return nil, nil } + if number == rpc.EarliestBlockNumber { + number = 0 + } + if number == rpc.FinalizedBlockNumber { + return b.chain.CurrentFinalizedBlock().Header(), nil + } + if number == rpc.SafeBlockNumber { + return b.chain.CurrentSafeBlock().Header(), nil + } if number == rpc.LatestBlockNumber { number = testHead } @@ -62,6 +71,15 @@ func (b *testBackend) BlockByNumber(ctx context.Context, number rpc.BlockNumber) if number > testHead { return nil, nil } + if number == rpc.EarliestBlockNumber { + number = 0 + } + if number == rpc.FinalizedBlockNumber { + return b.chain.CurrentFinalizedBlock(), nil + } + if number == rpc.SafeBlockNumber { + return b.chain.CurrentSafeBlock(), nil + } if number == rpc.LatestBlockNumber { number = testHead } @@ -109,6 +127,7 @@ func newTestBackend(t *testing.T, londonBlock *big.Int, pending bool) *testBacke config.LondonBlock = londonBlock config.ArrowGlacierBlock = londonBlock config.GrayGlacierBlock = londonBlock + config.TerminalTotalDifficulty = common.Big0 engine := ethash.NewFaker() db := rawdb.NewMemoryDatabase() genesis, err := gspec.Commit(db) @@ -150,6 +169,8 @@ func newTestBackend(t *testing.T, londonBlock *big.Int, pending bool) *testBacke t.Fatalf("Failed to create local chain, %v", err) } chain.InsertChain(blocks) + chain.SetFinalized(chain.GetBlockByNumber(25)) + chain.SetSafe(chain.GetBlockByNumber(25)) return &testBackend{chain: chain, pending: pending} } From 6fdc619413980f90770f13339a5d79a4672779ae Mon Sep 17 00:00:00 2001 From: Manoj Kumar Date: Tue, 2 Aug 2022 18:03:23 +0530 Subject: [PATCH 16/59] consensus/ethash: remove temp files created during DAG generation (#25381) This makes it remove not only the actual DAG file, but also the temporary file which the DAG data is written to while generating. --- consensus/ethash/ethash.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/consensus/ethash/ethash.go b/consensus/ethash/ethash.go index 0efb3590f..dfe00d4b9 100644 --- a/consensus/ethash/ethash.go +++ b/consensus/ethash/ethash.go @@ -278,8 +278,11 @@ func (c *cache) generate(dir string, limit int, lock bool, test bool) { // Iterate over all previous instances and delete old ones for ep := int(c.epoch) - limit; ep >= 0; ep-- { seed := seedHash(uint64(ep)*epochLength + 1) - path := filepath.Join(dir, fmt.Sprintf("cache-R%d-%x%s", algorithmRevision, seed[:8], endian)) - os.Remove(path) + path := filepath.Join(dir, fmt.Sprintf("cache-R%d-%x%s*", algorithmRevision, seed[:8], endian)) + files, _ := filepath.Glob(path) // find also the temp files that are generated. + for _, file := range files { + os.Remove(file) + } } }) } From d804a59ee1a0152e98bb8dbd5cfea522409a2a5e Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 2 Aug 2022 14:48:55 +0200 Subject: [PATCH 17/59] cmd/devp2p/internal/ethtest: update tests for eth/67 (#25306) --- cmd/devp2p/internal/ethtest/chain.go | 4 +- cmd/devp2p/internal/ethtest/chain_test.go | 41 +- cmd/devp2p/internal/ethtest/helpers.go | 324 ++++---------- cmd/devp2p/internal/ethtest/snapTypes.go | 24 +- cmd/devp2p/internal/ethtest/suite.go | 486 ++++++--------------- cmd/devp2p/internal/ethtest/suite_test.go | 2 +- cmd/devp2p/internal/ethtest/transaction.go | 50 +-- cmd/devp2p/internal/ethtest/types.go | 153 +++---- cmd/devp2p/rlpxcmd.go | 8 +- 9 files changed, 333 insertions(+), 759 deletions(-) diff --git a/cmd/devp2p/internal/ethtest/chain.go b/cmd/devp2p/internal/ethtest/chain.go index 65ffc6f81..df724f470 100644 --- a/cmd/devp2p/internal/ethtest/chain.go +++ b/cmd/devp2p/internal/ethtest/chain.go @@ -96,12 +96,12 @@ func (c *Chain) Head() *types.Block { return c.blocks[c.Len()-1] } -func (c *Chain) GetHeaders(req GetBlockHeaders) (BlockHeaders, error) { +func (c *Chain) GetHeaders(req *GetBlockHeaders) ([]*types.Header, error) { if req.Amount < 1 { return nil, fmt.Errorf("no block headers requested") } - headers := make(BlockHeaders, req.Amount) + headers := make([]*types.Header, req.Amount) var blockNumber uint64 // range over blocks to check if our chain has the requested header diff --git a/cmd/devp2p/internal/ethtest/chain_test.go b/cmd/devp2p/internal/ethtest/chain_test.go index 0f232b150..67221923a 100644 --- a/cmd/devp2p/internal/ethtest/chain_test.go +++ b/cmd/devp2p/internal/ethtest/chain_test.go @@ -21,6 +21,7 @@ import ( "strconv" "testing" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/p2p" "github.com/stretchr/testify/assert" @@ -140,18 +141,18 @@ func TestChain_GetHeaders(t *testing.T) { var tests = []struct { req GetBlockHeaders - expected BlockHeaders + expected []*types.Header }{ { req: GetBlockHeaders{ - Origin: eth.HashOrNumber{ - Number: uint64(2), + GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ + Origin: eth.HashOrNumber{Number: uint64(2)}, + Amount: uint64(5), + Skip: 1, + Reverse: false, }, - Amount: uint64(5), - Skip: 1, - Reverse: false, }, - expected: BlockHeaders{ + expected: []*types.Header{ chain.blocks[2].Header(), chain.blocks[4].Header(), chain.blocks[6].Header(), @@ -161,14 +162,14 @@ func TestChain_GetHeaders(t *testing.T) { }, { req: GetBlockHeaders{ - Origin: eth.HashOrNumber{ - Number: uint64(chain.Len() - 1), + GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ + Origin: eth.HashOrNumber{Number: uint64(chain.Len() - 1)}, + Amount: uint64(3), + Skip: 0, + Reverse: true, }, - Amount: uint64(3), - Skip: 0, - Reverse: true, }, - expected: BlockHeaders{ + expected: []*types.Header{ chain.blocks[chain.Len()-1].Header(), chain.blocks[chain.Len()-2].Header(), chain.blocks[chain.Len()-3].Header(), @@ -176,14 +177,14 @@ func TestChain_GetHeaders(t *testing.T) { }, { req: GetBlockHeaders{ - Origin: eth.HashOrNumber{ - Hash: chain.Head().Hash(), + GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ + Origin: eth.HashOrNumber{Hash: chain.Head().Hash()}, + Amount: uint64(1), + Skip: 0, + Reverse: false, }, - Amount: uint64(1), - Skip: 0, - Reverse: false, }, - expected: BlockHeaders{ + expected: []*types.Header{ chain.Head().Header(), }, }, @@ -191,7 +192,7 @@ func TestChain_GetHeaders(t *testing.T) { for i, tt := range tests { t.Run(strconv.Itoa(i), func(t *testing.T) { - headers, err := chain.GetHeaders(tt.req) + headers, err := chain.GetHeaders(&tt.req) if err != nil { t.Fatal(err) } diff --git a/cmd/devp2p/internal/ethtest/helpers.go b/cmd/devp2p/internal/ethtest/helpers.go index df754d6ce..eeeb4f93c 100644 --- a/cmd/devp2p/internal/ethtest/helpers.go +++ b/cmd/devp2p/internal/ethtest/helpers.go @@ -43,21 +43,6 @@ var ( timeout = 20 * time.Second ) -// Is_66 checks if the node supports the eth66 protocol version, -// and if not, exists the test suite -func (s *Suite) Is_66(t *utesting.T) { - conn, err := s.dial66() - if err != nil { - t.Fatalf("dial failed: %v", err) - } - if err := conn.handshake(); err != nil { - t.Fatalf("handshake failed: %v", err) - } - if conn.negotiatedProtoVersion < 66 { - t.Fail() - } -} - // dial attempts to dial the given node and perform a handshake, // returning the created Conn if successful. func (s *Suite) dial() (*Conn, error) { @@ -76,31 +61,16 @@ func (s *Suite) dial() (*Conn, error) { } // set default p2p capabilities conn.caps = []p2p.Cap{ - {Name: "eth", Version: 64}, - {Name: "eth", Version: 65}, + {Name: "eth", Version: 66}, + {Name: "eth", Version: 67}, } - conn.ourHighestProtoVersion = 65 + conn.ourHighestProtoVersion = 67 return &conn, nil } -// dial66 attempts to dial the given node and perform a handshake, -// returning the created Conn with additional eth66 capabilities if -// successful -func (s *Suite) dial66() (*Conn, error) { - conn, err := s.dial() - if err != nil { - return nil, fmt.Errorf("dial failed: %v", err) - } - conn.caps = append(conn.caps, p2p.Cap{Name: "eth", Version: 66}) - conn.ourHighestProtoVersion = 66 - return conn, nil -} - -// dial66 attempts to dial the given node and perform a handshake, -// returning the created Conn with additional snap/1 capabilities if -// successful. +// dialSnap creates a connection with snap/1 capability. func (s *Suite) dialSnap() (*Conn, error) { - conn, err := s.dial66() + conn, err := s.dial() if err != nil { return nil, fmt.Errorf("dial failed: %v", err) } @@ -235,60 +205,40 @@ loop: // createSendAndRecvConns creates two connections, one for sending messages to the // node, and one for receiving messages from the node. -func (s *Suite) createSendAndRecvConns(isEth66 bool) (*Conn, *Conn, error) { - var ( - sendConn *Conn - recvConn *Conn - err error - ) - if isEth66 { - sendConn, err = s.dial66() - if err != nil { - return nil, nil, fmt.Errorf("dial failed: %v", err) - } - recvConn, err = s.dial66() - if err != nil { - sendConn.Close() - return nil, nil, fmt.Errorf("dial failed: %v", err) - } - } else { - sendConn, err = s.dial() - if err != nil { - return nil, nil, fmt.Errorf("dial failed: %v", err) - } - recvConn, err = s.dial() - if err != nil { - sendConn.Close() - return nil, nil, fmt.Errorf("dial failed: %v", err) - } +func (s *Suite) createSendAndRecvConns() (*Conn, *Conn, error) { + sendConn, err := s.dial() + if err != nil { + return nil, nil, fmt.Errorf("dial failed: %v", err) + } + recvConn, err := s.dial() + if err != nil { + sendConn.Close() + return nil, nil, fmt.Errorf("dial failed: %v", err) } return sendConn, recvConn, nil } -func (c *Conn) readAndServe(chain *Chain, timeout time.Duration) Message { - if c.negotiatedProtoVersion == 66 { - _, msg := c.readAndServe66(chain, timeout) - return msg - } - return c.readAndServe65(chain, timeout) -} - // readAndServe serves GetBlockHeaders requests while waiting // on another message from the node. -func (c *Conn) readAndServe65(chain *Chain, timeout time.Duration) Message { +func (c *Conn) readAndServe(chain *Chain, timeout time.Duration) Message { start := time.Now() for time.Since(start) < timeout { - c.SetReadDeadline(time.Now().Add(5 * time.Second)) - switch msg := c.Read().(type) { + c.SetReadDeadline(time.Now().Add(10 * time.Second)) + + msg := c.Read() + switch msg := msg.(type) { case *Ping: c.Write(&Pong{}) case *GetBlockHeaders: - req := *msg - headers, err := chain.GetHeaders(req) + headers, err := chain.GetHeaders(msg) if err != nil { return errorf("could not get headers for inbound header request: %v", err) } - if err := c.Write(headers); err != nil { + resp := &BlockHeaders{ + RequestId: msg.ReqID(), + BlockHeadersPacket: eth.BlockHeadersPacket(headers), + } + if err := c.Write(resp); err != nil { return errorf("could not write to connection: %v", err) } default: @@ -298,54 +248,25 @@ func (c *Conn) readAndServe65(chain *Chain, timeout time.Duration) Message { return errorf("no message received within %v", timeout) } -// readAndServe66 serves eth66 GetBlockHeaders requests while waiting -// on another message from the node. -func (c *Conn) readAndServe66(chain *Chain, timeout time.Duration) (uint64, Message) { - start := time.Now() - for time.Since(start) < timeout { - c.SetReadDeadline(time.Now().Add(10 * time.Second)) - - reqID, msg := c.Read66() - - switch msg := msg.(type) { - case *Ping: - c.Write(&Pong{}) - case GetBlockHeaders: - headers, err := chain.GetHeaders(msg) - if err != nil { - return 0, errorf("could not get headers for inbound header request: %v", err) - } - resp := ð.BlockHeadersPacket66{ - RequestId: reqID, - BlockHeadersPacket: eth.BlockHeadersPacket(headers), - } - if err := c.Write66(resp, BlockHeaders{}.Code()); err != nil { - return 0, errorf("could not write to connection: %v", err) - } - default: - return reqID, msg - } - } - return 0, errorf("no message received within %v", timeout) -} - // headersRequest executes the given `GetBlockHeaders` request. -func (c *Conn) headersRequest(request *GetBlockHeaders, chain *Chain, isEth66 bool, reqID uint64) (BlockHeaders, error) { +func (c *Conn) headersRequest(request *GetBlockHeaders, chain *Chain, reqID uint64) ([]*types.Header, error) { defer c.SetReadDeadline(time.Time{}) c.SetReadDeadline(time.Now().Add(20 * time.Second)) - // if on eth66 connection, perform eth66 GetBlockHeaders request - if isEth66 { - return getBlockHeaders66(chain, c, request, reqID) - } + + // write request + request.RequestId = reqID if err := c.Write(request); err != nil { - return nil, err + return nil, fmt.Errorf("could not write to connection: %v", err) } - switch msg := c.readAndServe(chain, timeout).(type) { - case *BlockHeaders: - return *msg, nil - default: - return nil, fmt.Errorf("invalid message: %s", pretty.Sdump(msg)) + + // wait for response + msg := c.waitForResponse(chain, timeout, request.RequestId) + resp, ok := msg.(*BlockHeaders) + if !ok { + return nil, fmt.Errorf("unexpected message received: %s", pretty.Sdump(msg)) } + headers := []*types.Header(resp.BlockHeadersPacket) + return headers, nil } func (c *Conn) snapRequest(msg Message, id uint64, chain *Chain) (Message, error) { @@ -357,28 +278,8 @@ func (c *Conn) snapRequest(msg Message, id uint64, chain *Chain) (Message, error return c.ReadSnap(id) } -// getBlockHeaders66 executes the given `GetBlockHeaders` request over the eth66 protocol. -func getBlockHeaders66(chain *Chain, conn *Conn, request *GetBlockHeaders, id uint64) (BlockHeaders, error) { - // write request - packet := eth.GetBlockHeadersPacket(*request) - req := ð.GetBlockHeadersPacket66{ - RequestId: id, - GetBlockHeadersPacket: &packet, - } - if err := conn.Write66(req, GetBlockHeaders{}.Code()); err != nil { - return nil, fmt.Errorf("could not write to connection: %v", err) - } - // wait for response - msg := conn.waitForResponse(chain, timeout, req.RequestId) - headers, ok := msg.(BlockHeaders) - if !ok { - return nil, fmt.Errorf("unexpected message received: %s", pretty.Sdump(msg)) - } - return headers, nil -} - // headersMatch returns whether the received headers match the given request -func headersMatch(expected BlockHeaders, headers BlockHeaders) bool { +func headersMatch(expected []*types.Header, headers []*types.Header) bool { return reflect.DeepEqual(expected, headers) } @@ -386,8 +287,8 @@ func headersMatch(expected BlockHeaders, headers BlockHeaders) bool { // request ID is received. func (c *Conn) waitForResponse(chain *Chain, timeout time.Duration, requestID uint64) Message { for { - id, msg := c.readAndServe66(chain, timeout) - if id == requestID { + msg := c.readAndServe(chain, timeout) + if msg.ReqID() == requestID { return msg } } @@ -395,9 +296,9 @@ func (c *Conn) waitForResponse(chain *Chain, timeout time.Duration, requestID ui // sendNextBlock broadcasts the next block in the chain and waits // for the node to propagate the block and import it into its chain. -func (s *Suite) sendNextBlock(isEth66 bool) error { +func (s *Suite) sendNextBlock() error { // set up sending and receiving connections - sendConn, recvConn, err := s.createSendAndRecvConns(isEth66) + sendConn, recvConn, err := s.createSendAndRecvConns() if err != nil { return err } @@ -420,7 +321,7 @@ func (s *Suite) sendNextBlock(isEth66 bool) error { return fmt.Errorf("failed to announce block: %v", err) } // wait for client to update its chain - if err = s.waitForBlockImport(recvConn, nextBlock, isEth66); err != nil { + if err = s.waitForBlockImport(recvConn, nextBlock); err != nil { return fmt.Errorf("failed to receive confirmation of block import: %v", err) } // update test suite chain @@ -465,29 +366,22 @@ func (s *Suite) waitAnnounce(conn *Conn, blockAnnouncement *NewBlock) error { } } -func (s *Suite) waitForBlockImport(conn *Conn, block *types.Block, isEth66 bool) error { +func (s *Suite) waitForBlockImport(conn *Conn, block *types.Block) error { defer conn.SetReadDeadline(time.Time{}) conn.SetReadDeadline(time.Now().Add(20 * time.Second)) // create request req := &GetBlockHeaders{ - Origin: eth.HashOrNumber{ - Hash: block.Hash(), + GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ + Origin: eth.HashOrNumber{Hash: block.Hash()}, + Amount: 1, }, - Amount: 1, } + // loop until BlockHeaders response contains desired block, confirming the // node imported the block for { - var ( - headers BlockHeaders - err error - ) - if isEth66 { - requestID := uint64(54) - headers, err = conn.headersRequest(req, s.chain, eth66, requestID) - } else { - headers, err = conn.headersRequest(req, s.chain, eth65, 0) - } + requestID := uint64(54) + headers, err := conn.headersRequest(req, s.chain, requestID) if err != nil { return fmt.Errorf("GetBlockHeader request failed: %v", err) } @@ -503,8 +397,8 @@ func (s *Suite) waitForBlockImport(conn *Conn, block *types.Block, isEth66 bool) } } -func (s *Suite) oldAnnounce(isEth66 bool) error { - sendConn, receiveConn, err := s.createSendAndRecvConns(isEth66) +func (s *Suite) oldAnnounce() error { + sendConn, receiveConn, err := s.createSendAndRecvConns() if err != nil { return err } @@ -550,23 +444,13 @@ func (s *Suite) oldAnnounce(isEth66 bool) error { return nil } -func (s *Suite) maliciousHandshakes(t *utesting.T, isEth66 bool) error { - var ( - conn *Conn - err error - ) - if isEth66 { - conn, err = s.dial66() - if err != nil { - return fmt.Errorf("dial failed: %v", err) - } - } else { - conn, err = s.dial() - if err != nil { - return fmt.Errorf("dial failed: %v", err) - } +func (s *Suite) maliciousHandshakes(t *utesting.T) error { + conn, err := s.dial() + if err != nil { + return fmt.Errorf("dial failed: %v", err) } defer conn.Close() + // write hello to client pub0 := crypto.FromECDSAPub(&conn.ourKey.PublicKey)[1:] handshakes := []*Hello{ @@ -627,16 +511,9 @@ func (s *Suite) maliciousHandshakes(t *utesting.T, isEth66 bool) error { } } // dial for the next round - if isEth66 { - conn, err = s.dial66() - if err != nil { - return fmt.Errorf("dial failed: %v", err) - } - } else { - conn, err = s.dial() - if err != nil { - return fmt.Errorf("dial failed: %v", err) - } + conn, err = s.dial() + if err != nil { + return fmt.Errorf("dial failed: %v", err) } } return nil @@ -654,6 +531,7 @@ func (s *Suite) maliciousStatus(conn *Conn) error { Genesis: s.chain.blocks[0].Hash(), ForkID: s.chain.ForkID(), } + // get status msg, err := conn.statusExchange(s.chain, status) if err != nil { @@ -664,6 +542,7 @@ func (s *Suite) maliciousStatus(conn *Conn) error { default: return fmt.Errorf("expected status, got: %#v ", msg) } + // wait for disconnect switch msg := conn.readAndServe(s.chain, timeout).(type) { case *Disconnect: @@ -675,9 +554,9 @@ func (s *Suite) maliciousStatus(conn *Conn) error { } } -func (s *Suite) hashAnnounce(isEth66 bool) error { +func (s *Suite) hashAnnounce() error { // create connections - sendConn, recvConn, err := s.createSendAndRecvConns(isEth66) + sendConn, recvConn, err := s.createSendAndRecvConns() if err != nil { return fmt.Errorf("failed to create connections: %v", err) } @@ -689,6 +568,7 @@ func (s *Suite) hashAnnounce(isEth66 bool) error { if err := recvConn.peer(s.chain, nil); err != nil { return fmt.Errorf("peering failed: %v", err) } + // create NewBlockHashes announcement type anno struct { Hash common.Hash // Hash of one particular block being announced @@ -700,56 +580,29 @@ func (s *Suite) hashAnnounce(isEth66 bool) error { if err := sendConn.Write(newBlockHash); err != nil { return fmt.Errorf("failed to write to connection: %v", err) } + // Announcement sent, now wait for a header request - var ( - id uint64 - msg Message - blockHeaderReq GetBlockHeaders - ) - if isEth66 { - id, msg = sendConn.Read66() - switch msg := msg.(type) { - case GetBlockHeaders: - blockHeaderReq = msg - default: - return fmt.Errorf("unexpected %s", pretty.Sdump(msg)) - } - if blockHeaderReq.Amount != 1 { - return fmt.Errorf("unexpected number of block headers requested: %v", blockHeaderReq.Amount) - } - if blockHeaderReq.Origin.Hash != announcement.Hash { - return fmt.Errorf("unexpected block header requested. Announced:\n %v\n Remote request:\n%v", - pretty.Sdump(announcement), - pretty.Sdump(blockHeaderReq)) - } - if err := sendConn.Write66(ð.BlockHeadersPacket66{ - RequestId: id, - BlockHeadersPacket: eth.BlockHeadersPacket{ - nextBlock.Header(), - }, - }, BlockHeaders{}.Code()); err != nil { - return fmt.Errorf("failed to write to connection: %v", err) - } - } else { - msg = sendConn.Read() - switch msg := msg.(type) { - case *GetBlockHeaders: - blockHeaderReq = *msg - default: - return fmt.Errorf("unexpected %s", pretty.Sdump(msg)) - } - if blockHeaderReq.Amount != 1 { - return fmt.Errorf("unexpected number of block headers requested: %v", blockHeaderReq.Amount) - } - if blockHeaderReq.Origin.Hash != announcement.Hash { - return fmt.Errorf("unexpected block header requested. Announced:\n %v\n Remote request:\n%v", - pretty.Sdump(announcement), - pretty.Sdump(blockHeaderReq)) - } - if err := sendConn.Write(&BlockHeaders{nextBlock.Header()}); err != nil { - return fmt.Errorf("failed to write to connection: %v", err) - } + msg := sendConn.Read() + blockHeaderReq, ok := msg.(*GetBlockHeaders) + if !ok { + return fmt.Errorf("unexpected %s", pretty.Sdump(msg)) } + if blockHeaderReq.Amount != 1 { + return fmt.Errorf("unexpected number of block headers requested: %v", blockHeaderReq.Amount) + } + if blockHeaderReq.Origin.Hash != announcement.Hash { + return fmt.Errorf("unexpected block header requested. Announced:\n %v\n Remote request:\n%v", + pretty.Sdump(announcement), + pretty.Sdump(blockHeaderReq)) + } + err = sendConn.Write(&BlockHeaders{ + RequestId: blockHeaderReq.ReqID(), + BlockHeadersPacket: eth.BlockHeadersPacket{nextBlock.Header()}, + }) + if err != nil { + return fmt.Errorf("failed to write to connection: %v", err) + } + // wait for block announcement msg = recvConn.readAndServe(s.chain, timeout) switch msg := msg.(type) { @@ -762,6 +615,7 @@ func (s *Suite) hashAnnounce(isEth66 bool) error { return fmt.Errorf("unexpected block hash announcement, wanted %v, got %v", nextBlock.Hash(), hashes[0].Hash) } + case *NewBlock: // node should only propagate NewBlock without having requested the body if the body is empty nextBlockBody := nextBlock.Body() @@ -780,7 +634,7 @@ func (s *Suite) hashAnnounce(isEth66 bool) error { return fmt.Errorf("unexpected: %s", pretty.Sdump(msg)) } // confirm node imported block - if err := s.waitForBlockImport(recvConn, nextBlock, isEth66); err != nil { + if err := s.waitForBlockImport(recvConn, nextBlock); err != nil { return fmt.Errorf("error waiting for node to import new block: %v", err) } // update the chain diff --git a/cmd/devp2p/internal/ethtest/snapTypes.go b/cmd/devp2p/internal/ethtest/snapTypes.go index e18cd5925..6bcaa9291 100644 --- a/cmd/devp2p/internal/ethtest/snapTypes.go +++ b/cmd/devp2p/internal/ethtest/snapTypes.go @@ -21,32 +21,40 @@ import "github.com/ethereum/go-ethereum/eth/protocols/snap" // GetAccountRange represents an account range query. type GetAccountRange snap.GetAccountRangePacket -func (g GetAccountRange) Code() int { return 33 } +func (msg GetAccountRange) Code() int { return 33 } +func (msg GetAccountRange) ReqID() uint64 { return msg.ID } type AccountRange snap.AccountRangePacket -func (g AccountRange) Code() int { return 34 } +func (msg AccountRange) Code() int { return 34 } +func (msg AccountRange) ReqID() uint64 { return msg.ID } type GetStorageRanges snap.GetStorageRangesPacket -func (g GetStorageRanges) Code() int { return 35 } +func (msg GetStorageRanges) Code() int { return 35 } +func (msg GetStorageRanges) ReqID() uint64 { return msg.ID } type StorageRanges snap.StorageRangesPacket -func (g StorageRanges) Code() int { return 36 } +func (msg StorageRanges) Code() int { return 36 } +func (msg StorageRanges) ReqID() uint64 { return msg.ID } type GetByteCodes snap.GetByteCodesPacket -func (g GetByteCodes) Code() int { return 37 } +func (msg GetByteCodes) Code() int { return 37 } +func (msg GetByteCodes) ReqID() uint64 { return msg.ID } type ByteCodes snap.ByteCodesPacket -func (g ByteCodes) Code() int { return 38 } +func (msg ByteCodes) Code() int { return 38 } +func (msg ByteCodes) ReqID() uint64 { return msg.ID } type GetTrieNodes snap.GetTrieNodesPacket -func (g GetTrieNodes) Code() int { return 39 } +func (msg GetTrieNodes) Code() int { return 39 } +func (msg GetTrieNodes) ReqID() uint64 { return msg.ID } type TrieNodes snap.TrieNodesPacket -func (g TrieNodes) Code() int { return 40 } +func (msg TrieNodes) Code() int { return 40 } +func (msg TrieNodes) ReqID() uint64 { return msg.ID } diff --git a/cmd/devp2p/internal/ethtest/suite.go b/cmd/devp2p/internal/ethtest/suite.go index 4ddd65b95..7059b4ba7 100644 --- a/cmd/devp2p/internal/ethtest/suite.go +++ b/cmd/devp2p/internal/ethtest/suite.go @@ -49,79 +49,30 @@ func NewSuite(dest *enode.Node, chainfile string, genesisfile string) (*Suite, e }, nil } -func (s *Suite) AllEthTests() []utesting.Test { - return []utesting.Test{ - // status - {Name: "TestStatus65", Fn: s.TestStatus65}, - {Name: "TestStatus66", Fn: s.TestStatus66}, - // get block headers - {Name: "TestGetBlockHeaders65", Fn: s.TestGetBlockHeaders65}, - {Name: "TestGetBlockHeaders66", Fn: s.TestGetBlockHeaders66}, - {Name: "TestSimultaneousRequests66", Fn: s.TestSimultaneousRequests66}, - {Name: "TestSameRequestID66", Fn: s.TestSameRequestID66}, - {Name: "TestZeroRequestID66", Fn: s.TestZeroRequestID66}, - // get block bodies - {Name: "TestGetBlockBodies65", Fn: s.TestGetBlockBodies65}, - {Name: "TestGetBlockBodies66", Fn: s.TestGetBlockBodies66}, - // broadcast - {Name: "TestBroadcast65", Fn: s.TestBroadcast65}, - {Name: "TestBroadcast66", Fn: s.TestBroadcast66}, - {Name: "TestLargeAnnounce65", Fn: s.TestLargeAnnounce65}, - {Name: "TestLargeAnnounce66", Fn: s.TestLargeAnnounce66}, - {Name: "TestOldAnnounce65", Fn: s.TestOldAnnounce65}, - {Name: "TestOldAnnounce66", Fn: s.TestOldAnnounce66}, - {Name: "TestBlockHashAnnounce65", Fn: s.TestBlockHashAnnounce65}, - {Name: "TestBlockHashAnnounce66", Fn: s.TestBlockHashAnnounce66}, - // malicious handshakes + status - {Name: "TestMaliciousHandshake65", Fn: s.TestMaliciousHandshake65}, - {Name: "TestMaliciousStatus65", Fn: s.TestMaliciousStatus65}, - {Name: "TestMaliciousHandshake66", Fn: s.TestMaliciousHandshake66}, - {Name: "TestMaliciousStatus66", Fn: s.TestMaliciousStatus66}, - // test transactions - {Name: "TestTransaction65", Fn: s.TestTransaction65}, - {Name: "TestTransaction66", Fn: s.TestTransaction66}, - {Name: "TestMaliciousTx65", Fn: s.TestMaliciousTx65}, - {Name: "TestMaliciousTx66", Fn: s.TestMaliciousTx66}, - {Name: "TestLargeTxRequest66", Fn: s.TestLargeTxRequest66}, - {Name: "TestNewPooledTxs66", Fn: s.TestNewPooledTxs66}, - } -} - func (s *Suite) EthTests() []utesting.Test { return []utesting.Test{ - {Name: "TestStatus65", Fn: s.TestStatus65}, - {Name: "TestGetBlockHeaders65", Fn: s.TestGetBlockHeaders65}, - {Name: "TestGetBlockBodies65", Fn: s.TestGetBlockBodies65}, - {Name: "TestBroadcast65", Fn: s.TestBroadcast65}, - {Name: "TestLargeAnnounce65", Fn: s.TestLargeAnnounce65}, - {Name: "TestOldAnnounce65", Fn: s.TestOldAnnounce65}, - {Name: "TestBlockHashAnnounce65", Fn: s.TestBlockHashAnnounce65}, - {Name: "TestMaliciousHandshake65", Fn: s.TestMaliciousHandshake65}, - {Name: "TestMaliciousStatus65", Fn: s.TestMaliciousStatus65}, - {Name: "TestTransaction65", Fn: s.TestTransaction65}, - {Name: "TestMaliciousTx65", Fn: s.TestMaliciousTx65}, - } -} - -func (s *Suite) Eth66Tests() []utesting.Test { - return []utesting.Test{ - // only proceed with eth66 test suite if node supports eth 66 protocol - {Name: "TestStatus66", Fn: s.TestStatus66}, - {Name: "TestGetBlockHeaders66", Fn: s.TestGetBlockHeaders66}, - {Name: "TestSimultaneousRequests66", Fn: s.TestSimultaneousRequests66}, - {Name: "TestSameRequestID66", Fn: s.TestSameRequestID66}, - {Name: "TestZeroRequestID66", Fn: s.TestZeroRequestID66}, - {Name: "TestGetBlockBodies66", Fn: s.TestGetBlockBodies66}, - {Name: "TestBroadcast66", Fn: s.TestBroadcast66}, - {Name: "TestLargeAnnounce66", Fn: s.TestLargeAnnounce66}, - {Name: "TestOldAnnounce66", Fn: s.TestOldAnnounce66}, - {Name: "TestBlockHashAnnounce66", Fn: s.TestBlockHashAnnounce66}, - {Name: "TestMaliciousHandshake66", Fn: s.TestMaliciousHandshake66}, - {Name: "TestMaliciousStatus66", Fn: s.TestMaliciousStatus66}, - {Name: "TestTransaction66", Fn: s.TestTransaction66}, - {Name: "TestMaliciousTx66", Fn: s.TestMaliciousTx66}, - {Name: "TestLargeTxRequest66", Fn: s.TestLargeTxRequest66}, - {Name: "TestNewPooledTxs66", Fn: s.TestNewPooledTxs66}, + // status + {Name: "TestStatus", Fn: s.TestStatus}, + // get block headers + {Name: "TestGetBlockHeaders", Fn: s.TestGetBlockHeaders}, + {Name: "TestSimultaneousRequests", Fn: s.TestSimultaneousRequests}, + {Name: "TestSameRequestID", Fn: s.TestSameRequestID}, + {Name: "TestZeroRequestID", Fn: s.TestZeroRequestID}, + // get block bodies + {Name: "TestGetBlockBodies", Fn: s.TestGetBlockBodies}, + // broadcast + {Name: "TestBroadcast", Fn: s.TestBroadcast}, + {Name: "TestLargeAnnounce", Fn: s.TestLargeAnnounce}, + {Name: "TestOldAnnounce", Fn: s.TestOldAnnounce}, + {Name: "TestBlockHashAnnounce", Fn: s.TestBlockHashAnnounce}, + // malicious handshakes + status + {Name: "TestMaliciousHandshake", Fn: s.TestMaliciousHandshake}, + {Name: "TestMaliciousStatus", Fn: s.TestMaliciousStatus}, + // test transactions + {Name: "TestTransaction", Fn: s.TestTransaction}, + {Name: "TestMaliciousTx", Fn: s.TestMaliciousTx}, + {Name: "TestLargeTxRequest", Fn: s.TestLargeTxRequest}, + {Name: "TestNewPooledTxs", Fn: s.TestNewPooledTxs}, } } @@ -135,14 +86,9 @@ func (s *Suite) SnapTests() []utesting.Test { } } -var ( - eth66 = true // indicates whether suite should negotiate eth66 connection - eth65 = false // indicates whether suite should negotiate eth65 connection or below. -) - -// TestStatus65 attempts to connect to the given node and exchange -// a status message with it. -func (s *Suite) TestStatus65(t *utesting.T) { +// TestStatus attempts to connect to the given node and exchange +// a status message with it on the eth protocol. +func (s *Suite) TestStatus(t *utesting.T) { conn, err := s.dial() if err != nil { t.Fatalf("dial failed: %v", err) @@ -153,79 +99,32 @@ func (s *Suite) TestStatus65(t *utesting.T) { } } -// TestStatus66 attempts to connect to the given node and exchange -// a status message with it on the eth66 protocol. -func (s *Suite) TestStatus66(t *utesting.T) { - conn, err := s.dial66() - if err != nil { - t.Fatalf("dial failed: %v", err) - } - defer conn.Close() - if err := conn.peer(s.chain, nil); err != nil { - t.Fatalf("peering failed: %v", err) - } -} - -// TestGetBlockHeaders65 tests whether the given node can respond to -// a `GetBlockHeaders` request accurately. -func (s *Suite) TestGetBlockHeaders65(t *utesting.T) { +// TestGetBlockHeaders tests whether the given node can respond to +// an eth `GetBlockHeaders` request and that the response is accurate. +func (s *Suite) TestGetBlockHeaders(t *utesting.T) { conn, err := s.dial() if err != nil { t.Fatalf("dial failed: %v", err) } defer conn.Close() - if err := conn.peer(s.chain, nil); err != nil { - t.Fatalf("handshake(s) failed: %v", err) - } - // write request - req := &GetBlockHeaders{ - Origin: eth.HashOrNumber{ - Hash: s.chain.blocks[1].Hash(), - }, - Amount: 2, - Skip: 1, - Reverse: false, - } - headers, err := conn.headersRequest(req, s.chain, eth65, 0) - if err != nil { - t.Fatalf("GetBlockHeaders request failed: %v", err) - } - // check for correct headers - expected, err := s.chain.GetHeaders(*req) - if err != nil { - t.Fatalf("failed to get headers for given request: %v", err) - } - if !headersMatch(expected, headers) { - t.Fatalf("header mismatch: \nexpected %v \ngot %v", expected, headers) - } -} - -// TestGetBlockHeaders66 tests whether the given node can respond to -// an eth66 `GetBlockHeaders` request and that the response is accurate. -func (s *Suite) TestGetBlockHeaders66(t *utesting.T) { - conn, err := s.dial66() - if err != nil { - t.Fatalf("dial failed: %v", err) - } - defer conn.Close() if err = conn.peer(s.chain, nil); err != nil { t.Fatalf("peering failed: %v", err) } // write request req := &GetBlockHeaders{ - Origin: eth.HashOrNumber{ - Hash: s.chain.blocks[1].Hash(), + GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ + Origin: eth.HashOrNumber{Hash: s.chain.blocks[1].Hash()}, + Amount: 2, + Skip: 1, + Reverse: false, }, - Amount: 2, - Skip: 1, - Reverse: false, } - headers, err := conn.headersRequest(req, s.chain, eth66, 33) + headers, err := conn.headersRequest(req, s.chain, 33) if err != nil { t.Fatalf("could not get block headers: %v", err) } // check for correct headers - expected, err := s.chain.GetHeaders(*req) + expected, err := s.chain.GetHeaders(req) if err != nil { t.Fatalf("failed to get headers for given request: %v", err) } @@ -234,12 +133,12 @@ func (s *Suite) TestGetBlockHeaders66(t *utesting.T) { } } -// TestSimultaneousRequests66 sends two simultaneous `GetBlockHeader` requests from +// TestSimultaneousRequests sends two simultaneous `GetBlockHeader` requests from // the same connection with different request IDs and checks to make sure the node // responds with the correct headers per request. -func (s *Suite) TestSimultaneousRequests66(t *utesting.T) { +func (s *Suite) TestSimultaneousRequests(t *utesting.T) { // create a connection - conn, err := s.dial66() + conn, err := s.dial() if err != nil { t.Fatalf("dial failed: %v", err) } @@ -247,8 +146,9 @@ func (s *Suite) TestSimultaneousRequests66(t *utesting.T) { if err := conn.peer(s.chain, nil); err != nil { t.Fatalf("peering failed: %v", err) } + // create two requests - req1 := ð.GetBlockHeadersPacket66{ + req1 := &GetBlockHeaders{ RequestId: uint64(111), GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ Origin: eth.HashOrNumber{ @@ -259,7 +159,7 @@ func (s *Suite) TestSimultaneousRequests66(t *utesting.T) { Reverse: false, }, } - req2 := ð.GetBlockHeadersPacket66{ + req2 := &GetBlockHeaders{ RequestId: uint64(222), GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ Origin: eth.HashOrNumber{ @@ -270,46 +170,49 @@ func (s *Suite) TestSimultaneousRequests66(t *utesting.T) { Reverse: false, }, } + // write the first request - if err := conn.Write66(req1, GetBlockHeaders{}.Code()); err != nil { + if err := conn.Write(req1); err != nil { t.Fatalf("failed to write to connection: %v", err) } // write the second request - if err := conn.Write66(req2, GetBlockHeaders{}.Code()); err != nil { + if err := conn.Write(req2); err != nil { t.Fatalf("failed to write to connection: %v", err) } + // wait for responses msg := conn.waitForResponse(s.chain, timeout, req1.RequestId) - headers1, ok := msg.(BlockHeaders) + headers1, ok := msg.(*BlockHeaders) if !ok { t.Fatalf("unexpected %s", pretty.Sdump(msg)) } msg = conn.waitForResponse(s.chain, timeout, req2.RequestId) - headers2, ok := msg.(BlockHeaders) + headers2, ok := msg.(*BlockHeaders) if !ok { t.Fatalf("unexpected %s", pretty.Sdump(msg)) } + // check received headers for accuracy - expected1, err := s.chain.GetHeaders(GetBlockHeaders(*req1.GetBlockHeadersPacket)) + expected1, err := s.chain.GetHeaders(req1) if err != nil { t.Fatalf("failed to get expected headers for request 1: %v", err) } - expected2, err := s.chain.GetHeaders(GetBlockHeaders(*req2.GetBlockHeadersPacket)) + expected2, err := s.chain.GetHeaders(req2) if err != nil { t.Fatalf("failed to get expected headers for request 2: %v", err) } - if !headersMatch(expected1, headers1) { + if !headersMatch(expected1, headers1.BlockHeadersPacket) { t.Fatalf("header mismatch: \nexpected %v \ngot %v", expected1, headers1) } - if !headersMatch(expected2, headers2) { + if !headersMatch(expected2, headers2.BlockHeadersPacket) { t.Fatalf("header mismatch: \nexpected %v \ngot %v", expected2, headers2) } } -// TestSameRequestID66 sends two requests with the same request ID to a +// TestSameRequestID sends two requests with the same request ID to a // single node. -func (s *Suite) TestSameRequestID66(t *utesting.T) { - conn, err := s.dial66() +func (s *Suite) TestSameRequestID(t *utesting.T) { + conn, err := s.dial() if err != nil { t.Fatalf("dial failed: %v", err) } @@ -319,7 +222,7 @@ func (s *Suite) TestSameRequestID66(t *utesting.T) { } // create requests reqID := uint64(1234) - request1 := ð.GetBlockHeadersPacket66{ + request1 := &GetBlockHeaders{ RequestId: reqID, GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ Origin: eth.HashOrNumber{ @@ -328,7 +231,7 @@ func (s *Suite) TestSameRequestID66(t *utesting.T) { Amount: 2, }, } - request2 := ð.GetBlockHeadersPacket66{ + request2 := &GetBlockHeaders{ RequestId: reqID, GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ Origin: eth.HashOrNumber{ @@ -337,45 +240,48 @@ func (s *Suite) TestSameRequestID66(t *utesting.T) { Amount: 2, }, } + // write the requests - if err = conn.Write66(request1, GetBlockHeaders{}.Code()); err != nil { + if err = conn.Write(request1); err != nil { t.Fatalf("failed to write to connection: %v", err) } - if err = conn.Write66(request2, GetBlockHeaders{}.Code()); err != nil { + if err = conn.Write(request2); err != nil { t.Fatalf("failed to write to connection: %v", err) } + // wait for responses msg := conn.waitForResponse(s.chain, timeout, reqID) - headers1, ok := msg.(BlockHeaders) + headers1, ok := msg.(*BlockHeaders) if !ok { t.Fatalf("unexpected %s", pretty.Sdump(msg)) } msg = conn.waitForResponse(s.chain, timeout, reqID) - headers2, ok := msg.(BlockHeaders) + headers2, ok := msg.(*BlockHeaders) if !ok { t.Fatalf("unexpected %s", pretty.Sdump(msg)) } + // check if headers match - expected1, err := s.chain.GetHeaders(GetBlockHeaders(*request1.GetBlockHeadersPacket)) + expected1, err := s.chain.GetHeaders(request1) if err != nil { t.Fatalf("failed to get expected block headers: %v", err) } - expected2, err := s.chain.GetHeaders(GetBlockHeaders(*request2.GetBlockHeadersPacket)) + expected2, err := s.chain.GetHeaders(request2) if err != nil { t.Fatalf("failed to get expected block headers: %v", err) } - if !headersMatch(expected1, headers1) { + if !headersMatch(expected1, headers1.BlockHeadersPacket) { t.Fatalf("header mismatch: \nexpected %v \ngot %v", expected1, headers1) } - if !headersMatch(expected2, headers2) { + if !headersMatch(expected2, headers2.BlockHeadersPacket) { t.Fatalf("header mismatch: \nexpected %v \ngot %v", expected2, headers2) } } -// TestZeroRequestID_66 checks that a message with a request ID of zero is still handled +// TestZeroRequestID checks that a message with a request ID of zero is still handled // by the node. -func (s *Suite) TestZeroRequestID66(t *utesting.T) { - conn, err := s.dial66() +func (s *Suite) TestZeroRequestID(t *utesting.T) { + conn, err := s.dial() if err != nil { t.Fatalf("dial failed: %v", err) } @@ -384,16 +290,16 @@ func (s *Suite) TestZeroRequestID66(t *utesting.T) { t.Fatalf("peering failed: %v", err) } req := &GetBlockHeaders{ - Origin: eth.HashOrNumber{ - Number: 0, + GetBlockHeadersPacket: ð.GetBlockHeadersPacket{ + Origin: eth.HashOrNumber{Number: 0}, + Amount: 2, }, - Amount: 2, } - headers, err := conn.headersRequest(req, s.chain, eth66, 0) + headers, err := conn.headersRequest(req, s.chain, 0) if err != nil { t.Fatalf("failed to get block headers: %v", err) } - expected, err := s.chain.GetHeaders(*req) + expected, err := s.chain.GetHeaders(req) if err != nil { t.Fatalf("failed to get expected block headers: %v", err) } @@ -402,9 +308,9 @@ func (s *Suite) TestZeroRequestID66(t *utesting.T) { } } -// TestGetBlockBodies65 tests whether the given node can respond to +// TestGetBlockBodies tests whether the given node can respond to // a `GetBlockBodies` request and that the response is accurate. -func (s *Suite) TestGetBlockBodies65(t *utesting.T) { +func (s *Suite) TestGetBlockBodies(t *utesting.T) { conn, err := s.dial() if err != nil { t.Fatalf("dial failed: %v", err) @@ -415,126 +321,39 @@ func (s *Suite) TestGetBlockBodies65(t *utesting.T) { } // create block bodies request req := &GetBlockBodies{ - s.chain.blocks[54].Hash(), - s.chain.blocks[75].Hash(), - } - if err := conn.Write(req); err != nil { - t.Fatalf("could not write to connection: %v", err) - } - // wait for response - switch msg := conn.readAndServe(s.chain, timeout).(type) { - case *BlockBodies: - t.Logf("received %d block bodies", len(*msg)) - if len(*msg) != len(*req) { - t.Fatalf("wrong bodies in response: expected %d bodies, "+ - "got %d", len(*req), len(*msg)) - } - default: - t.Fatalf("unexpected: %s", pretty.Sdump(msg)) - } -} - -// TestGetBlockBodies66 tests whether the given node can respond to -// a `GetBlockBodies` request and that the response is accurate over -// the eth66 protocol. -func (s *Suite) TestGetBlockBodies66(t *utesting.T) { - conn, err := s.dial66() - if err != nil { - t.Fatalf("dial failed: %v", err) - } - defer conn.Close() - if err := conn.peer(s.chain, nil); err != nil { - t.Fatalf("peering failed: %v", err) - } - // create block bodies request - req := ð.GetBlockBodiesPacket66{ RequestId: uint64(55), GetBlockBodiesPacket: eth.GetBlockBodiesPacket{ s.chain.blocks[54].Hash(), s.chain.blocks[75].Hash(), }, } - if err := conn.Write66(req, GetBlockBodies{}.Code()); err != nil { + if err := conn.Write(req); err != nil { t.Fatalf("could not write to connection: %v", err) } // wait for block bodies response msg := conn.waitForResponse(s.chain, timeout, req.RequestId) - blockBodies, ok := msg.(BlockBodies) + resp, ok := msg.(*BlockBodies) if !ok { t.Fatalf("unexpected: %s", pretty.Sdump(msg)) } - t.Logf("received %d block bodies", len(blockBodies)) - if len(blockBodies) != len(req.GetBlockBodiesPacket) { + bodies := resp.BlockBodiesPacket + t.Logf("received %d block bodies", len(bodies)) + if len(bodies) != len(req.GetBlockBodiesPacket) { t.Fatalf("wrong bodies in response: expected %d bodies, "+ - "got %d", len(req.GetBlockBodiesPacket), len(blockBodies)) + "got %d", len(req.GetBlockBodiesPacket), len(bodies)) } } -// TestBroadcast65 tests whether a block announcement is correctly -// propagated to the given node's peer(s). -func (s *Suite) TestBroadcast65(t *utesting.T) { - if err := s.sendNextBlock(eth65); err != nil { +// TestBroadcast tests whether a block announcement is correctly +// propagated to the node's peers. +func (s *Suite) TestBroadcast(t *utesting.T) { + if err := s.sendNextBlock(); err != nil { t.Fatalf("block broadcast failed: %v", err) } } -// TestBroadcast66 tests whether a block announcement is correctly -// propagated to the given node's peer(s) on the eth66 protocol. -func (s *Suite) TestBroadcast66(t *utesting.T) { - if err := s.sendNextBlock(eth66); err != nil { - t.Fatalf("block broadcast failed: %v", err) - } -} - -// TestLargeAnnounce65 tests the announcement mechanism with a large block. -func (s *Suite) TestLargeAnnounce65(t *utesting.T) { - nextBlock := len(s.chain.blocks) - blocks := []*NewBlock{ - { - Block: largeBlock(), - TD: s.fullChain.TotalDifficultyAt(nextBlock), - }, - { - Block: s.fullChain.blocks[nextBlock], - TD: largeNumber(2), - }, - { - Block: largeBlock(), - TD: largeNumber(2), - }, - } - - for i, blockAnnouncement := range blocks { - t.Logf("Testing malicious announcement: %v\n", i) - conn, err := s.dial() - if err != nil { - t.Fatalf("dial failed: %v", err) - } - if err = conn.peer(s.chain, nil); err != nil { - t.Fatalf("peering failed: %v", err) - } - if err = conn.Write(blockAnnouncement); err != nil { - t.Fatalf("could not write to connection: %v", err) - } - // Invalid announcement, check that peer disconnected - switch msg := conn.readAndServe(s.chain, time.Second*8).(type) { - case *Disconnect: - case *Error: - break - default: - t.Fatalf("unexpected: %s wanted disconnect", pretty.Sdump(msg)) - } - conn.Close() - } - // Test the last block as a valid block - if err := s.sendNextBlock(eth65); err != nil { - t.Fatalf("failed to broadcast next block: %v", err) - } -} - -// TestLargeAnnounce66 tests the announcement mechanism with a large -// block over the eth66 protocol. -func (s *Suite) TestLargeAnnounce66(t *utesting.T) { +// TestLargeAnnounce tests the announcement mechanism with a large block. +func (s *Suite) TestLargeAnnounce(t *utesting.T) { nextBlock := len(s.chain.blocks) blocks := []*NewBlock{ { @@ -553,7 +372,7 @@ func (s *Suite) TestLargeAnnounce66(t *utesting.T) { for i, blockAnnouncement := range blocks[0:3] { t.Logf("Testing malicious announcement: %v\n", i) - conn, err := s.dial66() + conn, err := s.dial() if err != nil { t.Fatalf("dial failed: %v", err) } @@ -564,7 +383,7 @@ func (s *Suite) TestLargeAnnounce66(t *utesting.T) { t.Fatalf("could not write to connection: %v", err) } // Invalid announcement, check that peer disconnected - switch msg := conn.readAndServe(s.chain, time.Second*8).(type) { + switch msg := conn.readAndServe(s.chain, 8*time.Second).(type) { case *Disconnect: case *Error: break @@ -574,58 +393,35 @@ func (s *Suite) TestLargeAnnounce66(t *utesting.T) { conn.Close() } // Test the last block as a valid block - if err := s.sendNextBlock(eth66); err != nil { + if err := s.sendNextBlock(); err != nil { t.Fatalf("failed to broadcast next block: %v", err) } } -// TestOldAnnounce65 tests the announcement mechanism with an old block. -func (s *Suite) TestOldAnnounce65(t *utesting.T) { - if err := s.oldAnnounce(eth65); err != nil { +// TestOldAnnounce tests the announcement mechanism with an old block. +func (s *Suite) TestOldAnnounce(t *utesting.T) { + if err := s.oldAnnounce(); err != nil { t.Fatal(err) } } -// TestOldAnnounce66 tests the announcement mechanism with an old block, -// over the eth66 protocol. -func (s *Suite) TestOldAnnounce66(t *utesting.T) { - if err := s.oldAnnounce(eth66); err != nil { - t.Fatal(err) - } -} - -// TestBlockHashAnnounce65 sends a new block hash announcement and expects +// TestBlockHashAnnounce sends a new block hash announcement and expects // the node to perform a `GetBlockHeaders` request. -func (s *Suite) TestBlockHashAnnounce65(t *utesting.T) { - if err := s.hashAnnounce(eth65); err != nil { +func (s *Suite) TestBlockHashAnnounce(t *utesting.T) { + if err := s.hashAnnounce(); err != nil { t.Fatalf("block hash announcement failed: %v", err) } } -// TestBlockHashAnnounce66 sends a new block hash announcement and expects -// the node to perform a `GetBlockHeaders` request. -func (s *Suite) TestBlockHashAnnounce66(t *utesting.T) { - if err := s.hashAnnounce(eth66); err != nil { - t.Fatalf("block hash announcement failed: %v", err) - } -} - -// TestMaliciousHandshake65 tries to send malicious data during the handshake. -func (s *Suite) TestMaliciousHandshake65(t *utesting.T) { - if err := s.maliciousHandshakes(t, eth65); err != nil { +// TestMaliciousHandshake tries to send malicious data during the handshake. +func (s *Suite) TestMaliciousHandshake(t *utesting.T) { + if err := s.maliciousHandshakes(t); err != nil { t.Fatal(err) } } -// TestMaliciousHandshake66 tries to send malicious data during the handshake. -func (s *Suite) TestMaliciousHandshake66(t *utesting.T) { - if err := s.maliciousHandshakes(t, eth66); err != nil { - t.Fatal(err) - } -} - -// TestMaliciousStatus65 sends a status package with a large total difficulty. -func (s *Suite) TestMaliciousStatus65(t *utesting.T) { +// TestMaliciousStatus sends a status package with a large total difficulty. +func (s *Suite) TestMaliciousStatus(t *utesting.T) { conn, err := s.dial() if err != nil { t.Fatalf("dial failed: %v", err) @@ -637,58 +433,28 @@ func (s *Suite) TestMaliciousStatus65(t *utesting.T) { } } -// TestMaliciousStatus66 sends a status package with a large total -// difficulty over the eth66 protocol. -func (s *Suite) TestMaliciousStatus66(t *utesting.T) { - conn, err := s.dial66() - if err != nil { - t.Fatalf("dial failed: %v", err) - } - defer conn.Close() - - if err := s.maliciousStatus(conn); err != nil { - t.Fatal(err) - } -} - -// TestTransaction65 sends a valid transaction to the node and +// TestTransaction sends a valid transaction to the node and // checks if the transaction gets propagated. -func (s *Suite) TestTransaction65(t *utesting.T) { - if err := s.sendSuccessfulTxs(t, eth65); err != nil { +func (s *Suite) TestTransaction(t *utesting.T) { + if err := s.sendSuccessfulTxs(t); err != nil { t.Fatal(err) } } -// TestTransaction66 sends a valid transaction to the node and -// checks if the transaction gets propagated. -func (s *Suite) TestTransaction66(t *utesting.T) { - if err := s.sendSuccessfulTxs(t, eth66); err != nil { - t.Fatal(err) - } -} - -// TestMaliciousTx65 sends several invalid transactions and tests whether +// TestMaliciousTx sends several invalid transactions and tests whether // the node will propagate them. -func (s *Suite) TestMaliciousTx65(t *utesting.T) { - if err := s.sendMaliciousTxs(t, eth65); err != nil { +func (s *Suite) TestMaliciousTx(t *utesting.T) { + if err := s.sendMaliciousTxs(t); err != nil { t.Fatal(err) } } -// TestMaliciousTx66 sends several invalid transactions and tests whether -// the node will propagate them. -func (s *Suite) TestMaliciousTx66(t *utesting.T) { - if err := s.sendMaliciousTxs(t, eth66); err != nil { - t.Fatal(err) - } -} - -// TestLargeTxRequest66 tests whether a node can fulfill a large GetPooledTransactions +// TestLargeTxRequest tests whether a node can fulfill a large GetPooledTransactions // request. -func (s *Suite) TestLargeTxRequest66(t *utesting.T) { +func (s *Suite) TestLargeTxRequest(t *utesting.T) { // send the next block to ensure the node is no longer syncing and // is able to accept txs - if err := s.sendNextBlock(eth66); err != nil { + if err := s.sendNextBlock(); err != nil { t.Fatalf("failed to send next block: %v", err) } // send 2000 transactions to the node @@ -701,7 +467,7 @@ func (s *Suite) TestLargeTxRequest66(t *utesting.T) { } // set up connection to receive to ensure node is peered with the receiving connection // before tx request is sent - conn, err := s.dial66() + conn, err := s.dial() if err != nil { t.Fatalf("dial failed: %v", err) } @@ -714,17 +480,17 @@ func (s *Suite) TestLargeTxRequest66(t *utesting.T) { for _, hash := range hashMap { hashes = append(hashes, hash) } - getTxReq := ð.GetPooledTransactionsPacket66{ + getTxReq := &GetPooledTransactions{ RequestId: 1234, GetPooledTransactionsPacket: hashes, } - if err = conn.Write66(getTxReq, GetPooledTransactions{}.Code()); err != nil { + if err = conn.Write(getTxReq); err != nil { t.Fatalf("could not write to conn: %v", err) } // check that all received transactions match those that were sent to node switch msg := conn.waitForResponse(s.chain, timeout, getTxReq.RequestId).(type) { - case PooledTransactions: - for _, gotTx := range msg { + case *PooledTransactions: + for _, gotTx := range msg.PooledTransactionsPacket { if _, exists := hashMap[gotTx.Hash()]; !exists { t.Fatalf("unexpected tx received: %v", gotTx.Hash()) } @@ -734,12 +500,12 @@ func (s *Suite) TestLargeTxRequest66(t *utesting.T) { } } -// TestNewPooledTxs_66 tests whether a node will do a GetPooledTransactions +// TestNewPooledTxs tests whether a node will do a GetPooledTransactions // request upon receiving a NewPooledTransactionHashes announcement. -func (s *Suite) TestNewPooledTxs66(t *utesting.T) { +func (s *Suite) TestNewPooledTxs(t *utesting.T) { // send the next block to ensure the node is no longer syncing and // is able to accept txs - if err := s.sendNextBlock(eth66); err != nil { + if err := s.sendNextBlock(); err != nil { t.Fatalf("failed to send next block: %v", err) } @@ -757,7 +523,7 @@ func (s *Suite) TestNewPooledTxs66(t *utesting.T) { announce := NewPooledTransactionHashes(hashes) // send announcement - conn, err := s.dial66() + conn, err := s.dial() if err != nil { t.Fatalf("dial failed: %v", err) } @@ -771,11 +537,11 @@ func (s *Suite) TestNewPooledTxs66(t *utesting.T) { // wait for GetPooledTxs request for { - _, msg := conn.readAndServe66(s.chain, timeout) + msg := conn.readAndServe(s.chain, timeout) switch msg := msg.(type) { - case GetPooledTransactions: - if len(msg) != len(hashes) { - t.Fatalf("unexpected number of txs requested: wanted %d, got %d", len(hashes), len(msg)) + case *GetPooledTransactions: + if len(msg.GetPooledTransactionsPacket) != len(hashes) { + t.Fatalf("unexpected number of txs requested: wanted %d, got %d", len(hashes), len(msg.GetPooledTransactionsPacket)) } return // ignore propagated txs from previous tests diff --git a/cmd/devp2p/internal/ethtest/suite_test.go b/cmd/devp2p/internal/ethtest/suite_test.go index 924c80d01..8a2b132fa 100644 --- a/cmd/devp2p/internal/ethtest/suite_test.go +++ b/cmd/devp2p/internal/ethtest/suite_test.go @@ -45,7 +45,7 @@ func TestEthSuite(t *testing.T) { if err != nil { t.Fatalf("could not create new test suite: %v", err) } - for _, test := range suite.Eth66Tests() { + for _, test := range suite.EthTests() { t.Run(test.Name, func(t *testing.T) { result := utesting.RunTAP([]utesting.Test{{Name: test.Name, Fn: test.Fn}}, os.Stdout) if result[0].Failed { diff --git a/cmd/devp2p/internal/ethtest/transaction.go b/cmd/devp2p/internal/ethtest/transaction.go index 5d722f417..c4748bf8f 100644 --- a/cmd/devp2p/internal/ethtest/transaction.go +++ b/cmd/devp2p/internal/ethtest/transaction.go @@ -32,7 +32,7 @@ import ( //var faucetAddr = common.HexToAddress("0x71562b71999873DB5b286dF957af199Ec94617F7") var faucetKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") -func (s *Suite) sendSuccessfulTxs(t *utesting.T, isEth66 bool) error { +func (s *Suite) sendSuccessfulTxs(t *utesting.T) error { tests := []*types.Transaction{ getNextTxFromChain(s), unknownTx(s), @@ -48,15 +48,15 @@ func (s *Suite) sendSuccessfulTxs(t *utesting.T, isEth66 bool) error { prevTx = tests[i-1] } // write tx to connection - if err := sendSuccessfulTx(s, tx, prevTx, isEth66); err != nil { + if err := sendSuccessfulTx(s, tx, prevTx); err != nil { return fmt.Errorf("send successful tx test failed: %v", err) } } return nil } -func sendSuccessfulTx(s *Suite, tx *types.Transaction, prevTx *types.Transaction, isEth66 bool) error { - sendConn, recvConn, err := s.createSendAndRecvConns(isEth66) +func sendSuccessfulTx(s *Suite, tx *types.Transaction, prevTx *types.Transaction) error { + sendConn, recvConn, err := s.createSendAndRecvConns() if err != nil { return err } @@ -73,8 +73,10 @@ func sendSuccessfulTx(s *Suite, tx *types.Transaction, prevTx *types.Transaction if err = recvConn.peer(s.chain, nil); err != nil { return fmt.Errorf("peering failed: %v", err) } + // update last nonce seen nonce = tx.Nonce() + // Wait for the transaction announcement for { switch msg := recvConn.readAndServe(s.chain, timeout).(type) { @@ -114,7 +116,7 @@ func sendSuccessfulTx(s *Suite, tx *types.Transaction, prevTx *types.Transaction } } -func (s *Suite) sendMaliciousTxs(t *utesting.T, isEth66 bool) error { +func (s *Suite) sendMaliciousTxs(t *utesting.T) error { badTxs := []*types.Transaction{ getOldTxFromChain(s), invalidNonceTx(s), @@ -122,16 +124,9 @@ func (s *Suite) sendMaliciousTxs(t *utesting.T, isEth66 bool) error { hugeGasPrice(s), hugeData(s), } + // setup receiving connection before sending malicious txs - var ( - recvConn *Conn - err error - ) - if isEth66 { - recvConn, err = s.dial66() - } else { - recvConn, err = s.dial() - } + recvConn, err := s.dial() if err != nil { return fmt.Errorf("dial failed: %v", err) } @@ -139,9 +134,10 @@ func (s *Suite) sendMaliciousTxs(t *utesting.T, isEth66 bool) error { if err = recvConn.peer(s.chain, nil); err != nil { return fmt.Errorf("peering failed: %v", err) } + for i, tx := range badTxs { t.Logf("Testing malicious tx propagation: %v\n", i) - if err = sendMaliciousTx(s, tx, isEth66); err != nil { + if err = sendMaliciousTx(s, tx); err != nil { return fmt.Errorf("malicious tx test failed:\ntx: %v\nerror: %v", tx, err) } } @@ -149,17 +145,8 @@ func (s *Suite) sendMaliciousTxs(t *utesting.T, isEth66 bool) error { return checkMaliciousTxPropagation(s, badTxs, recvConn) } -func sendMaliciousTx(s *Suite, tx *types.Transaction, isEth66 bool) error { - // setup connection - var ( - conn *Conn - err error - ) - if isEth66 { - conn, err = s.dial66() - } else { - conn, err = s.dial() - } +func sendMaliciousTx(s *Suite, tx *types.Transaction) error { + conn, err := s.dial() if err != nil { return fmt.Errorf("dial failed: %v", err) } @@ -167,6 +154,7 @@ func sendMaliciousTx(s *Suite, tx *types.Transaction, isEth66 bool) error { if err = conn.peer(s.chain, nil); err != nil { return fmt.Errorf("peering failed: %v", err) } + // write malicious tx if err = conn.Write(&Transactions{tx}); err != nil { return fmt.Errorf("failed to write to connection: %v", err) @@ -182,7 +170,7 @@ func sendMultipleSuccessfulTxs(t *utesting.T, s *Suite, txs []*types.Transaction txMsg := Transactions(txs) t.Logf("sending %d txs\n", len(txs)) - sendConn, recvConn, err := s.createSendAndRecvConns(true) + sendConn, recvConn, err := s.createSendAndRecvConns() if err != nil { return err } @@ -194,15 +182,19 @@ func sendMultipleSuccessfulTxs(t *utesting.T, s *Suite, txs []*types.Transaction if err = recvConn.peer(s.chain, nil); err != nil { return fmt.Errorf("peering failed: %v", err) } + // Send the transactions if err = sendConn.Write(&txMsg); err != nil { return fmt.Errorf("failed to write message to connection: %v", err) } + // update nonce nonce = txs[len(txs)-1].Nonce() - // Wait for the transaction announcement(s) and make sure all sent txs are being propagated + + // Wait for the transaction announcement(s) and make sure all sent txs are being propagated. + // all txs should be announced within 3 announcements. recvHashes := make([]common.Hash, 0) - // all txs should be announced within 3 announcements + for i := 0; i < 3; i++ { switch msg := recvConn.readAndServe(s.chain, timeout).(type) { case *Transactions: diff --git a/cmd/devp2p/internal/ethtest/types.go b/cmd/devp2p/internal/ethtest/types.go index e69d94bb5..2c5cb94c6 100644 --- a/cmd/devp2p/internal/ethtest/types.go +++ b/cmd/devp2p/internal/ethtest/types.go @@ -29,6 +29,7 @@ import ( type Message interface { Code() int + ReqID() uint64 } type Error struct { @@ -37,9 +38,11 @@ type Error struct { func (e *Error) Unwrap() error { return e.err } func (e *Error) Error() string { return e.err.Error() } -func (e *Error) Code() int { return -1 } func (e *Error) String() string { return e.Error() } +func (e *Error) Code() int { return -1 } +func (e *Error) ReqID() uint64 { return 0 } + func errorf(format string, args ...interface{}) *Error { return &Error{fmt.Errorf(format, args...)} } @@ -56,73 +59,88 @@ type Hello struct { Rest []rlp.RawValue `rlp:"tail"` } -func (h Hello) Code() int { return 0x00 } +func (msg Hello) Code() int { return 0x00 } +func (msg Hello) ReqID() uint64 { return 0 } // Disconnect is the RLP structure for a disconnect message. type Disconnect struct { Reason p2p.DiscReason } -func (d Disconnect) Code() int { return 0x01 } +func (msg Disconnect) Code() int { return 0x01 } +func (msg Disconnect) ReqID() uint64 { return 0 } type Ping struct{} -func (p Ping) Code() int { return 0x02 } +func (msg Ping) Code() int { return 0x02 } +func (msg Ping) ReqID() uint64 { return 0 } type Pong struct{} -func (p Pong) Code() int { return 0x03 } +func (msg Pong) Code() int { return 0x03 } +func (msg Pong) ReqID() uint64 { return 0 } // Status is the network packet for the status message for eth/64 and later. type Status eth.StatusPacket -func (s Status) Code() int { return 16 } +func (msg Status) Code() int { return 16 } +func (msg Status) ReqID() uint64 { return 0 } // NewBlockHashes is the network packet for the block announcements. type NewBlockHashes eth.NewBlockHashesPacket -func (nbh NewBlockHashes) Code() int { return 17 } +func (msg NewBlockHashes) Code() int { return 17 } +func (msg NewBlockHashes) ReqID() uint64 { return 0 } type Transactions eth.TransactionsPacket -func (t Transactions) Code() int { return 18 } +func (msg Transactions) Code() int { return 18 } +func (msg Transactions) ReqID() uint64 { return 18 } // GetBlockHeaders represents a block header query. -type GetBlockHeaders eth.GetBlockHeadersPacket +type GetBlockHeaders eth.GetBlockHeadersPacket66 -func (g GetBlockHeaders) Code() int { return 19 } +func (msg GetBlockHeaders) Code() int { return 19 } +func (msg GetBlockHeaders) ReqID() uint64 { return msg.RequestId } -type BlockHeaders eth.BlockHeadersPacket +type BlockHeaders eth.BlockHeadersPacket66 -func (bh BlockHeaders) Code() int { return 20 } +func (msg BlockHeaders) Code() int { return 20 } +func (msg BlockHeaders) ReqID() uint64 { return msg.RequestId } // GetBlockBodies represents a GetBlockBodies request -type GetBlockBodies eth.GetBlockBodiesPacket +type GetBlockBodies eth.GetBlockBodiesPacket66 -func (gbb GetBlockBodies) Code() int { return 21 } +func (msg GetBlockBodies) Code() int { return 21 } +func (msg GetBlockBodies) ReqID() uint64 { return msg.RequestId } // BlockBodies is the network packet for block content distribution. -type BlockBodies eth.BlockBodiesPacket +type BlockBodies eth.BlockBodiesPacket66 -func (bb BlockBodies) Code() int { return 22 } +func (msg BlockBodies) Code() int { return 22 } +func (msg BlockBodies) ReqID() uint64 { return msg.RequestId } // NewBlock is the network packet for the block propagation message. type NewBlock eth.NewBlockPacket -func (nb NewBlock) Code() int { return 23 } +func (msg NewBlock) Code() int { return 23 } +func (msg NewBlock) ReqID() uint64 { return 0 } // NewPooledTransactionHashes is the network packet for the tx hash propagation message. type NewPooledTransactionHashes eth.NewPooledTransactionHashesPacket -func (nb NewPooledTransactionHashes) Code() int { return 24 } +func (msg NewPooledTransactionHashes) Code() int { return 24 } +func (msg NewPooledTransactionHashes) ReqID() uint64 { return 0 } -type GetPooledTransactions eth.GetPooledTransactionsPacket +type GetPooledTransactions eth.GetPooledTransactionsPacket66 -func (gpt GetPooledTransactions) Code() int { return 25 } +func (msg GetPooledTransactions) Code() int { return 25 } +func (msg GetPooledTransactions) ReqID() uint64 { return msg.RequestId } -type PooledTransactions eth.PooledTransactionsPacket +type PooledTransactions eth.PooledTransactionsPacket66 -func (pt PooledTransactions) Code() int { return 26 } +func (msg PooledTransactions) Code() int { return 26 } +func (msg PooledTransactions) ReqID() uint64 { return msg.RequestId } // Conn represents an individual connection with a peer type Conn struct { @@ -135,62 +153,13 @@ type Conn struct { caps []p2p.Cap } -// Read reads an eth packet from the connection. +// Read reads an eth66 packet from the connection. func (c *Conn) Read() Message { code, rawData, _, err := c.Conn.Read() if err != nil { return errorf("could not read from connection: %v", err) } - var msg Message - switch int(code) { - case (Hello{}).Code(): - msg = new(Hello) - case (Ping{}).Code(): - msg = new(Ping) - case (Pong{}).Code(): - msg = new(Pong) - case (Disconnect{}).Code(): - msg = new(Disconnect) - case (Status{}).Code(): - msg = new(Status) - case (GetBlockHeaders{}).Code(): - msg = new(GetBlockHeaders) - case (BlockHeaders{}).Code(): - msg = new(BlockHeaders) - case (GetBlockBodies{}).Code(): - msg = new(GetBlockBodies) - case (BlockBodies{}).Code(): - msg = new(BlockBodies) - case (NewBlock{}).Code(): - msg = new(NewBlock) - case (NewBlockHashes{}).Code(): - msg = new(NewBlockHashes) - case (Transactions{}).Code(): - msg = new(Transactions) - case (NewPooledTransactionHashes{}).Code(): - msg = new(NewPooledTransactionHashes) - case (GetPooledTransactions{}.Code()): - msg = new(GetPooledTransactions) - case (PooledTransactions{}.Code()): - msg = new(PooledTransactions) - default: - return errorf("invalid message code: %d", code) - } - // if message is devp2p, decode here - if err := rlp.DecodeBytes(rawData, msg); err != nil { - return errorf("could not rlp decode message: %v", err) - } - return msg -} - -// Read66 reads an eth66 packet from the connection. -func (c *Conn) Read66() (uint64, Message) { - code, rawData, _, err := c.Conn.Read() - if err != nil { - return 0, errorf("could not read from connection: %v", err) - } - var msg Message switch int(code) { case (Hello{}).Code(): @@ -206,27 +175,27 @@ func (c *Conn) Read66() (uint64, Message) { case (GetBlockHeaders{}).Code(): ethMsg := new(eth.GetBlockHeadersPacket66) if err := rlp.DecodeBytes(rawData, ethMsg); err != nil { - return 0, errorf("could not rlp decode message: %v", err) + return errorf("could not rlp decode message: %v", err) } - return ethMsg.RequestId, GetBlockHeaders(*ethMsg.GetBlockHeadersPacket) + return (*GetBlockHeaders)(ethMsg) case (BlockHeaders{}).Code(): ethMsg := new(eth.BlockHeadersPacket66) if err := rlp.DecodeBytes(rawData, ethMsg); err != nil { - return 0, errorf("could not rlp decode message: %v", err) + return errorf("could not rlp decode message: %v", err) } - return ethMsg.RequestId, BlockHeaders(ethMsg.BlockHeadersPacket) + return (*BlockHeaders)(ethMsg) case (GetBlockBodies{}).Code(): ethMsg := new(eth.GetBlockBodiesPacket66) if err := rlp.DecodeBytes(rawData, ethMsg); err != nil { - return 0, errorf("could not rlp decode message: %v", err) + return errorf("could not rlp decode message: %v", err) } - return ethMsg.RequestId, GetBlockBodies(ethMsg.GetBlockBodiesPacket) + return (*GetBlockBodies)(ethMsg) case (BlockBodies{}).Code(): ethMsg := new(eth.BlockBodiesPacket66) if err := rlp.DecodeBytes(rawData, ethMsg); err != nil { - return 0, errorf("could not rlp decode message: %v", err) + return errorf("could not rlp decode message: %v", err) } - return ethMsg.RequestId, BlockBodies(ethMsg.BlockBodiesPacket) + return (*BlockBodies)(ethMsg) case (NewBlock{}).Code(): msg = new(NewBlock) case (NewBlockHashes{}).Code(): @@ -238,26 +207,26 @@ func (c *Conn) Read66() (uint64, Message) { case (GetPooledTransactions{}.Code()): ethMsg := new(eth.GetPooledTransactionsPacket66) if err := rlp.DecodeBytes(rawData, ethMsg); err != nil { - return 0, errorf("could not rlp decode message: %v", err) + return errorf("could not rlp decode message: %v", err) } - return ethMsg.RequestId, GetPooledTransactions(ethMsg.GetPooledTransactionsPacket) + return (*GetPooledTransactions)(ethMsg) case (PooledTransactions{}.Code()): ethMsg := new(eth.PooledTransactionsPacket66) if err := rlp.DecodeBytes(rawData, ethMsg); err != nil { - return 0, errorf("could not rlp decode message: %v", err) + return errorf("could not rlp decode message: %v", err) } - return ethMsg.RequestId, PooledTransactions(ethMsg.PooledTransactionsPacket) + return (*PooledTransactions)(ethMsg) default: msg = errorf("invalid message code: %d", code) } if msg != nil { if err := rlp.DecodeBytes(rawData, msg); err != nil { - return 0, errorf("could not rlp decode message: %v", err) + return errorf("could not rlp decode message: %v", err) } - return 0, msg + return msg } - return 0, errorf("invalid message: %s", string(rawData)) + return errorf("invalid message: %s", string(rawData)) } // Write writes a eth packet to the connection. @@ -270,16 +239,6 @@ func (c *Conn) Write(msg Message) error { return err } -// Write66 writes an eth66 packet to the connection. -func (c *Conn) Write66(req eth.Packet, code int) error { - payload, err := rlp.EncodeToBytes(req) - if err != nil { - return err - } - _, err = c.Conn.Write(uint64(code), payload) - return err -} - // ReadSnap reads a snap/1 response with the given id from the connection. func (c *Conn) ReadSnap(id uint64) (Message, error) { respId := id + 1 diff --git a/cmd/devp2p/rlpxcmd.go b/cmd/devp2p/rlpxcmd.go index 07978e4f8..42b38120c 100644 --- a/cmd/devp2p/rlpxcmd.go +++ b/cmd/devp2p/rlpxcmd.go @@ -22,7 +22,6 @@ import ( "github.com/ethereum/go-ethereum/cmd/devp2p/internal/ethtest" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/internal/utesting" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/rlpx" "github.com/ethereum/go-ethereum/rlp" @@ -110,12 +109,7 @@ func rlpxEthTest(ctx *cli.Context) error { if err != nil { exit(err) } - // check if given node supports eth66, and if so, run eth66 protocol tests as well - is66Failed, _ := utesting.Run(utesting.Test{Name: "Is_66", Fn: suite.Is_66}) - if is66Failed { - return runTests(ctx, suite.EthTests()) - } - return runTests(ctx, suite.AllEthTests()) + return runTests(ctx, suite.EthTests()) } // rlpxSnapTest runs the snap protocol test suite. From 9244f87dc1c8869a2632176f719e515217720a43 Mon Sep 17 00:00:00 2001 From: Tristan-Wilson <87238672+Tristan-Wilson@users.noreply.github.com> Date: Wed, 3 Aug 2022 07:50:12 -0700 Subject: [PATCH 18/59] node, rpc: add ReadHeaderTimeout config option (#25338) This change makes http.Server.ReadHeaderTimeout configurable separately from ReadTimeout for RPC servers. The default is set to the same as ReadTimeout, which in order to cause no change in existing deployments. --- node/endpoints.go | 13 +++++++++---- node/rpcstack.go | 1 + rpc/http.go | 15 ++++++++++++--- 3 files changed, 22 insertions(+), 7 deletions(-) diff --git a/node/endpoints.go b/node/endpoints.go index efc311e7e..14c12fd1f 100644 --- a/node/endpoints.go +++ b/node/endpoints.go @@ -39,10 +39,11 @@ func StartHTTPEndpoint(endpoint string, timeouts rpc.HTTPTimeouts, handler http. CheckTimeouts(&timeouts) // Bundle and start the HTTP server httpSrv := &http.Server{ - Handler: handler, - ReadTimeout: timeouts.ReadTimeout, - WriteTimeout: timeouts.WriteTimeout, - IdleTimeout: timeouts.IdleTimeout, + Handler: handler, + ReadTimeout: timeouts.ReadTimeout, + ReadHeaderTimeout: timeouts.ReadHeaderTimeout, + WriteTimeout: timeouts.WriteTimeout, + IdleTimeout: timeouts.IdleTimeout, } go httpSrv.Serve(listener) return httpSrv, listener.Addr(), err @@ -75,6 +76,10 @@ func CheckTimeouts(timeouts *rpc.HTTPTimeouts) { log.Warn("Sanitizing invalid HTTP read timeout", "provided", timeouts.ReadTimeout, "updated", rpc.DefaultHTTPTimeouts.ReadTimeout) timeouts.ReadTimeout = rpc.DefaultHTTPTimeouts.ReadTimeout } + if timeouts.ReadHeaderTimeout < time.Second { + log.Warn("Sanitizing invalid HTTP read header timeout", "provided", timeouts.ReadHeaderTimeout, "updated", rpc.DefaultHTTPTimeouts.ReadHeaderTimeout) + timeouts.ReadHeaderTimeout = rpc.DefaultHTTPTimeouts.ReadHeaderTimeout + } if timeouts.WriteTimeout < time.Second { log.Warn("Sanitizing invalid HTTP write timeout", "provided", timeouts.WriteTimeout, "updated", rpc.DefaultHTTPTimeouts.WriteTimeout) timeouts.WriteTimeout = rpc.DefaultHTTPTimeouts.WriteTimeout diff --git a/node/rpcstack.go b/node/rpcstack.go index 9b5873e90..5d411fa61 100644 --- a/node/rpcstack.go +++ b/node/rpcstack.go @@ -134,6 +134,7 @@ func (h *httpServer) start() error { if h.timeouts != (rpc.HTTPTimeouts{}) { CheckTimeouts(&h.timeouts) h.server.ReadTimeout = h.timeouts.ReadTimeout + h.server.ReadHeaderTimeout = h.timeouts.ReadHeaderTimeout h.server.WriteTimeout = h.timeouts.WriteTimeout h.server.IdleTimeout = h.timeouts.IdleTimeout } diff --git a/rpc/http.go b/rpc/http.go index 9f4464957..858d80858 100644 --- a/rpc/http.go +++ b/rpc/http.go @@ -87,6 +87,14 @@ type HTTPTimeouts struct { // ReadHeaderTimeout. It is valid to use them both. ReadTimeout time.Duration + // ReadHeaderTimeout is the amount of time allowed to read + // request headers. The connection's read deadline is reset + // after reading the headers and the Handler can decide what + // is considered too slow for the body. If ReadHeaderTimeout + // is zero, the value of ReadTimeout is used. If both are + // zero, there is no timeout. + ReadHeaderTimeout time.Duration + // WriteTimeout is the maximum duration before timing out // writes of the response. It is reset whenever a new // request's header is read. Like ReadTimeout, it does not @@ -103,9 +111,10 @@ type HTTPTimeouts struct { // DefaultHTTPTimeouts represents the default timeout values used if further // configuration is not provided. var DefaultHTTPTimeouts = HTTPTimeouts{ - ReadTimeout: 30 * time.Second, - WriteTimeout: 30 * time.Second, - IdleTimeout: 120 * time.Second, + ReadTimeout: 30 * time.Second, + ReadHeaderTimeout: 30 * time.Second, + WriteTimeout: 30 * time.Second, + IdleTimeout: 120 * time.Second, } // DialHTTPWithClient creates a new RPC client that connects to an RPC server over HTTP From 6b6261b51fa46f7ed36d0589db641c14569ce036 Mon Sep 17 00:00:00 2001 From: yong <33920876+yzhaoyu@users.noreply.github.com> Date: Wed, 3 Aug 2022 22:56:57 +0800 Subject: [PATCH 19/59] core/types: fix typo in comment (#25359) --- core/types/bloom9.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/types/bloom9.go b/core/types/bloom9.go index 1793c2adc..a560a2072 100644 --- a/core/types/bloom9.go +++ b/core/types/bloom9.go @@ -154,7 +154,7 @@ func bloomValues(data []byte, hashbuf []byte) (uint, byte, uint, byte, uint, byt return i1, v1, i2, v2, i3, v3 } -// BloomLookup is a convenience-method to check presence int he bloom filter +// BloomLookup is a convenience-method to check presence in the bloom filter func BloomLookup(bin Bloom, topic bytesBacked) bool { return bin.Test(topic.Bytes()) } From 5fb463dddc928eec38de80f63ebdd9d7820d1a72 Mon Sep 17 00:00:00 2001 From: Seungbae Yu Date: Thu, 4 Aug 2022 00:02:09 +0900 Subject: [PATCH 20/59] core: preallocate batch size in bloomIndexer (#25289) This change reduces allocations when committing bloombits indexes by creating the database batch with a larger initial size. --- core/bloom_indexer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/bloom_indexer.go b/core/bloom_indexer.go index 856746a1c..68a35d811 100644 --- a/core/bloom_indexer.go +++ b/core/bloom_indexer.go @@ -75,7 +75,7 @@ func (b *BloomIndexer) Process(ctx context.Context, header *types.Header) error // Commit implements core.ChainIndexerBackend, finalizing the bloom section and // writing it out into the database. func (b *BloomIndexer) Commit() error { - batch := b.db.NewBatch() + batch := b.db.NewBatchWithSize((int(b.size) / 8) * types.BloomBitLength) for i := 0; i < types.BloomBitLength; i++ { bits, err := b.gen.Bitset(uint(i)) if err != nil { From 948e08d55bf185e4b80ca7a10d689cbfd3f7fcdd Mon Sep 17 00:00:00 2001 From: lightclient <14004106+lightclient@users.noreply.github.com> Date: Wed, 3 Aug 2022 10:18:45 -0600 Subject: [PATCH 21/59] internal/ethapi: don't estimate gas if no limit provided in eth_createAccessList (#25467) Because the goal of eth_createAccessList is providing the caller with the largest-possible access list, it's generally not important that the gas limit used by the tracer will match the usage of the call exactly. Avoiding the gas estimation step is a performance improvement. As long as the call does not branch based on gas limit, the returned access list will be accurate. --- internal/ethapi/api.go | 17 +++++------------ 1 file changed, 5 insertions(+), 12 deletions(-) diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 90322033b..939dd6939 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -1392,9 +1392,11 @@ func AccessList(ctx context.Context, b Backend, blockNrOrHash rpc.BlockNumberOrH if db == nil || err != nil { return nil, 0, nil, err } - // If the gas amount is not set, extract this as it will depend on access - // lists and we'll need to reestimate every time - nogas := args.Gas == nil + // If the gas amount is not set, default to RPC gas cap. + if args.Gas == nil { + tmp := hexutil.Uint64(b.RPCGasCap()) + args.Gas = &tmp + } // Ensure any missing fields are filled, extract the recipient and input data if err := args.setDefaults(ctx, b); err != nil { @@ -1420,15 +1422,6 @@ func AccessList(ctx context.Context, b Backend, blockNrOrHash rpc.BlockNumberOrH accessList := prevTracer.AccessList() log.Trace("Creating access list", "input", accessList) - // If no gas amount was specified, each unique access list needs it's own - // gas calculation. This is quite expensive, but we need to be accurate - // and it's convered by the sender only anyway. - if nogas { - args.Gas = nil - if err := args.setDefaults(ctx, b); err != nil { - return nil, 0, nil, err // shouldn't happen, just in case - } - } // Copy the original db so we don't modify it statedb := db.Copy() // Set the accesslist to the last al From f809cf6ea61388b1ff0b7580770648ad86be412c Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 3 Aug 2022 19:08:32 +0200 Subject: [PATCH 22/59] graphql: embed *Resolver instead of backend interface (#25468) This creates some infrastructure to share resources between graphql API objects. --- graphql/graphql.go | 132 ++++++++++++++++++++++----------------------- graphql/service.go | 4 -- 2 files changed, 66 insertions(+), 70 deletions(-) diff --git a/graphql/graphql.go b/graphql/graphql.go index 0654fd1af..0949c3480 100644 --- a/graphql/graphql.go +++ b/graphql/graphql.go @@ -76,14 +76,14 @@ func (b *Long) UnmarshalGraphQL(input interface{}) error { // Account represents an Ethereum account at a particular block. type Account struct { - backend ethapi.Backend + r *Resolver address common.Address blockNrOrHash rpc.BlockNumberOrHash } // getState fetches the StateDB object for an account. func (a *Account) getState(ctx context.Context) (*state.StateDB, error) { - state, _, err := a.backend.StateAndHeaderByNumberOrHash(ctx, a.blockNrOrHash) + state, _, err := a.r.backend.StateAndHeaderByNumberOrHash(ctx, a.blockNrOrHash) return state, err } @@ -106,7 +106,7 @@ func (a *Account) Balance(ctx context.Context) (hexutil.Big, error) { func (a *Account) TransactionCount(ctx context.Context) (hexutil.Uint64, error) { // Ask transaction pool for the nonce which includes pending transactions if blockNr, ok := a.blockNrOrHash.Number(); ok && blockNr == rpc.PendingBlockNumber { - nonce, err := a.backend.GetPoolNonce(ctx, a.address) + nonce, err := a.r.backend.GetPoolNonce(ctx, a.address) if err != nil { return 0, err } @@ -137,7 +137,7 @@ func (a *Account) Storage(ctx context.Context, args struct{ Slot common.Hash }) // Log represents an individual log message. All arguments are mandatory. type Log struct { - backend ethapi.Backend + r *Resolver transaction *Transaction log *types.Log } @@ -148,7 +148,7 @@ func (l *Log) Transaction(ctx context.Context) *Transaction { func (l *Log) Account(ctx context.Context, args BlockNumberArgs) *Account { return &Account{ - backend: l.backend, + r: l.r, address: l.log.Address, blockNrOrHash: args.NumberOrLatest(), } @@ -183,30 +183,30 @@ func (at *AccessTuple) StorageKeys(ctx context.Context) []common.Hash { // Transaction represents an Ethereum transaction. // backend and hash are mandatory; all others will be fetched when required. type Transaction struct { - backend ethapi.Backend - hash common.Hash - tx *types.Transaction - block *Block - index uint64 + r *Resolver + hash common.Hash + tx *types.Transaction + block *Block + index uint64 } // resolve returns the internal transaction object, fetching it if needed. func (t *Transaction) resolve(ctx context.Context) (*types.Transaction, error) { if t.tx == nil { // Try to return an already finalized transaction - tx, blockHash, _, index, err := t.backend.GetTransaction(ctx, t.hash) + tx, blockHash, _, index, err := t.r.backend.GetTransaction(ctx, t.hash) if err == nil && tx != nil { t.tx = tx blockNrOrHash := rpc.BlockNumberOrHashWithHash(blockHash, false) t.block = &Block{ - backend: t.backend, + r: t.r, numberOrHash: &blockNrOrHash, } t.index = index return t.tx, nil } // No finalized transaction, try to retrieve it from the pool - t.tx = t.backend.GetPoolTransaction(t.hash) + t.tx = t.r.backend.GetPoolTransaction(t.hash) } return t.tx, nil } @@ -354,7 +354,7 @@ func (t *Transaction) To(ctx context.Context, args BlockNumberArgs) (*Account, e return nil, nil } return &Account{ - backend: t.backend, + r: t.r, address: *to, blockNrOrHash: args.NumberOrLatest(), }, nil @@ -365,10 +365,10 @@ func (t *Transaction) From(ctx context.Context, args BlockNumberArgs) (*Account, if err != nil || tx == nil { return nil, err } - signer := types.LatestSigner(t.backend.ChainConfig()) + signer := types.LatestSigner(t.r.backend.ChainConfig()) from, _ := types.Sender(signer, tx) return &Account{ - backend: t.backend, + r: t.r, address: from, blockNrOrHash: args.NumberOrLatest(), }, nil @@ -443,7 +443,7 @@ func (t *Transaction) CreatedContract(ctx context.Context, args BlockNumberArgs) return nil, err } return &Account{ - backend: t.backend, + r: t.r, address: receipt.ContractAddress, blockNrOrHash: args.NumberOrLatest(), }, nil @@ -457,7 +457,7 @@ func (t *Transaction) Logs(ctx context.Context) (*[]*Log, error) { ret := make([]*Log, 0, len(receipt.Logs)) for _, log := range receipt.Logs { ret = append(ret, &Log{ - backend: t.backend, + r: t.r, transaction: t, log: log, }) @@ -539,7 +539,7 @@ type BlockType int // backend, and numberOrHash are mandatory. All other fields are lazily fetched // when required. type Block struct { - backend ethapi.Backend + r *Resolver numberOrHash *rpc.BlockNumberOrHash hash common.Hash header *types.Header @@ -558,7 +558,7 @@ func (b *Block) resolve(ctx context.Context) (*types.Block, error) { b.numberOrHash = &latest } var err error - b.block, err = b.backend.BlockByNumberOrHash(ctx, *b.numberOrHash) + b.block, err = b.r.backend.BlockByNumberOrHash(ctx, *b.numberOrHash) if b.block != nil && b.header == nil { b.header = b.block.Header() if hash, ok := b.numberOrHash.Hash(); ok { @@ -578,9 +578,9 @@ func (b *Block) resolveHeader(ctx context.Context) (*types.Header, error) { var err error if b.header == nil { if b.hash != (common.Hash{}) { - b.header, err = b.backend.HeaderByHash(ctx, b.hash) + b.header, err = b.r.backend.HeaderByHash(ctx, b.hash) } else { - b.header, err = b.backend.HeaderByNumberOrHash(ctx, *b.numberOrHash) + b.header, err = b.r.backend.HeaderByNumberOrHash(ctx, *b.numberOrHash) } } return b.header, err @@ -598,7 +598,7 @@ func (b *Block) resolveReceipts(ctx context.Context) ([]*types.Receipt, error) { } hash = header.Hash() } - receipts, err := b.backend.GetReceipts(ctx, hash) + receipts, err := b.r.backend.GetReceipts(ctx, hash) if err != nil { return nil, err } @@ -659,7 +659,7 @@ func (b *Block) NextBaseFeePerGas(ctx context.Context) (*hexutil.Big, error) { if err != nil { return nil, err } - chaincfg := b.backend.ChainConfig() + chaincfg := b.r.backend.ChainConfig() if header.BaseFee == nil { // Make sure next block doesn't enable EIP-1559 if !chaincfg.IsLondon(new(big.Int).Add(header.Number, common.Big1)) { @@ -679,7 +679,7 @@ func (b *Block) Parent(ctx context.Context) (*Block, error) { } num := rpc.BlockNumberOrHashWithNumber(rpc.BlockNumber(b.header.Number.Uint64() - 1)) return &Block{ - backend: b.backend, + r: b.r, numberOrHash: &num, hash: b.header.ParentHash, }, nil @@ -767,7 +767,7 @@ func (b *Block) Ommers(ctx context.Context) (*[]*Block, error) { for _, uncle := range block.Uncles() { blockNumberOrHash := rpc.BlockNumberOrHashWithHash(uncle.Hash(), false) ret = append(ret, &Block{ - backend: b.backend, + r: b.r, numberOrHash: &blockNumberOrHash, header: uncle, }) @@ -800,7 +800,7 @@ func (b *Block) TotalDifficulty(ctx context.Context) (hexutil.Big, error) { } h = header.Hash() } - td := b.backend.GetTd(ctx, h) + td := b.r.backend.GetTd(ctx, h) if td == nil { return hexutil.Big{}, fmt.Errorf("total difficulty not found %x", b.hash) } @@ -853,7 +853,7 @@ func (b *Block) Miner(ctx context.Context, args BlockNumberArgs) (*Account, erro return nil, err } return &Account{ - backend: b.backend, + r: b.r, address: header.Coinbase, blockNrOrHash: args.NumberOrLatest(), }, nil @@ -876,11 +876,11 @@ func (b *Block) Transactions(ctx context.Context) (*[]*Transaction, error) { ret := make([]*Transaction, 0, len(block.Transactions())) for i, tx := range block.Transactions() { ret = append(ret, &Transaction{ - backend: b.backend, - hash: tx.Hash(), - tx: tx, - block: b, - index: uint64(i), + r: b.r, + hash: tx.Hash(), + tx: tx, + block: b, + index: uint64(i), }) } return &ret, nil @@ -897,11 +897,11 @@ func (b *Block) TransactionAt(ctx context.Context, args struct{ Index int32 }) ( } tx := txs[args.Index] return &Transaction{ - backend: b.backend, - hash: tx.Hash(), - tx: tx, - block: b, - index: uint64(args.Index), + r: b.r, + hash: tx.Hash(), + tx: tx, + block: b, + index: uint64(args.Index), }, nil } @@ -917,7 +917,7 @@ func (b *Block) OmmerAt(ctx context.Context, args struct{ Index int32 }) (*Block uncle := uncles[args.Index] blockNumberOrHash := rpc.BlockNumberOrHashWithHash(uncle.Hash(), false) return &Block{ - backend: b.backend, + r: b.r, numberOrHash: &blockNumberOrHash, header: uncle, }, nil @@ -944,7 +944,7 @@ type BlockFilterCriteria struct { // runFilter accepts a filter and executes it, returning all its results as // `Log` objects. -func runFilter(ctx context.Context, be ethapi.Backend, filter *filters.Filter) ([]*Log, error) { +func runFilter(ctx context.Context, r *Resolver, filter *filters.Filter) ([]*Log, error) { logs, err := filter.Logs(ctx) if err != nil || logs == nil { return nil, err @@ -952,8 +952,8 @@ func runFilter(ctx context.Context, be ethapi.Backend, filter *filters.Filter) ( ret := make([]*Log, 0, len(logs)) for _, log := range logs { ret = append(ret, &Log{ - backend: be, - transaction: &Transaction{backend: be, hash: log.TxHash}, + r: r, + transaction: &Transaction{r: r, hash: log.TxHash}, log: log, }) } @@ -978,10 +978,10 @@ func (b *Block) Logs(ctx context.Context, args struct{ Filter BlockFilterCriteri hash = header.Hash() } // Construct the range filter - filter := filters.NewBlockFilter(b.backend, hash, addresses, topics) + filter := filters.NewBlockFilter(b.r.backend, hash, addresses, topics) // Run the filter and return all the logs - return runFilter(ctx, b.backend, filter) + return runFilter(ctx, b.r, filter) } func (b *Block) Account(ctx context.Context, args struct { @@ -994,7 +994,7 @@ func (b *Block) Account(ctx context.Context, args struct { } } return &Account{ - backend: b.backend, + r: b.r, address: args.Address, blockNrOrHash: *b.numberOrHash, }, nil @@ -1041,7 +1041,7 @@ func (b *Block) Call(ctx context.Context, args struct { return nil, err } } - result, err := ethapi.DoCall(ctx, b.backend, args.Data, *b.numberOrHash, nil, b.backend.RPCEVMTimeout(), b.backend.RPCGasCap()) + result, err := ethapi.DoCall(ctx, b.r.backend, args.Data, *b.numberOrHash, nil, b.r.backend.RPCEVMTimeout(), b.r.backend.RPCGasCap()) if err != nil { return nil, err } @@ -1066,31 +1066,31 @@ func (b *Block) EstimateGas(ctx context.Context, args struct { return 0, err } } - gas, err := ethapi.DoEstimateGas(ctx, b.backend, args.Data, *b.numberOrHash, b.backend.RPCGasCap()) + gas, err := ethapi.DoEstimateGas(ctx, b.r.backend, args.Data, *b.numberOrHash, b.r.backend.RPCGasCap()) return Long(gas), err } type Pending struct { - backend ethapi.Backend + r *Resolver } func (p *Pending) TransactionCount(ctx context.Context) (int32, error) { - txs, err := p.backend.GetPoolTransactions() + txs, err := p.r.backend.GetPoolTransactions() return int32(len(txs)), err } func (p *Pending) Transactions(ctx context.Context) (*[]*Transaction, error) { - txs, err := p.backend.GetPoolTransactions() + txs, err := p.r.backend.GetPoolTransactions() if err != nil { return nil, err } ret := make([]*Transaction, 0, len(txs)) for i, tx := range txs { ret = append(ret, &Transaction{ - backend: p.backend, - hash: tx.Hash(), - tx: tx, - index: uint64(i), + r: p.r, + hash: tx.Hash(), + tx: tx, + index: uint64(i), }) } return &ret, nil @@ -1101,7 +1101,7 @@ func (p *Pending) Account(ctx context.Context, args struct { }) *Account { pendingBlockNr := rpc.BlockNumberOrHashWithNumber(rpc.PendingBlockNumber) return &Account{ - backend: p.backend, + r: p.r, address: args.Address, blockNrOrHash: pendingBlockNr, } @@ -1111,7 +1111,7 @@ func (p *Pending) Call(ctx context.Context, args struct { Data ethapi.TransactionArgs }) (*CallResult, error) { pendingBlockNr := rpc.BlockNumberOrHashWithNumber(rpc.PendingBlockNumber) - result, err := ethapi.DoCall(ctx, p.backend, args.Data, pendingBlockNr, nil, p.backend.RPCEVMTimeout(), p.backend.RPCGasCap()) + result, err := ethapi.DoCall(ctx, p.r.backend, args.Data, pendingBlockNr, nil, p.r.backend.RPCEVMTimeout(), p.r.backend.RPCGasCap()) if err != nil { return nil, err } @@ -1131,7 +1131,7 @@ func (p *Pending) EstimateGas(ctx context.Context, args struct { Data ethapi.TransactionArgs }) (Long, error) { pendingBlockNr := rpc.BlockNumberOrHashWithNumber(rpc.PendingBlockNumber) - gas, err := ethapi.DoEstimateGas(ctx, p.backend, args.Data, pendingBlockNr, p.backend.RPCGasCap()) + gas, err := ethapi.DoEstimateGas(ctx, p.r.backend, args.Data, pendingBlockNr, p.r.backend.RPCGasCap()) return Long(gas), err } @@ -1152,19 +1152,19 @@ func (r *Resolver) Block(ctx context.Context, args struct { number := rpc.BlockNumber(*args.Number) numberOrHash := rpc.BlockNumberOrHashWithNumber(number) block = &Block{ - backend: r.backend, + r: r, numberOrHash: &numberOrHash, } } else if args.Hash != nil { numberOrHash := rpc.BlockNumberOrHashWithHash(*args.Hash, false) block = &Block{ - backend: r.backend, + r: r, numberOrHash: &numberOrHash, } } else { numberOrHash := rpc.BlockNumberOrHashWithNumber(rpc.LatestBlockNumber) block = &Block{ - backend: r.backend, + r: r, numberOrHash: &numberOrHash, } } @@ -1199,7 +1199,7 @@ func (r *Resolver) Blocks(ctx context.Context, args struct { for i := from; i <= to; i++ { numberOrHash := rpc.BlockNumberOrHashWithNumber(i) block := &Block{ - backend: r.backend, + r: r, numberOrHash: &numberOrHash, } // Resolve the header to check for existence. @@ -1218,13 +1218,13 @@ func (r *Resolver) Blocks(ctx context.Context, args struct { } func (r *Resolver) Pending(ctx context.Context) *Pending { - return &Pending{r.backend} + return &Pending{r} } func (r *Resolver) Transaction(ctx context.Context, args struct{ Hash common.Hash }) (*Transaction, error) { tx := &Transaction{ - backend: r.backend, - hash: args.Hash, + r: r, + hash: args.Hash, } // Resolve the transaction; if it doesn't exist, return nil. t, err := tx.resolve(ctx) @@ -1284,8 +1284,8 @@ func (r *Resolver) Logs(ctx context.Context, args struct{ Filter FilterCriteria topics = *args.Filter.Topics } // Construct the range filter - filter := filters.NewRangeFilter(filters.Backend(r.backend), begin, end, addresses, topics) - return runFilter(ctx, r.backend, filter) + filter := filters.NewRangeFilter(r.backend, begin, end, addresses, topics) + return runFilter(ctx, r, filter) } func (r *Resolver) GasPrice(ctx context.Context) (hexutil.Big, error) { diff --git a/graphql/service.go b/graphql/service.go index 396a47700..1a2ffaa94 100644 --- a/graphql/service.go +++ b/graphql/service.go @@ -56,10 +56,6 @@ func (h handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // New constructs a new GraphQL service instance. func New(stack *node.Node, backend ethapi.Backend, cors, vhosts []string) error { - if backend == nil { - panic("missing backend") - } - // check if http server with given endpoint exists and enable graphQL on it return newHandler(stack, backend, cors, vhosts) } From 733d76a88d7a28434ab46bfe4f8e45821e7568b5 Mon Sep 17 00:00:00 2001 From: Delweng Date: Thu, 4 Aug 2022 15:42:03 +0800 Subject: [PATCH 23/59] node: remove noop path.Join (#25475) Signed-off-by: Delweng --- node/config_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/node/config_test.go b/node/config_test.go index 62b2fbbd1..e8af8ddcd 100644 --- a/node/config_test.go +++ b/node/config_test.go @@ -115,7 +115,7 @@ func TestNodeKeyPersistency(t *testing.T) { } config := &Config{Name: "unit-test", DataDir: dir, P2P: p2p.Config{PrivateKey: key}} config.NodeKey() - if _, err := os.Stat(filepath.Join(keyfile)); err == nil { + if _, err := os.Stat(keyfile); err == nil { t.Fatalf("one-shot node key persisted to data directory") } @@ -136,7 +136,7 @@ func TestNodeKeyPersistency(t *testing.T) { // Configure a new node and ensure the previously persisted key is loaded config = &Config{Name: "unit-test", DataDir: dir} config.NodeKey() - blob2, err := os.ReadFile(filepath.Join(keyfile)) + blob2, err := os.ReadFile(keyfile) if err != nil { t.Fatalf("failed to read previously persisted node key: %v", err) } From 8b53b92eb4628124ef7416ed515b9ffd0e0d71ff Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Thu, 4 Aug 2022 16:03:20 +0800 Subject: [PATCH 24/59] core, trie: rework trie committer (#25320) * all: rework trie and trie committer * all: get rid of internal cache in trie * all: fixes * trie: polish * core, trie: address comments * trie: fix imports * core/state: address comments * core/state/snapshot: polish * trie: remove unused code * trie: update tests * trie: don't set db as nil * trie: address comments * trie: unskip test --- core/blockchain.go | 6 +- core/state/database.go | 10 +- core/state/metrics.go | 12 +- core/state/snapshot/generate.go | 5 +- core/state/snapshot/generate_test.go | 52 +++---- core/state/state_object.go | 11 +- core/state/statedb.go | 47 ++++--- eth/protocols/snap/sync_test.go | 108 +++++++++++---- light/postprocess.go | 26 +++- light/trie.go | 6 +- tests/fuzzers/stacktrie/trie_fuzzer.go | 5 +- tests/fuzzers/trie/trie-fuzzer.go | 14 +- trie/committer.go | 185 +++++++++---------------- trie/database.go | 44 +++++- trie/iterator.go | 3 +- trie/iterator_test.go | 56 +++++--- trie/nodeset.go | 94 +++++++++++++ trie/proof.go | 14 +- trie/secure_trie.go | 16 ++- trie/secure_trie_test.go | 15 +- trie/sync_test.go | 13 +- trie/trie.go | 116 ++++++---------- trie/trie_test.go | 150 +++++++++----------- trie/util_test.go | 11 +- 24 files changed, 587 insertions(+), 432 deletions(-) create mode 100644 trie/nodeset.go diff --git a/core/blockchain.go b/core/blockchain.go index 506034b53..3638a1dce 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1244,7 +1244,7 @@ func (bc *BlockChain) writeKnownBlock(block *types.Block) error { // writeBlockWithState writes block, metadata and corresponding state data to the // database. -func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.Receipt, logs []*types.Log, state *state.StateDB) error { +func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.Receipt, state *state.StateDB) error { // Calculate the total difficulty of the block ptd := bc.GetTd(block.ParentHash(), block.NumberU64()-1) if ptd == nil { @@ -1339,7 +1339,7 @@ func (bc *BlockChain) WriteBlockAndSetHead(block *types.Block, receipts []*types // writeBlockAndSetHead is the internal implementation of WriteBlockAndSetHead. // This function expects the chain mutex to be held. func (bc *BlockChain) writeBlockAndSetHead(block *types.Block, receipts []*types.Receipt, logs []*types.Log, state *state.StateDB, emitHeadEvent bool) (status WriteStatus, err error) { - if err := bc.writeBlockWithState(block, receipts, logs, state); err != nil { + if err := bc.writeBlockWithState(block, receipts, state); err != nil { return NonStatTy, err } currentBlock := bc.CurrentBlock() @@ -1703,7 +1703,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals, setHead bool) var status WriteStatus if !setHead { // Don't set the head, only insert the block - err = bc.writeBlockWithState(block, receipts, logs, statedb) + err = bc.writeBlockWithState(block, receipts, statedb) } else { status, err = bc.writeBlockAndSetHead(block, receipts, logs, statedb, false) } diff --git a/core/state/database.go b/core/state/database.go index ce5d8d731..8f662ecd3 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -88,9 +88,13 @@ type Trie interface { // can be used even if the trie doesn't have one. Hash() common.Hash - // Commit writes all nodes to the trie's memory database, tracking the internal - // and external (for account tries) references. - Commit(onleaf trie.LeafCallback) (common.Hash, int, error) + // Commit collects all dirty nodes in the trie and replace them with the + // corresponding node hash. All collected nodes(including dirty leaves if + // collectLeaf is true) will be encapsulated into a nodeset for return. + // The returned nodeset can be nil if the trie is clean(nothing to commit). + // Once the trie is committed, it's not usable anymore. A new trie must + // be created with new root and updated trie database for following usage + Commit(collectLeaf bool) (common.Hash, *trie.NodeSet, error) // NodeIterator returns an iterator that returns nodes of the trie. Iteration // starts at the key after the given start key. diff --git a/core/state/metrics.go b/core/state/metrics.go index 7b40ff37a..35d2df92d 100644 --- a/core/state/metrics.go +++ b/core/state/metrics.go @@ -19,10 +19,10 @@ package state import "github.com/ethereum/go-ethereum/metrics" var ( - accountUpdatedMeter = metrics.NewRegisteredMeter("state/update/account", nil) - storageUpdatedMeter = metrics.NewRegisteredMeter("state/update/storage", nil) - accountDeletedMeter = metrics.NewRegisteredMeter("state/delete/account", nil) - storageDeletedMeter = metrics.NewRegisteredMeter("state/delete/storage", nil) - accountCommittedMeter = metrics.NewRegisteredMeter("state/commit/account", nil) - storageCommittedMeter = metrics.NewRegisteredMeter("state/commit/storage", nil) + accountUpdatedMeter = metrics.NewRegisteredMeter("state/update/account", nil) + storageUpdatedMeter = metrics.NewRegisteredMeter("state/update/storage", nil) + accountDeletedMeter = metrics.NewRegisteredMeter("state/delete/account", nil) + storageDeletedMeter = metrics.NewRegisteredMeter("state/delete/storage", nil) + accountTrieCommittedMeter = metrics.NewRegisteredMeter("state/commit/accountnodes", nil) + storageTriesCommittedMeter = metrics.NewRegisteredMeter("state/commit/storagenodes", nil) ) diff --git a/core/state/snapshot/generate.go b/core/state/snapshot/generate.go index 36055856e..bf714db4c 100644 --- a/core/state/snapshot/generate.go +++ b/core/state/snapshot/generate.go @@ -367,7 +367,10 @@ func (dl *diskLayer) generateRange(ctx *generatorContext, owner common.Hash, roo for i, key := range result.keys { snapTrie.Update(key, result.vals[i]) } - root, _, _ := snapTrie.Commit(nil) + root, nodes, _ := snapTrie.Commit(false) + if nodes != nil { + snapTrieDb.Update(trie.NewWithNodeSet(nodes)) + } snapTrieDb.Commit(root, false, nil) } // Construct the trie for state iteration, reuse the trie diff --git a/core/state/snapshot/generate_test.go b/core/state/snapshot/generate_test.go index fe81993e9..8d89ca59a 100644 --- a/core/state/snapshot/generate_test.go +++ b/core/state/snapshot/generate_test.go @@ -143,6 +143,7 @@ type testHelper struct { diskdb ethdb.Database triedb *trie.Database accTrie *trie.SecureTrie + nodes *trie.MergedNodeSet } func newHelper() *testHelper { @@ -153,6 +154,7 @@ func newHelper() *testHelper { diskdb: diskdb, triedb: triedb, accTrie: accTrie, + nodes: trie.NewMergedNodeSet(), } } @@ -184,17 +186,22 @@ func (t *testHelper) makeStorageTrie(stateRoot, owner common.Hash, keys []string for i, k := range keys { stTrie.Update([]byte(k), []byte(vals[i])) } - var root common.Hash if !commit { - root = stTrie.Hash() - } else { - root, _, _ = stTrie.Commit(nil) + return stTrie.Hash().Bytes() + } + root, nodes, _ := stTrie.Commit(false) + if nodes != nil { + t.nodes.Merge(nodes) } return root.Bytes() } func (t *testHelper) Commit() common.Hash { - root, _, _ := t.accTrie.Commit(nil) + root, nodes, _ := t.accTrie.Commit(true) + if nodes != nil { + t.nodes.Merge(nodes) + } + t.triedb.Update(t.nodes) t.triedb.Commit(root, false, nil) return root } @@ -378,7 +385,7 @@ func TestGenerateCorruptAccountTrie(t *testing.T) { helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) // 0x19ead688e907b0fab07176120dceec244a72aff2f0aa51e8b827584e378772f4 - root, _, _ := helper.accTrie.Commit(nil) // Root: 0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978 + root := helper.Commit() // Root: 0xa04693ea110a31037fb5ee814308a6f1d76bdab0b11676bdf4541d2de55ba978 // Delete an account trie leaf and ensure the generator chokes helper.triedb.Commit(root, false, nil) @@ -413,18 +420,8 @@ func TestGenerateMissingStorageTrie(t *testing.T) { helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) // 0x65145f923027566669a1ae5ccac66f945b55ff6eaeb17d2ea8e048b7d381f2d7 stRoot = helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: emptyCode.Bytes()}) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2 - root, _, _ := helper.accTrie.Commit(nil) - // We can only corrupt the disk database, so flush the tries out - helper.triedb.Reference( - common.BytesToHash(stRoot), - common.HexToHash("0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e"), - ) - helper.triedb.Reference( - common.BytesToHash(stRoot), - common.HexToHash("0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2"), - ) - helper.triedb.Commit(root, false, nil) + root := helper.Commit() // Delete a storage trie root and ensure the generator chokes helper.diskdb.Delete(stRoot) @@ -458,18 +455,7 @@ func TestGenerateCorruptStorageTrie(t *testing.T) { stRoot = helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: emptyCode.Bytes()}) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2 - root, _, _ := helper.accTrie.Commit(nil) - - // We can only corrupt the disk database, so flush the tries out - helper.triedb.Reference( - common.BytesToHash(stRoot), - common.HexToHash("0x9250573b9c18c664139f3b6a7a8081b7d8f8916a8fcc5d94feec6c29f5fd4e9e"), - ) - helper.triedb.Reference( - common.BytesToHash(stRoot), - common.HexToHash("0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2"), - ) - helper.triedb.Commit(root, false, nil) + root := helper.Commit() // Delete a storage trie leaf and ensure the generator chokes helper.diskdb.Delete(common.HexToHash("0x18a0f4d79cff4459642dd7604f303886ad9d77c30cf3d7d7cedb3a693ab6d371").Bytes()) @@ -825,10 +811,12 @@ func populateDangling(disk ethdb.KeyValueStore) { // This test will populate some dangling storages to see if they can be cleaned up. func TestGenerateCompleteSnapshotWithDanglingStorage(t *testing.T) { var helper = newHelper() - stRoot := helper.makeStorageTrie(common.Hash{}, common.Hash{}, []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) helper.addAccount("acc-2", &Account{Balance: big.NewInt(1), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) + + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addAccount("acc-3", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) helper.addSnapStorage("acc-1", []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}) @@ -858,10 +846,12 @@ func TestGenerateCompleteSnapshotWithDanglingStorage(t *testing.T) { // This test will populate some dangling storages to see if they can be cleaned up. func TestGenerateBrokenSnapshotWithDanglingStorage(t *testing.T) { var helper = newHelper() - stRoot := helper.makeStorageTrie(common.Hash{}, common.Hash{}, []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) + stRoot := helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-1")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addTrieAccount("acc-1", &Account{Balance: big.NewInt(1), Root: stRoot, CodeHash: emptyCode.Bytes()}) helper.addTrieAccount("acc-2", &Account{Balance: big.NewInt(2), Root: emptyRoot.Bytes(), CodeHash: emptyCode.Bytes()}) + + helper.makeStorageTrie(common.Hash{}, hashData([]byte("acc-3")), []string{"key-1", "key-2", "key-3"}, []string{"val-1", "val-2", "val-3"}, true) helper.addTrieAccount("acc-3", &Account{Balance: big.NewInt(3), Root: stRoot, CodeHash: emptyCode.Bytes()}) populateDangling(helper.diskdb) diff --git a/core/state/state_object.go b/core/state/state_object.go index bc1ca1f40..a23df8954 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -28,6 +28,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/trie" ) var emptyCodeHash = crypto.Keccak256(nil) @@ -375,23 +376,23 @@ func (s *stateObject) updateRoot(db Database) { // CommitTrie the storage trie of the object to db. // This updates the trie root. -func (s *stateObject) CommitTrie(db Database) (int, error) { +func (s *stateObject) CommitTrie(db Database) (*trie.NodeSet, error) { // If nothing changed, don't bother with hashing anything if s.updateTrie(db) == nil { - return 0, nil + return nil, nil } if s.dbErr != nil { - return 0, s.dbErr + return nil, s.dbErr } // Track the amount of time wasted on committing the storage trie if metrics.EnabledExpensive { defer func(start time.Time) { s.db.StorageCommits += time.Since(start) }(time.Now()) } - root, committed, err := s.trie.Commit(nil) + root, nodes, err := s.trie.Commit(false) if err == nil { s.data.Root = root } - return committed, err + return nodes, err } // AddBalance adds amount to s's balance. diff --git a/core/state/statedb.go b/core/state/statedb.go index e945ab595..322bc540b 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -774,7 +774,7 @@ func (s *StateDB) GetRefund() uint64 { return s.refund } -// Finalise finalises the state by removing the s destructed objects and clears +// Finalise finalises the state by removing the destructed objects and clears // the journal as well as the refunds. Finalise, however, will not push any updates // into the tries just yet. Only IntermediateRoot or Commit will do that. func (s *StateDB) Finalise(deleteEmptyObjects bool) { @@ -844,7 +844,7 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { // Although naively it makes sense to retrieve the account trie and then do // the contract storage and account updates sequentially, that short circuits // the account prefetcher. Instead, let's process all the storage updates - // first, giving the account prefeches just a few more milliseconds of time + // first, giving the account prefetches just a few more milliseconds of time // to pull useful data from disk. for addr := range s.stateObjectsPending { if obj := s.stateObjects[addr]; !obj.deleted { @@ -907,7 +907,11 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { s.IntermediateRoot(deleteEmptyObjects) // Commit objects to the trie, measuring the elapsed time - var storageCommitted int + var ( + accountTrieNodes int + storageTrieNodes int + nodes = trie.NewMergedNodeSet() + ) codeWriter := s.db.TrieDB().DiskDB().NewBatch() for addr := range s.stateObjectsDirty { if obj := s.stateObjects[addr]; !obj.deleted { @@ -917,11 +921,17 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { obj.dirtyCode = false } // Write any storage changes in the state object to its storage trie - committed, err := obj.CommitTrie(s.db) + set, err := obj.CommitTrie(s.db) if err != nil { return common.Hash{}, err } - storageCommitted += committed + // Merge the dirty nodes of storage trie into global set + if set != nil { + if err := nodes.Merge(set); err != nil { + return common.Hash{}, err + } + storageTrieNodes += set.Len() + } } } if len(s.stateObjectsDirty) > 0 { @@ -937,21 +947,17 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { if metrics.EnabledExpensive { start = time.Now() } - // The onleaf func is called _serially_, so we can reuse the same account - // for unmarshalling every time. - var account types.StateAccount - root, accountCommitted, err := s.trie.Commit(func(_ [][]byte, _ []byte, leaf []byte, parent common.Hash, _ []byte) error { - if err := rlp.DecodeBytes(leaf, &account); err != nil { - return nil - } - if account.Root != emptyRoot { - s.db.TrieDB().Reference(account.Root, parent) - } - return nil - }) + root, set, err := s.trie.Commit(true) if err != nil { return common.Hash{}, err } + // Merge the dirty nodes of account trie into global set + if set != nil { + if err := nodes.Merge(set); err != nil { + return common.Hash{}, err + } + accountTrieNodes = set.Len() + } if metrics.EnabledExpensive { s.AccountCommits += time.Since(start) @@ -959,8 +965,8 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { storageUpdatedMeter.Mark(int64(s.StorageUpdated)) accountDeletedMeter.Mark(int64(s.AccountDeleted)) storageDeletedMeter.Mark(int64(s.StorageDeleted)) - accountCommittedMeter.Mark(int64(accountCommitted)) - storageCommittedMeter.Mark(int64(storageCommitted)) + accountTrieCommittedMeter.Mark(int64(accountTrieNodes)) + storageTriesCommittedMeter.Mark(int64(storageTrieNodes)) s.AccountUpdated, s.AccountDeleted = 0, 0 s.StorageUpdated, s.StorageDeleted = 0, 0 } @@ -984,6 +990,9 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { } s.snap, s.snapDestructs, s.snapAccounts, s.snapStorage = nil, nil, nil, nil } + if err := s.db.TrieDB().Update(nodes); err != nil { + return common.Hash{}, err + } s.originalRoot = root return root, err } diff --git a/eth/protocols/snap/sync_test.go b/eth/protocols/snap/sync_test.go index 85e4dc5e4..a13e8d308 100644 --- a/eth/protocols/snap/sync_test.go +++ b/eth/protocols/snap/sync_test.go @@ -1348,9 +1348,11 @@ func getCodeByHash(hash common.Hash) []byte { // makeAccountTrieNoStorage spits out a trie, along with the leafs func makeAccountTrieNoStorage(n int) (*trie.Trie, entrySlice) { - db := trie.NewDatabase(rawdb.NewMemoryDatabase()) - accTrie := trie.NewEmpty(db) - var entries entrySlice + var ( + db = trie.NewDatabase(rawdb.NewMemoryDatabase()) + accTrie = trie.NewEmpty(db) + entries entrySlice + ) for i := uint64(1); i <= uint64(n); i++ { value, _ := rlp.EncodeToBytes(&types.StateAccount{ Nonce: i, @@ -1364,7 +1366,13 @@ func makeAccountTrieNoStorage(n int) (*trie.Trie, entrySlice) { entries = append(entries, elem) } sort.Sort(entries) - accTrie.Commit(nil) + + // Commit the state changes into db and re-create the trie + // for accessing later. + root, nodes, _ := accTrie.Commit(false) + db.Update(trie.NewWithNodeSet(nodes)) + + accTrie, _ = trie.New(common.Hash{}, root, db) return accTrie, entries } @@ -1376,8 +1384,8 @@ func makeBoundaryAccountTrie(n int) (*trie.Trie, entrySlice) { entries entrySlice boundaries []common.Hash - db = trie.NewDatabase(rawdb.NewMemoryDatabase()) - trie = trie.NewEmpty(db) + db = trie.NewDatabase(rawdb.NewMemoryDatabase()) + accTrie = trie.NewEmpty(db) ) // Initialize boundaries var next common.Hash @@ -1404,7 +1412,7 @@ func makeBoundaryAccountTrie(n int) (*trie.Trie, entrySlice) { CodeHash: getCodeHash(uint64(i)), }) elem := &kv{boundaries[i].Bytes(), value} - trie.Update(elem.k, elem.v) + accTrie.Update(elem.k, elem.v) entries = append(entries, elem) } // Fill other accounts if required @@ -1416,12 +1424,18 @@ func makeBoundaryAccountTrie(n int) (*trie.Trie, entrySlice) { CodeHash: getCodeHash(i), }) elem := &kv{key32(i), value} - trie.Update(elem.k, elem.v) + accTrie.Update(elem.k, elem.v) entries = append(entries, elem) } sort.Sort(entries) - trie.Commit(nil) - return trie, entries + + // Commit the state changes into db and re-create the trie + // for accessing later. + root, nodes, _ := accTrie.Commit(false) + db.Update(trie.NewWithNodeSet(nodes)) + + accTrie, _ = trie.New(common.Hash{}, root, db) + return accTrie, entries } // makeAccountTrieWithStorageWithUniqueStorage creates an account trie where each accounts @@ -1431,8 +1445,10 @@ func makeAccountTrieWithStorageWithUniqueStorage(accounts, slots int, code bool) db = trie.NewDatabase(rawdb.NewMemoryDatabase()) accTrie = trie.NewEmpty(db) entries entrySlice + storageRoots = make(map[common.Hash]common.Hash) storageTries = make(map[common.Hash]*trie.Trie) storageEntries = make(map[common.Hash]entrySlice) + nodes = trie.NewMergedNodeSet() ) // Create n accounts in the trie for i := uint64(1); i <= uint64(accounts); i++ { @@ -1442,9 +1458,9 @@ func makeAccountTrieWithStorageWithUniqueStorage(accounts, slots int, code bool) codehash = getCodeHash(i) } // Create a storage trie - stTrie, stEntries := makeStorageTrieWithSeed(common.BytesToHash(key), uint64(slots), i, db) - stRoot := stTrie.Hash() - stTrie.Commit(nil) + stRoot, stNodes, stEntries := makeStorageTrieWithSeed(common.BytesToHash(key), uint64(slots), i, db) + nodes.Merge(stNodes) + value, _ := rlp.EncodeToBytes(&types.StateAccount{ Nonce: i, Balance: big.NewInt(int64(i)), @@ -1455,12 +1471,25 @@ func makeAccountTrieWithStorageWithUniqueStorage(accounts, slots int, code bool) accTrie.Update(elem.k, elem.v) entries = append(entries, elem) - storageTries[common.BytesToHash(key)] = stTrie + storageRoots[common.BytesToHash(key)] = stRoot storageEntries[common.BytesToHash(key)] = stEntries } sort.Sort(entries) - accTrie.Commit(nil) + // Commit account trie + root, set, _ := accTrie.Commit(true) + nodes.Merge(set) + + // Commit gathered dirty nodes into database + db.Update(nodes) + + // Re-create tries with new root + accTrie, _ = trie.New(common.Hash{}, root, db) + for i := uint64(1); i <= uint64(accounts); i++ { + key := key32(i) + trie, _ := trie.New(common.BytesToHash(key), storageRoots[common.BytesToHash(key)], db) + storageTries[common.BytesToHash(key)] = trie + } return accTrie, entries, storageTries, storageEntries } @@ -1470,8 +1499,10 @@ func makeAccountTrieWithStorage(accounts, slots int, code, boundary bool) (*trie db = trie.NewDatabase(rawdb.NewMemoryDatabase()) accTrie = trie.NewEmpty(db) entries entrySlice + storageRoots = make(map[common.Hash]common.Hash) storageTries = make(map[common.Hash]*trie.Trie) storageEntries = make(map[common.Hash]entrySlice) + nodes = trie.NewMergedNodeSet() ) // Create n accounts in the trie for i := uint64(1); i <= uint64(accounts); i++ { @@ -1482,16 +1513,16 @@ func makeAccountTrieWithStorage(accounts, slots int, code, boundary bool) (*trie } // Make a storage trie var ( - stTrie *trie.Trie + stRoot common.Hash + stNodes *trie.NodeSet stEntries entrySlice ) if boundary { - stTrie, stEntries = makeBoundaryStorageTrie(common.BytesToHash(key), slots, db) + stRoot, stNodes, stEntries = makeBoundaryStorageTrie(common.BytesToHash(key), slots, db) } else { - stTrie, stEntries = makeStorageTrieWithSeed(common.BytesToHash(key), uint64(slots), 0, db) + stRoot, stNodes, stEntries = makeStorageTrieWithSeed(common.BytesToHash(key), uint64(slots), 0, db) } - stRoot := stTrie.Hash() - stTrie.Commit(nil) + nodes.Merge(stNodes) value, _ := rlp.EncodeToBytes(&types.StateAccount{ Nonce: i, @@ -1502,19 +1533,40 @@ func makeAccountTrieWithStorage(accounts, slots int, code, boundary bool) (*trie elem := &kv{key, value} accTrie.Update(elem.k, elem.v) entries = append(entries, elem) + // we reuse the same one for all accounts - storageTries[common.BytesToHash(key)] = stTrie + storageRoots[common.BytesToHash(key)] = stRoot storageEntries[common.BytesToHash(key)] = stEntries } sort.Sort(entries) - accTrie.Commit(nil) + + // Commit account trie + root, set, _ := accTrie.Commit(true) + nodes.Merge(set) + + // Commit gathered dirty nodes into database + db.Update(nodes) + + // Re-create tries with new root + accTrie, err := trie.New(common.Hash{}, root, db) + if err != nil { + panic(err) + } + for i := uint64(1); i <= uint64(accounts); i++ { + key := key32(i) + trie, err := trie.New(common.BytesToHash(key), storageRoots[common.BytesToHash(key)], db) + if err != nil { + panic(err) + } + storageTries[common.BytesToHash(key)] = trie + } return accTrie, entries, storageTries, storageEntries } // makeStorageTrieWithSeed fills a storage trie with n items, returning the // not-yet-committed trie and the sorted entries. The seeds can be used to ensure // that tries are unique. -func makeStorageTrieWithSeed(owner common.Hash, n, seed uint64, db *trie.Database) (*trie.Trie, entrySlice) { +func makeStorageTrieWithSeed(owner common.Hash, n, seed uint64, db *trie.Database) (common.Hash, *trie.NodeSet, entrySlice) { trie, _ := trie.New(owner, common.Hash{}, db) var entries entrySlice for i := uint64(1); i <= n; i++ { @@ -1530,14 +1582,14 @@ func makeStorageTrieWithSeed(owner common.Hash, n, seed uint64, db *trie.Databas entries = append(entries, elem) } sort.Sort(entries) - trie.Commit(nil) - return trie, entries + root, nodes, _ := trie.Commit(false) + return root, nodes, entries } // makeBoundaryStorageTrie constructs a storage trie. Instead of filling // storage slots normally, this function will fill a few slots which have // boundary hash. -func makeBoundaryStorageTrie(owner common.Hash, n int, db *trie.Database) (*trie.Trie, entrySlice) { +func makeBoundaryStorageTrie(owner common.Hash, n int, db *trie.Database) (common.Hash, *trie.NodeSet, entrySlice) { var ( entries entrySlice boundaries []common.Hash @@ -1581,8 +1633,8 @@ func makeBoundaryStorageTrie(owner common.Hash, n int, db *trie.Database) (*trie entries = append(entries, elem) } sort.Sort(entries) - trie.Commit(nil) - return trie, entries + root, nodes, _ := trie.Commit(false) + return root, nodes, entries } func verifyTrie(db ethdb.KeyValueStore, root common.Hash, t *testing.T) { diff --git a/light/postprocess.go b/light/postprocess.go index c09b00e71..0e50dab96 100644 --- a/light/postprocess.go +++ b/light/postprocess.go @@ -217,7 +217,18 @@ func (c *ChtIndexerBackend) Process(ctx context.Context, header *types.Header) e // Commit implements core.ChainIndexerBackend func (c *ChtIndexerBackend) Commit() error { - root, _, err := c.trie.Commit(nil) + root, nodes, err := c.trie.Commit(false) + if err != nil { + return err + } + // Commit trie changes into trie database in case it's not nil. + if nodes != nil { + if err := c.triedb.Update(trie.NewWithNodeSet(nodes)); err != nil { + return err + } + } + // Re-create trie with newly generated root and updated database. + c.trie, err = trie.New(common.Hash{}, root, c.triedb) if err != nil { return err } @@ -453,7 +464,18 @@ func (b *BloomTrieIndexerBackend) Commit() error { b.trie.Delete(encKey[:]) } } - root, _, err := b.trie.Commit(nil) + root, nodes, err := b.trie.Commit(false) + if err != nil { + return err + } + // Commit trie changes into trie database in case it's not nil. + if nodes != nil { + if err := b.triedb.Update(trie.NewWithNodeSet(nodes)); err != nil { + return err + } + } + // Re-create trie with newly generated root and updated database. + b.trie, err = trie.New(common.Hash{}, root, b.triedb) if err != nil { return err } diff --git a/light/trie.go b/light/trie.go index 931ba30cb..a2ef8ebff 100644 --- a/light/trie.go +++ b/light/trie.go @@ -137,11 +137,11 @@ func (t *odrTrie) TryDelete(key []byte) error { }) } -func (t *odrTrie) Commit(onleaf trie.LeafCallback) (common.Hash, int, error) { +func (t *odrTrie) Commit(collectLeaf bool) (common.Hash, *trie.NodeSet, error) { if t.trie == nil { - return t.id.Root, 0, nil + return t.id.Root, nil, nil } - return t.trie.Commit(onleaf) + return t.trie.Commit(collectLeaf) } func (t *odrTrie) Hash() common.Hash { diff --git a/tests/fuzzers/stacktrie/trie_fuzzer.go b/tests/fuzzers/stacktrie/trie_fuzzer.go index 17d67a875..e6165df08 100644 --- a/tests/fuzzers/stacktrie/trie_fuzzer.go +++ b/tests/fuzzers/stacktrie/trie_fuzzer.go @@ -173,10 +173,13 @@ func (f *fuzzer) fuzz() int { return 0 } // Flush trie -> database - rootA, _, err := trieA.Commit(nil) + rootA, nodes, err := trieA.Commit(false) if err != nil { panic(err) } + if nodes != nil { + dbA.Update(trie.NewWithNodeSet(nodes)) + } // Flush memdb -> disk (sponge) dbA.Commit(rootA, false, nil) diff --git a/tests/fuzzers/trie/trie-fuzzer.go b/tests/fuzzers/trie/trie-fuzzer.go index ca1509085..f36b613d4 100644 --- a/tests/fuzzers/trie/trie-fuzzer.go +++ b/tests/fuzzers/trie/trie-fuzzer.go @@ -51,9 +51,8 @@ const ( opUpdate = iota opDelete opGet - opCommit opHash - opReset + opCommit opItercheckhash opProve opMax // boundary value, not an actual op @@ -157,15 +156,18 @@ func runRandTest(rt randTest) error { if string(v) != want { rt[i].err = fmt.Errorf("mismatch for key %#x, got %#x want %#x", step.key, v, want) } - case opCommit: - _, _, rt[i].err = tr.Commit(nil) case opHash: tr.Hash() - case opReset: - hash, _, err := tr.Commit(nil) + case opCommit: + hash, nodes, err := tr.Commit(false) if err != nil { return err } + if nodes != nil { + if err := triedb.Update(trie.NewWithNodeSet(nodes)); err != nil { + return err + } + } newtr, err := trie.New(common.Hash{}, hash, triedb) if err != nil { return err diff --git a/trie/committer.go b/trie/committer.go index 7a392abab..d9f0ecf3d 100644 --- a/trie/committer.go +++ b/trie/committer.go @@ -17,72 +17,48 @@ package trie import ( - "errors" "fmt" - "sync" "github.com/ethereum/go-ethereum/common" ) -// leafChanSize is the size of the leafCh. It's a pretty arbitrary number, to allow -// some parallelism but not incur too much memory overhead. -const leafChanSize = 200 - -// leaf represents a trie leaf value +// leaf represents a trie leaf node type leaf struct { - size int // size of the rlp data (estimate) - hash common.Hash // hash of rlp data - node node // the node to commit - path []byte // the path from the root node + blob []byte // raw blob of leaf + parent common.Hash // the hash of parent node } -// committer is a type used for the trie Commit operation. A committer has some -// internal preallocated temp space, and also a callback that is invoked when -// leaves are committed. The leafs are passed through the `leafCh`, to allow -// some level of parallelism. -// By 'some level' of parallelism, it's still the case that all leaves will be -// processed sequentially - onleaf will never be called in parallel or out of order. +// committer is the tool used for the trie Commit operation. The committer will +// capture all dirty nodes during the commit process and keep them cached in +// insertion order. type committer struct { - onleaf LeafCallback - leafCh chan *leaf -} - -// committers live in a global sync.Pool -var committerPool = sync.Pool{ - New: func() interface{} { - return &committer{} - }, + nodes *NodeSet + collectLeaf bool } // newCommitter creates a new committer or picks one from the pool. -func newCommitter() *committer { - return committerPool.Get().(*committer) -} - -func returnCommitterToPool(h *committer) { - h.onleaf = nil - h.leafCh = nil - committerPool.Put(h) +func newCommitter(owner common.Hash, collectLeaf bool) *committer { + return &committer{ + nodes: NewNodeSet(owner), + collectLeaf: collectLeaf, + } } // Commit collapses a node down into a hash node and inserts it into the database -func (c *committer) Commit(n node, db *Database) (hashNode, int, error) { - if db == nil { - return nil, 0, errors.New("no db provided") - } - h, committed, err := c.commit(nil, n, db) +func (c *committer) Commit(n node) (hashNode, *NodeSet, error) { + h, err := c.commit(nil, n) if err != nil { - return nil, 0, err + return nil, nil, err } - return h.(hashNode), committed, nil + return h.(hashNode), c.nodes, nil } // commit collapses a node down into a hash node and inserts it into the database -func (c *committer) commit(path []byte, n node, db *Database) (node, int, error) { +func (c *committer) commit(path []byte, n node) (node, error) { // if this path is clean, use available cached data hash, dirty := n.cache() if hash != nil && !dirty { - return hash, 0, nil + return hash, nil } // Commit children, then parent, and remove the dirty flag. switch cn := n.(type) { @@ -92,36 +68,35 @@ func (c *committer) commit(path []byte, n node, db *Database) (node, int, error) // If the child is fullNode, recursively commit, // otherwise it can only be hashNode or valueNode. - var childCommitted int if _, ok := cn.Val.(*fullNode); ok { - childV, committed, err := c.commit(append(path, cn.Key...), cn.Val, db) + childV, err := c.commit(append(path, cn.Key...), cn.Val) if err != nil { - return nil, 0, err + return nil, err } - collapsed.Val, childCommitted = childV, committed + collapsed.Val = childV } // The key needs to be copied, since we're delivering it to database collapsed.Key = hexToCompact(cn.Key) - hashedNode := c.store(path, collapsed, db) + hashedNode := c.store(path, collapsed) if hn, ok := hashedNode.(hashNode); ok { - return hn, childCommitted + 1, nil + return hn, nil } - return collapsed, childCommitted, nil + return collapsed, nil case *fullNode: - hashedKids, childCommitted, err := c.commitChildren(path, cn, db) + hashedKids, err := c.commitChildren(path, cn) if err != nil { - return nil, 0, err + return nil, err } collapsed := cn.copy() collapsed.Children = hashedKids - hashedNode := c.store(path, collapsed, db) + hashedNode := c.store(path, collapsed) if hn, ok := hashedNode.(hashNode); ok { - return hn, childCommitted + 1, nil + return hn, nil } - return collapsed, childCommitted, nil + return collapsed, nil case hashNode: - return cn, 0, nil + return cn, nil default: // nil, valuenode shouldn't be committed panic(fmt.Sprintf("%T: invalid node: %v", n, n)) @@ -129,11 +104,8 @@ func (c *committer) commit(path []byte, n node, db *Database) (node, int, error) } // commitChildren commits the children of the given fullnode -func (c *committer) commitChildren(path []byte, n *fullNode, db *Database) ([17]node, int, error) { - var ( - committed int - children [17]node - ) +func (c *committer) commitChildren(path []byte, n *fullNode) ([17]node, error) { + var children [17]node for i := 0; i < 16; i++ { child := n.Children[i] if child == nil { @@ -149,88 +121,63 @@ func (c *committer) commitChildren(path []byte, n *fullNode, db *Database) ([17] // Commit the child recursively and store the "hashed" value. // Note the returned node can be some embedded nodes, so it's // possible the type is not hashNode. - hashed, childCommitted, err := c.commit(append(path, byte(i)), child, db) + hashed, err := c.commit(append(path, byte(i)), child) if err != nil { - return children, 0, err + return children, err } children[i] = hashed - committed += childCommitted } // For the 17th child, it's possible the type is valuenode. if n.Children[16] != nil { children[16] = n.Children[16] } - return children, committed, nil + return children, nil } // store hashes the node n and if we have a storage layer specified, it writes // the key/value pair to it and tracks any node->child references as well as any // node->external trie references. -func (c *committer) store(path []byte, n node, db *Database) node { +func (c *committer) store(path []byte, n node) node { // Larger nodes are replaced by their hash and stored in the database. - var ( - hash, _ = n.cache() - size int - ) + var hash, _ = n.cache() + + // This was not generated - must be a small node stored in the parent. + // In theory, we should check if the node is leaf here (embedded node + // usually is leaf node). But small value(less than 32bytes) is not + // our target(leaves in account trie only). if hash == nil { - // This was not generated - must be a small node stored in the parent. - // In theory, we should apply the leafCall here if it's not nil(embedded - // node usually contains value). But small value(less than 32bytes) is - // not our target. return n - } else { - // We have the hash already, estimate the RLP encoding-size of the node. - // The size is used for mem tracking, does not need to be exact - size = estimateSize(n) } - // If we're using channel-based leaf-reporting, send to channel. - // The leaf channel will be active only when there an active leaf-callback - if c.leafCh != nil { - c.leafCh <- &leaf{ - size: size, - hash: common.BytesToHash(hash), - node: n, - path: path, + // We have the hash already, estimate the RLP encoding-size of the node. + // The size is used for mem tracking, does not need to be exact + var ( + size = estimateSize(n) + nhash = common.BytesToHash(hash) + mnode = &memoryNode{ + hash: nhash, + node: simplifyNode(n), + size: uint16(size), + } + ) + // Collect the dirty node to nodeset for return. + c.nodes.add(string(path), mnode) + + // Collect the corresponding leaf node if it's required. We don't check + // full node since it's impossible to store value in fullNode. The key + // length of leaves should be exactly same. + if c.collectLeaf { + if sn, ok := n.(*shortNode); ok { + if val, ok := sn.Val.(valueNode); ok { + c.nodes.addLeaf(&leaf{blob: val, parent: nhash}) + } } - } else if db != nil { - // No leaf-callback used, but there's still a database. Do serial - // insertion - db.insert(common.BytesToHash(hash), size, n) } return hash } -// commitLoop does the actual insert + leaf callback for nodes. -func (c *committer) commitLoop(db *Database) { - for item := range c.leafCh { - var ( - hash = item.hash - size = item.size - n = item.node - ) - // We are pooling the trie nodes into an intermediate memory cache - db.insert(hash, size, n) - - if c.onleaf != nil { - switch n := n.(type) { - case *shortNode: - if child, ok := n.Val.(valueNode); ok { - c.onleaf(nil, nil, child, hash, nil) - } - case *fullNode: - // For children in range [0, 15], it's impossible - // to contain valueNode. Only check the 17th child. - if n.Children[16] != nil { - c.onleaf(nil, nil, n.Children[16].(valueNode), hash, nil) - } - } - } - } -} - // estimateSize estimates the size of an rlp-encoded node, without actually // rlp-encoding it (zero allocs). This method has been experimentally tried, and with a trie -// with 1000 leafs, the only errors above 1% are on small shortnodes, where this +// with 1000 leaves, the only errors above 1% are on small shortnodes, where this // method overestimates by 2 or 3 bytes (e.g. 37 instead of 35) func estimateSize(n node) int { switch n := n.(type) { diff --git a/trie/database.go b/trie/database.go index 8e1788a21..81f0477ae 100644 --- a/trie/database.go +++ b/trie/database.go @@ -28,6 +28,7 @@ import ( "github.com/VictoriaMetrics/fastcache" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" @@ -305,14 +306,10 @@ func (db *Database) DiskDB() ethdb.KeyValueStore { return db.diskdb } -// insert inserts a collapsed trie node into the memory database. -// The blob size must be specified to allow proper size tracking. +// insert inserts a simplified trie node into the memory database. // All nodes inserted by this function will be reference tracked // and in theory should only used for **trie nodes** insertion. func (db *Database) insert(hash common.Hash, size int, node node) { - db.lock.Lock() - defer db.lock.Unlock() - // If the node's already cached, skip if _, ok := db.dirties[hash]; ok { return @@ -321,7 +318,7 @@ func (db *Database) insert(hash common.Hash, size int, node node) { // Create the cached entry for this node entry := &cachedNode{ - node: simplifyNode(node), + node: node, size: uint16(size), flushPrev: db.newest, } @@ -763,6 +760,41 @@ func (c *cleaner) Delete(key []byte) error { panic("not implemented") } +// Update inserts the dirty nodes in provided nodeset into database and +// link the account trie with multiple storage tries if necessary. +func (db *Database) Update(nodes *MergedNodeSet) error { + db.lock.Lock() + defer db.lock.Unlock() + + // Insert dirty nodes into the database. In the same tree, it must be + // ensured that children are inserted first, then parent so that children + // can be linked with their parent correctly. The order of writing between + // different tries(account trie, storage tries) is not required. + for owner, subset := range nodes.sets { + for _, path := range subset.paths { + n, ok := subset.nodes[path] + if !ok { + return fmt.Errorf("missing node %x %v", owner, path) + } + db.insert(n.hash, int(n.size), n.node) + } + } + // Link up the account trie and storage trie if the node points + // to an account trie leaf. + if set, present := nodes.sets[common.Hash{}]; present { + for _, n := range set.leaves { + var account types.StateAccount + if err := rlp.DecodeBytes(n.blob, &account); err != nil { + return err + } + if account.Root != emptyRoot { + db.reference(account.Root, n.parent) + } + } + } + return nil +} + // Size returns the current storage size of the memory cache in front of the // persistent database layer. func (db *Database) Size() (common.StorageSize, common.StorageSize) { diff --git a/trie/iterator.go b/trie/iterator.go index e0006ee05..1e76625c6 100644 --- a/trie/iterator.go +++ b/trie/iterator.go @@ -375,8 +375,7 @@ func (it *nodeIterator) resolveHash(hash hashNode, path []byte) (node, error) { } } } - resolved, err := it.trie.resolveHash(hash, path) - return resolved, err + return it.trie.resolveHash(hash, path) } func (it *nodeIterator) resolveBlob(hash hashNode, path []byte) ([]byte, error) { diff --git a/trie/iterator_test.go b/trie/iterator_test.go index e3e6d0e3a..0a4c04c8b 100644 --- a/trie/iterator_test.go +++ b/trie/iterator_test.go @@ -31,7 +31,7 @@ import ( ) func TestEmptyIterator(t *testing.T) { - trie := newEmpty() + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) iter := trie.NodeIterator(nil) seen := make(map[string]struct{}) @@ -44,7 +44,8 @@ func TestEmptyIterator(t *testing.T) { } func TestIterator(t *testing.T) { - trie := newEmpty() + db := NewDatabase(rawdb.NewMemoryDatabase()) + trie := NewEmpty(db) vals := []struct{ k, v string }{ {"do", "verb"}, {"ether", "wookiedoo"}, @@ -59,8 +60,13 @@ func TestIterator(t *testing.T) { all[val.k] = val.v trie.Update([]byte(val.k), []byte(val.v)) } - trie.Commit(nil) + root, nodes, err := trie.Commit(false) + if err != nil { + t.Fatalf("Failed to commit trie %v", err) + } + db.Update(NewWithNodeSet(nodes)) + trie, _ = New(common.Hash{}, root, db) found := make(map[string]string) it := NewIterator(trie.NodeIterator(nil)) for it.Next() { @@ -80,7 +86,7 @@ type kv struct { } func TestIteratorLargeData(t *testing.T) { - trie := newEmpty() + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) vals := make(map[string]*kv) for i := byte(0); i < 255; i++ { @@ -173,7 +179,7 @@ var testdata2 = []kvs{ } func TestIteratorSeek(t *testing.T) { - trie := newEmpty() + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) for _, val := range testdata1 { trie.Update([]byte(val.k), []byte(val.v)) } @@ -214,17 +220,23 @@ func checkIteratorOrder(want []kvs, it *Iterator) error { } func TestDifferenceIterator(t *testing.T) { - triea := newEmpty() + dba := NewDatabase(rawdb.NewMemoryDatabase()) + triea := NewEmpty(dba) for _, val := range testdata1 { triea.Update([]byte(val.k), []byte(val.v)) } - triea.Commit(nil) + rootA, nodesA, _ := triea.Commit(false) + dba.Update(NewWithNodeSet(nodesA)) + triea, _ = New(common.Hash{}, rootA, dba) - trieb := newEmpty() + dbb := NewDatabase(rawdb.NewMemoryDatabase()) + trieb := NewEmpty(dbb) for _, val := range testdata2 { trieb.Update([]byte(val.k), []byte(val.v)) } - trieb.Commit(nil) + rootB, nodesB, _ := trieb.Commit(false) + dbb.Update(NewWithNodeSet(nodesB)) + trieb, _ = New(common.Hash{}, rootB, dbb) found := make(map[string]string) di, _ := NewDifferenceIterator(triea.NodeIterator(nil), trieb.NodeIterator(nil)) @@ -250,17 +262,23 @@ func TestDifferenceIterator(t *testing.T) { } func TestUnionIterator(t *testing.T) { - triea := newEmpty() + dba := NewDatabase(rawdb.NewMemoryDatabase()) + triea := NewEmpty(dba) for _, val := range testdata1 { triea.Update([]byte(val.k), []byte(val.v)) } - triea.Commit(nil) + rootA, nodesA, _ := triea.Commit(false) + dba.Update(NewWithNodeSet(nodesA)) + triea, _ = New(common.Hash{}, rootA, dba) - trieb := newEmpty() + dbb := NewDatabase(rawdb.NewMemoryDatabase()) + trieb := NewEmpty(dbb) for _, val := range testdata2 { trieb.Update([]byte(val.k), []byte(val.v)) } - trieb.Commit(nil) + rootB, nodesB, _ := trieb.Commit(false) + dbb.Update(NewWithNodeSet(nodesB)) + trieb, _ = New(common.Hash{}, rootB, dbb) di, _ := NewUnionIterator([]NodeIterator{triea.NodeIterator(nil), trieb.NodeIterator(nil)}) it := NewIterator(di) @@ -316,7 +334,8 @@ func testIteratorContinueAfterError(t *testing.T, memonly bool) { for _, val := range testdata1 { tr.Update([]byte(val.k), []byte(val.v)) } - tr.Commit(nil) + _, nodes, _ := tr.Commit(false) + triedb.Update(NewWithNodeSet(nodes)) if !memonly { triedb.Commit(tr.Hash(), true, nil) } @@ -407,7 +426,8 @@ func testIteratorContinueAfterSeekError(t *testing.T, memonly bool) { for _, val := range testdata1 { ctr.Update([]byte(val.k), []byte(val.v)) } - root, _, _ := ctr.Commit(nil) + root, nodes, _ := ctr.Commit(false) + triedb.Update(NewWithNodeSet(nodes)) if !memonly { triedb.Commit(root, true, nil) } @@ -525,7 +545,8 @@ func makeLargeTestTrie() (*Database, *SecureTrie, *loggingDb) { val = crypto.Keccak256(val) trie.Update(key, val) } - trie.Commit(nil) + _, nodes, _ := trie.Commit(false) + triedb.Update(NewWithNodeSet(nodes)) // Return the generated trie return triedb, trie, logDb } @@ -564,7 +585,8 @@ func TestIteratorNodeBlob(t *testing.T) { all[val.k] = val.v trie.Update([]byte(val.k), []byte(val.v)) } - trie.Commit(nil) + _, nodes, _ := trie.Commit(false) + triedb.Update(NewWithNodeSet(nodes)) triedb.Cap(0) found := make(map[common.Hash][]byte) diff --git a/trie/nodeset.go b/trie/nodeset.go new file mode 100644 index 000000000..08b9b35eb --- /dev/null +++ b/trie/nodeset.go @@ -0,0 +1,94 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package trie + +import ( + "fmt" + + "github.com/ethereum/go-ethereum/common" +) + +// memoryNode is all the information we know about a single cached trie node +// in the memory. +type memoryNode struct { + hash common.Hash // Node hash, computed by hashing rlp value + size uint16 // Byte size of the useful cached data + node node // Cached collapsed trie node, or raw rlp data +} + +// NodeSet contains all dirty nodes collected during the commit operation. +// Each node is keyed by path. It's not thread-safe to use. +type NodeSet struct { + owner common.Hash // the identifier of the trie + paths []string // the path of dirty nodes, sort by insertion order + nodes map[string]*memoryNode // the map of dirty nodes, keyed by node path + leaves []*leaf // the list of dirty leaves +} + +// NewNodeSet initializes an empty node set to be used for tracking dirty nodes +// from a specific account or storage trie. The owner is zero for the account +// trie and the owning account address hash for storage tries. +func NewNodeSet(owner common.Hash) *NodeSet { + return &NodeSet{ + owner: owner, + nodes: make(map[string]*memoryNode), + } +} + +// add caches node with provided path and node object. +func (set *NodeSet) add(path string, node *memoryNode) { + set.paths = append(set.paths, path) + set.nodes[path] = node +} + +// addLeaf caches the provided leaf node. +func (set *NodeSet) addLeaf(node *leaf) { + set.leaves = append(set.leaves, node) +} + +// Len returns the number of dirty nodes contained in the set. +func (set *NodeSet) Len() int { + return len(set.nodes) +} + +// MergedNodeSet represents a merged dirty node set for a group of tries. +type MergedNodeSet struct { + sets map[common.Hash]*NodeSet +} + +// NewMergedNodeSet initializes an empty merged set. +func NewMergedNodeSet() *MergedNodeSet { + return &MergedNodeSet{sets: make(map[common.Hash]*NodeSet)} +} + +// NewWithNodeSet constructs a merged nodeset with the provided single set. +func NewWithNodeSet(set *NodeSet) *MergedNodeSet { + merged := NewMergedNodeSet() + merged.Merge(set) + return merged +} + +// Merge merges the provided dirty nodes of a trie into the set. The assumption +// is held that no duplicated set belonging to the same trie will be merged twice. +func (set *MergedNodeSet) Merge(other *NodeSet) error { + _, present := set.sets[other.owner] + if present { + return fmt.Errorf("duplicate trie for owner %#x", other.owner) + } + set.sets[other.owner] = other + return nil +} diff --git a/trie/proof.go b/trie/proof.go index 9bf910756..fe3662c51 100644 --- a/trie/proof.go +++ b/trie/proof.go @@ -22,6 +22,7 @@ import ( "fmt" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/log" ) @@ -35,9 +36,12 @@ import ( // with the node that proves the absence of the key. func (t *Trie) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) error { // Collect all nodes on the path to key. + var ( + prefix []byte + nodes []node + tn = t.root + ) key = keybytesToHex(key) - var nodes []node - tn := t.root for len(key) > 0 && tn != nil { switch n := tn.(type) { case *shortNode: @@ -46,16 +50,18 @@ func (t *Trie) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) e tn = nil } else { tn = n.Val + prefix = append(prefix, n.Key...) key = key[len(n.Key):] } nodes = append(nodes, n) case *fullNode: tn = n.Children[key[0]] + prefix = append(prefix, key[0]) key = key[1:] nodes = append(nodes, n) case hashNode: var err error - tn, err = t.resolveHash(n, nil) + tn, err = t.resolveHash(n, prefix) if err != nil { log.Error(fmt.Sprintf("Unhandled trie error: %v", err)) return err @@ -553,7 +559,7 @@ func VerifyRangeProof(rootHash common.Hash, firstKey []byte, lastKey []byte, key } // Rebuild the trie with the leaf stream, the shape of trie // should be same with the original one. - tr := newWithRootNode(root) + tr := &Trie{root: root, db: NewDatabase(rawdb.NewMemoryDatabase())} if empty { tr.root = nil } diff --git a/trie/secure_trie.go b/trie/secure_trie.go index 967194df9..59772815e 100644 --- a/trie/secure_trie.go +++ b/trie/secure_trie.go @@ -160,12 +160,14 @@ func (t *SecureTrie) GetKey(shaKey []byte) []byte { return t.preimages.preimage(common.BytesToHash(shaKey)) } -// Commit writes all nodes and the secure hash pre-images to the trie's database. -// Nodes are stored with their sha3 hash as the key. -// -// Committing flushes nodes from memory. Subsequent Get calls will load nodes -// from the database. -func (t *SecureTrie) Commit(onleaf LeafCallback) (common.Hash, int, error) { +// Commit collects all dirty nodes in the trie and replace them with the +// corresponding node hash. All collected nodes(including dirty leaves if +// collectLeaf is true) will be encapsulated into a nodeset for return. +// The returned nodeset can be nil if the trie is clean(nothing to commit). +// All cached preimages will be also flushed if preimages recording is enabled. +// Once the trie is committed, it's not usable anymore. A new trie must +// be created with new root and updated trie database for following usage +func (t *SecureTrie) Commit(collectLeaf bool) (common.Hash, *NodeSet, error) { // Write all the pre-images to the actual disk database if len(t.getSecKeyCache()) > 0 { if t.preimages != nil { @@ -178,7 +180,7 @@ func (t *SecureTrie) Commit(onleaf LeafCallback) (common.Hash, int, error) { t.secKeyCache = make(map[string][]byte) } // Commit the trie to its intermediate node database - return t.trie.Commit(onleaf) + return t.trie.Commit(collectLeaf) } // Hash returns the root hash of SecureTrie. It does not write to the diff --git a/trie/secure_trie_test.go b/trie/secure_trie_test.go index beea5845a..524d1db94 100644 --- a/trie/secure_trie_test.go +++ b/trie/secure_trie_test.go @@ -18,6 +18,7 @@ package trie import ( "bytes" + "fmt" "runtime" "sync" "testing" @@ -57,9 +58,15 @@ func makeTestSecureTrie() (*Database, *SecureTrie, map[string][]byte) { trie.Update(key, val) } } - trie.Commit(nil) - - // Return the generated trie + root, nodes, err := trie.Commit(false) + if err != nil { + panic(fmt.Errorf("failed to commit trie %v", err)) + } + if err := triedb.Update(NewWithNodeSet(nodes)); err != nil { + panic(fmt.Errorf("failed to commit db %v", err)) + } + // Re-create the trie based on the new state + trie, _ = NewSecure(common.Hash{}, root, triedb) return triedb, trie, content } @@ -135,7 +142,7 @@ func TestSecureTrieConcurrency(t *testing.T) { tries[index].Update(key, val) } } - tries[index].Commit(nil) + tries[index].Commit(false) }(i) } // Wait for all threads to finish diff --git a/trie/sync_test.go b/trie/sync_test.go index 472c31a63..c2ddb98c7 100644 --- a/trie/sync_test.go +++ b/trie/sync_test.go @@ -18,6 +18,7 @@ package trie import ( "bytes" + "fmt" "testing" "github.com/ethereum/go-ethereum/common" @@ -50,9 +51,15 @@ func makeTestTrie() (*Database, *SecureTrie, map[string][]byte) { trie.Update(key, val) } } - trie.Commit(nil) - - // Return the generated trie + root, nodes, err := trie.Commit(false) + if err != nil { + panic(fmt.Errorf("failed to commit trie %v", err)) + } + if err := triedb.Update(NewWithNodeSet(nodes)); err != nil { + panic(fmt.Errorf("failed to commit db %v", err)) + } + // Re-create the trie based on the new state + trie, _ = NewSecure(common.Hash{}, root, triedb) return triedb, trie, content } diff --git a/trie/trie.go b/trie/trie.go index 1e168402a..08f2480db 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -21,10 +21,8 @@ import ( "bytes" "errors" "fmt" - "sync" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" @@ -55,23 +53,28 @@ var ( // for extracting the raw states(leaf nodes) with corresponding paths. type LeafCallback func(keys [][]byte, path []byte, leaf []byte, parent common.Hash, parentPath []byte) error -// Trie is a Merkle Patricia Trie. -// The zero value is an empty trie with no database. -// Use New to create a trie that sits on top of a database. +// Trie is a Merkle Patricia Trie. Use New to create a trie that sits on +// top of a database. Whenever trie performs a commit operation, the generated +// nodes will be gathered and returned in a set. Once the trie is committed, +// it's not usable anymore. Callers have to re-create the trie with new root +// based on the updated trie database. // // Trie is not safe for concurrent use. type Trie struct { - db *Database root node owner common.Hash // Keep track of the number leaves which have been inserted since the last // hashing operation. This number will not directly map to the number of - // actually unhashed nodes + // actually unhashed nodes. unhashed int - // tracer is the state diff tracer can be used to track newly added/deleted - // trie node. It will be reset after each commit operation. + // db is the handler trie can retrieve nodes from. It's + // only for reading purpose and not available for writing. + db *Database + + // tracer is the tool to track the trie changes. + // It will be reset after each commit operation. tracer *tracer } @@ -83,10 +86,10 @@ func (t *Trie) newFlag() nodeFlag { // Copy returns a copy of Trie. func (t *Trie) Copy() *Trie { return &Trie{ - db: t.db, root: t.root, owner: t.owner, unhashed: t.unhashed, + db: t.db, tracer: t.tracer.copy(), } } @@ -99,33 +102,9 @@ func (t *Trie) Copy() *Trie { // New will panic if db is nil and returns a MissingNodeError if root does // not exist in the database. Accessing the trie loads nodes from db on demand. func New(owner common.Hash, root common.Hash, db *Database) (*Trie, error) { - return newTrie(owner, root, db) -} - -// NewEmpty is a shortcut to create empty tree. It's mostly used in tests. -func NewEmpty(db *Database) *Trie { - tr, _ := newTrie(common.Hash{}, common.Hash{}, db) - return tr -} - -// newWithRootNode initializes the trie with the given root node. -// It's only used by range prover. -func newWithRootNode(root node) *Trie { - return &Trie{ - root: root, - //tracer: newTracer(), - db: NewDatabase(rawdb.NewMemoryDatabase()), - } -} - -// newTrie is the internal function used to construct the trie with given parameters. -func newTrie(owner common.Hash, root common.Hash, db *Database) (*Trie, error) { - if db == nil { - panic("trie.New called without a database") - } trie := &Trie{ - db: db, owner: owner, + db: db, //tracer: newTracer(), } if root != (common.Hash{}) && root != emptyRoot { @@ -138,6 +117,12 @@ func newTrie(owner common.Hash, root common.Hash, db *Database) (*Trie, error) { return trie, nil } +// NewEmpty is a shortcut to create empty tree. It's mostly used in tests. +func NewEmpty(db *Database) *Trie { + tr, _ := New(common.Hash{}, common.Hash{}, db) + return tr +} + // NodeIterator returns an iterator that returns nodes of the trie. Iteration starts at // the key after the given start key. func (t *Trie) NodeIterator(start []byte) NodeIterator { @@ -512,7 +497,7 @@ func (t *Trie) delete(n node, prefix, key []byte) (bool, node, error) { // shortNode{..., shortNode{...}}. Since the entry // might not be loaded yet, resolve it just for this // check. - cnode, err := t.resolve(n.Children[pos], prefix) + cnode, err := t.resolve(n.Children[pos], append(prefix, byte(pos))) if err != nil { return false, nil, err } @@ -572,6 +557,8 @@ func (t *Trie) resolve(n node, prefix []byte) (node, error) { return n, nil } +// resolveHash loads node from the underlying database with the provided +// node hash and path prefix. func (t *Trie) resolveHash(n hashNode, prefix []byte) (node, error) { hash := common.BytesToHash(n) if node := t.db.node(hash); node != nil { @@ -580,6 +567,8 @@ func (t *Trie) resolveHash(n hashNode, prefix []byte) (node, error) { return nil, &MissingNodeError{Owner: t.owner, NodeHash: hash, Path: prefix} } +// resolveHash loads rlp-encoded node blob from the underlying database +// with the provided node hash and path prefix. func (t *Trie) resolveBlob(n hashNode, prefix []byte) ([]byte, error) { hash := common.BytesToHash(n) blob, _ := t.db.Node(hash) @@ -597,56 +586,37 @@ func (t *Trie) Hash() common.Hash { return common.BytesToHash(hash.(hashNode)) } -// Commit writes all nodes to the trie's memory database, tracking the internal -// and external (for account tries) references. -func (t *Trie) Commit(onleaf LeafCallback) (common.Hash, int, error) { - if t.db == nil { - panic("commit called on trie with nil database") - } +// Commit collects all dirty nodes in the trie and replace them with the +// corresponding node hash. All collected nodes(including dirty leaves if +// collectLeaf is true) will be encapsulated into a nodeset for return. +// The returned nodeset can be nil if the trie is clean(nothing to commit). +// Once the trie is committed, it's not usable anymore. A new trie must +// be created with new root and updated trie database for following usage +func (t *Trie) Commit(collectLeaf bool) (common.Hash, *NodeSet, error) { defer t.tracer.reset() if t.root == nil { - return emptyRoot, 0, nil + return emptyRoot, nil, nil } // Derive the hash for all dirty nodes first. We hold the assumption // in the following procedure that all nodes are hashed. rootHash := t.Hash() - h := newCommitter() - defer returnCommitterToPool(h) - // Do a quick check if we really need to commit, before we spin - // up goroutines. This can happen e.g. if we load a trie for reading storage - // values, but don't write to it. + // Do a quick check if we really need to commit. This can happen e.g. + // if we load a trie for reading storage values, but don't write to it. if hashedNode, dirty := t.root.cache(); !dirty { // Replace the root node with the origin hash in order to // ensure all resolved nodes are dropped after the commit. t.root = hashedNode - return rootHash, 0, nil - } - var wg sync.WaitGroup - if onleaf != nil { - h.onleaf = onleaf - h.leafCh = make(chan *leaf, leafChanSize) - wg.Add(1) - go func() { - defer wg.Done() - h.commitLoop(t.db) - }() - } - newRoot, committed, err := h.Commit(t.root, t.db) - if onleaf != nil { - // The leafch is created in newCommitter if there was an onleaf callback - // provided. The commitLoop only _reads_ from it, and the commit - // operation was the sole writer. Therefore, it's safe to close this - // channel here. - close(h.leafCh) - wg.Wait() + return rootHash, nil, nil } + h := newCommitter(t.owner, collectLeaf) + newRoot, nodes, err := h.Commit(t.root) if err != nil { - return common.Hash{}, 0, err + return common.Hash{}, nil, err } t.root = newRoot - return rootHash, committed, nil + return rootHash, nodes, nil } // hashRoot calculates the root hash of the given trie @@ -667,10 +637,6 @@ func (t *Trie) Reset() { t.root = nil t.owner = common.Hash{} t.unhashed = 0 + //t.db = nil t.tracer.reset() } - -// Owner returns the associated trie owner. -func (t *Trie) Owner() common.Hash { - return t.owner -} diff --git a/trie/trie_test.go b/trie/trie_test.go index 135e94e3d..3e29600bb 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -24,7 +24,6 @@ import ( "hash" "math/big" "math/rand" - "os" "reflect" "testing" "testing/quick" @@ -35,7 +34,6 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/ethdb/leveldb" "github.com/ethereum/go-ethereum/ethdb/memorydb" "github.com/ethereum/go-ethereum/rlp" "golang.org/x/crypto/sha3" @@ -46,12 +44,6 @@ func init() { spew.Config.DisableMethods = false } -// Used for testing -func newEmpty() *Trie { - trie := NewEmpty(NewDatabase(memorydb.New())) - return trie -} - func TestEmptyTrie(t *testing.T) { trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) res := trie.Hash() @@ -91,7 +83,8 @@ func testMissingNode(t *testing.T, memonly bool) { trie := NewEmpty(triedb) updateString(trie, "120000", "qwerqwerqwerqwerqwerqwerqwerqwer") updateString(trie, "123456", "asdfasdfasdfasdfasdfasdfasdfasdf") - root, _, _ := trie.Commit(nil) + root, nodes, _ := trie.Commit(false) + triedb.Update(NewWithNodeSet(nodes)) if !memonly { triedb.Commit(root, true, nil) } @@ -157,7 +150,7 @@ func testMissingNode(t *testing.T, memonly bool) { } func TestInsert(t *testing.T) { - trie := newEmpty() + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) updateString(trie, "doe", "reindeer") updateString(trie, "dog", "puppy") @@ -169,11 +162,11 @@ func TestInsert(t *testing.T) { t.Errorf("case 1: exp %x got %x", exp, root) } - trie = newEmpty() + trie = NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) updateString(trie, "A", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") exp = common.HexToHash("d23786fb4a010da3ce639d66d5e904a11dbc02746d1ce25029e53290cabf28ab") - root, _, err := trie.Commit(nil) + root, _, err := trie.Commit(false) if err != nil { t.Fatalf("commit error: %v", err) } @@ -183,7 +176,8 @@ func TestInsert(t *testing.T) { } func TestGet(t *testing.T) { - trie := newEmpty() + db := NewDatabase(rawdb.NewMemoryDatabase()) + trie := NewEmpty(db) updateString(trie, "doe", "reindeer") updateString(trie, "dog", "puppy") updateString(trie, "dogglesworth", "cat") @@ -193,21 +187,21 @@ func TestGet(t *testing.T) { if !bytes.Equal(res, []byte("puppy")) { t.Errorf("expected puppy got %x", res) } - unknown := getString(trie, "unknown") if unknown != nil { t.Errorf("expected nil got %x", unknown) } - if i == 1 { return } - trie.Commit(nil) + root, nodes, _ := trie.Commit(false) + db.Update(NewWithNodeSet(nodes)) + trie, _ = New(common.Hash{}, root, db) } } func TestDelete(t *testing.T) { - trie := newEmpty() + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) vals := []struct{ k, v string }{ {"do", "verb"}, {"ether", "wookiedoo"}, @@ -234,7 +228,7 @@ func TestDelete(t *testing.T) { } func TestEmptyValues(t *testing.T) { - trie := newEmpty() + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) vals := []struct{ k, v string }{ {"do", "verb"}, @@ -258,7 +252,8 @@ func TestEmptyValues(t *testing.T) { } func TestReplication(t *testing.T) { - trie := newEmpty() + triedb := NewDatabase(rawdb.NewMemoryDatabase()) + trie := NewEmpty(triedb) vals := []struct{ k, v string }{ {"do", "verb"}, {"ether", "wookiedoo"}, @@ -271,13 +266,14 @@ func TestReplication(t *testing.T) { for _, val := range vals { updateString(trie, val.k, val.v) } - exp, _, err := trie.Commit(nil) + exp, nodes, err := trie.Commit(false) if err != nil { t.Fatalf("commit error: %v", err) } + triedb.Update(NewWithNodeSet(nodes)) // create a new trie on top of the database and check that lookups work. - trie2, err := New(common.Hash{}, exp, trie.db) + trie2, err := New(common.Hash{}, exp, triedb) if err != nil { t.Fatalf("can't recreate trie at %x: %v", exp, err) } @@ -286,7 +282,7 @@ func TestReplication(t *testing.T) { t.Errorf("trie2 doesn't have %q => %q", kv.k, kv.v) } } - hash, _, err := trie2.Commit(nil) + hash, nodes, err := trie2.Commit(false) if err != nil { t.Fatalf("commit error: %v", err) } @@ -294,6 +290,14 @@ func TestReplication(t *testing.T) { t.Errorf("root failure. expected %x got %x", exp, hash) } + // recreate the trie after commit + if nodes != nil { + triedb.Update(NewWithNodeSet(nodes)) + } + trie2, err = New(common.Hash{}, hash, triedb) + if err != nil { + t.Fatalf("can't recreate trie at %x: %v", exp, err) + } // perform some insertions on the new trie. vals2 := []struct{ k, v string }{ {"do", "verb"}, @@ -315,7 +319,7 @@ func TestReplication(t *testing.T) { } func TestLargeValue(t *testing.T) { - trie := newEmpty() + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) trie.Update([]byte("key1"), []byte{99, 99, 99, 99}) trie.Update([]byte("key2"), bytes.Repeat([]byte{1}, 32)) trie.Hash() @@ -369,9 +373,8 @@ const ( opUpdate = iota opDelete opGet - opCommit opHash - opReset + opCommit opItercheckhash opNodeDiff opMax // boundary value, not an actual op @@ -433,17 +436,17 @@ func runRandTest(rt randTest) bool { if string(v) != want { rt[i].err = fmt.Errorf("mismatch for key %#x, got %#x want %#x", step.key, v, want) } - case opCommit: - _, _, rt[i].err = tr.Commit(nil) - origTrie = tr.Copy() case opHash: tr.Hash() - case opReset: - hash, _, err := tr.Commit(nil) + case opCommit: + hash, nodes, err := tr.Commit(false) if err != nil { rt[i].err = err return false } + if nodes != nil { + triedb.Update(NewWithNodeSet(nodes)) + } newtr, err := New(common.Hash{}, hash, triedb) if err != nil { rt[i].err = err @@ -533,44 +536,31 @@ func TestRandom(t *testing.T) { } } -func BenchmarkGet(b *testing.B) { benchGet(b, false) } -func BenchmarkGetDB(b *testing.B) { benchGet(b, true) } +func BenchmarkGet(b *testing.B) { benchGet(b) } func BenchmarkUpdateBE(b *testing.B) { benchUpdate(b, binary.BigEndian) } func BenchmarkUpdateLE(b *testing.B) { benchUpdate(b, binary.LittleEndian) } const benchElemCount = 20000 -func benchGet(b *testing.B, commit bool) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) - if commit { - tmpdb := tempDB(b) - trie = NewEmpty(tmpdb) - } +func benchGet(b *testing.B) { + triedb := NewDatabase(rawdb.NewMemoryDatabase()) + trie := NewEmpty(triedb) k := make([]byte, 32) for i := 0; i < benchElemCount; i++ { binary.LittleEndian.PutUint64(k, uint64(i)) trie.Update(k, k) } binary.LittleEndian.PutUint64(k, benchElemCount/2) - if commit { - trie.Commit(nil) - } b.ResetTimer() for i := 0; i < b.N; i++ { trie.Get(k) } b.StopTimer() - - if commit { - ldb := trie.db.diskdb.(*leveldb.Database) - ldb.Close() - os.RemoveAll(ldb.Path()) - } } func benchUpdate(b *testing.B, e binary.ByteOrder) *Trie { - trie := newEmpty() + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) k := make([]byte, 32) b.ReportAllocs() for i := 0; i < b.N; i++ { @@ -600,7 +590,7 @@ func BenchmarkHash(b *testing.B) { // entries, then adding N more. addresses, accounts := makeAccounts(2 * b.N) // Insert the accounts into the trie and hash it - trie := newEmpty() + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) i := 0 for ; i < len(addresses)/2; i++ { trie.Update(crypto.Keccak256(addresses[i][:]), accounts[i]) @@ -621,22 +611,17 @@ func BenchmarkHash(b *testing.B) { // insert into the trie before measuring the hashing. func BenchmarkCommitAfterHash(b *testing.B) { b.Run("no-onleaf", func(b *testing.B) { - benchmarkCommitAfterHash(b, nil) + benchmarkCommitAfterHash(b, false) }) - var a types.StateAccount - onleaf := func(paths [][]byte, hexpath []byte, leaf []byte, parent common.Hash, parentPath []byte) error { - rlp.DecodeBytes(leaf, &a) - return nil - } b.Run("with-onleaf", func(b *testing.B) { - benchmarkCommitAfterHash(b, onleaf) + benchmarkCommitAfterHash(b, true) }) } -func benchmarkCommitAfterHash(b *testing.B, onleaf LeafCallback) { +func benchmarkCommitAfterHash(b *testing.B, collectLeaf bool) { // Make the random benchmark deterministic addresses, accounts := makeAccounts(b.N) - trie := newEmpty() + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) for i := 0; i < len(addresses); i++ { trie.Update(crypto.Keccak256(addresses[i][:]), accounts[i]) } @@ -644,13 +629,13 @@ func benchmarkCommitAfterHash(b *testing.B, onleaf LeafCallback) { trie.Hash() b.ResetTimer() b.ReportAllocs() - trie.Commit(onleaf) + trie.Commit(collectLeaf) } func TestTinyTrie(t *testing.T) { // Create a realistic account trie to hash _, accounts := makeAccounts(5) - trie := newEmpty() + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) trie.Update(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000001337"), accounts[3]) if exp, root := common.HexToHash("8c6a85a4d9fda98feff88450299e574e5378e32391f75a055d470ac0653f1005"), trie.Hash(); exp != root { t.Errorf("1: got %x, exp %x", root, exp) @@ -663,7 +648,7 @@ func TestTinyTrie(t *testing.T) { if exp, root := common.HexToHash("0608c1d1dc3905fa22204c7a0e43644831c3b6d3def0f274be623a948197e64a"), trie.Hash(); exp != root { t.Errorf("3: got %x, exp %x", root, exp) } - checktr := NewEmpty(trie.db) + checktr := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) it := NewIterator(trie.NodeIterator(nil)) for it.Next() { checktr.Update(it.Key, it.Value) @@ -676,19 +661,19 @@ func TestTinyTrie(t *testing.T) { func TestCommitAfterHash(t *testing.T) { // Create a realistic account trie to hash addresses, accounts := makeAccounts(1000) - trie := newEmpty() + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) for i := 0; i < len(addresses); i++ { trie.Update(crypto.Keccak256(addresses[i][:]), accounts[i]) } // Insert the accounts into the trie and hash it trie.Hash() - trie.Commit(nil) + trie.Commit(false) root := trie.Hash() exp := common.HexToHash("72f9d3f3fe1e1dd7b8936442e7642aef76371472d94319900790053c493f3fe6") if exp != root { t.Errorf("got %x, exp %x", root, exp) } - root, _, _ = trie.Commit(nil) + root, _, _ = trie.Commit(false) if exp != root { t.Errorf("got %x, exp %x", root, exp) } @@ -797,7 +782,8 @@ func TestCommitSequence(t *testing.T) { trie.Update(crypto.Keccak256(addresses[i][:]), accounts[i]) } // Flush trie -> database - root, _, _ := trie.Commit(nil) + root, nodes, _ := trie.Commit(false) + db.Update(NewWithNodeSet(nodes)) // Flush memdb -> disk (sponge) db.Commit(root, false, func(c common.Hash) { // And spongify the callback-order @@ -849,7 +835,8 @@ func TestCommitSequenceRandomBlobs(t *testing.T) { trie.Update(key, val) } // Flush trie -> database - root, _, _ := trie.Commit(nil) + root, nodes, _ := trie.Commit(false) + db.Update(NewWithNodeSet(nodes)) // Flush memdb -> disk (sponge) db.Commit(root, false, func(c common.Hash) { // And spongify the callback-order @@ -875,7 +862,7 @@ func TestCommitSequenceStackTrie(t *testing.T) { stackTrieSponge := &spongeDb{sponge: sha3.NewLegacyKeccak256(), id: "b"} stTrie := NewStackTrie(stackTrieSponge) // Fill the trie with elements - for i := 1; i < count; i++ { + for i := 0; i < count; i++ { // For the stack trie, we need to do inserts in proper order key := make([]byte, 32) binary.BigEndian.PutUint64(key, uint64(i)) @@ -891,8 +878,9 @@ func TestCommitSequenceStackTrie(t *testing.T) { stTrie.TryUpdate(key, val) } // Flush trie -> database - root, _, _ := trie.Commit(nil) + root, nodes, _ := trie.Commit(false) // Flush memdb -> disk (sponge) + db.Update(NewWithNodeSet(nodes)) db.Commit(root, false, nil) // And flush stacktrie -> disk stRoot, err := stTrie.Commit() @@ -936,8 +924,9 @@ func TestCommitSequenceSmallRoot(t *testing.T) { trie.TryUpdate(key, []byte{0x1}) stTrie.TryUpdate(key, []byte{0x1}) // Flush trie -> database - root, _, _ := trie.Commit(nil) + root, nodes, _ := trie.Commit(false) // Flush memdb -> disk (sponge) + db.Update(NewWithNodeSet(nodes)) db.Commit(root, false, nil) // And flush stacktrie -> disk stRoot, err := stTrie.Commit() @@ -999,7 +988,7 @@ func BenchmarkHashFixedSize(b *testing.B) { func benchmarkHashFixedSize(b *testing.B, addresses [][20]byte, accounts [][]byte) { b.ReportAllocs() - trie := newEmpty() + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) for i := 0; i < len(addresses); i++ { trie.Update(crypto.Keccak256(addresses[i][:]), accounts[i]) } @@ -1050,14 +1039,14 @@ func BenchmarkCommitAfterHashFixedSize(b *testing.B) { func benchmarkCommitAfterHashFixedSize(b *testing.B, addresses [][20]byte, accounts [][]byte) { b.ReportAllocs() - trie := newEmpty() + trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) for i := 0; i < len(addresses); i++ { trie.Update(crypto.Keccak256(addresses[i][:]), accounts[i]) } // Insert the accounts into the trie and hash it trie.Hash() b.StartTimer() - trie.Commit(nil) + trie.Commit(false) b.StopTimer() } @@ -1102,26 +1091,19 @@ func BenchmarkDerefRootFixedSize(b *testing.B) { func benchmarkDerefRootFixedSize(b *testing.B, addresses [][20]byte, accounts [][]byte) { b.ReportAllocs() - trie := newEmpty() + triedb := NewDatabase(rawdb.NewMemoryDatabase()) + trie := NewEmpty(triedb) for i := 0; i < len(addresses); i++ { trie.Update(crypto.Keccak256(addresses[i][:]), accounts[i]) } h := trie.Hash() - trie.Commit(nil) + _, nodes, _ := trie.Commit(false) + triedb.Update(NewWithNodeSet(nodes)) b.StartTimer() - trie.db.Dereference(h) + triedb.Dereference(h) b.StopTimer() } -func tempDB(tb testing.TB) *Database { - dir := tb.TempDir() - diskdb, err := leveldb.New(dir, 256, 0, "", false) - if err != nil { - panic(fmt.Sprintf("can't create temporary database: %v", err)) - } - return NewDatabase(diskdb) -} - func getString(trie *Trie, k string) []byte { return trie.Get([]byte(k)) } diff --git a/trie/util_test.go b/trie/util_test.go index 589eca624..252dc09e0 100644 --- a/trie/util_test.go +++ b/trie/util_test.go @@ -19,12 +19,14 @@ package trie import ( "testing" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" ) // Tests if the trie diffs are tracked correctly. func TestTrieTracer(t *testing.T) { - trie := NewEmpty(NewDatabase(rawdb.NewMemoryDatabase())) + db := NewDatabase(rawdb.NewMemoryDatabase()) + trie := NewEmpty(db) trie.tracer = newTracer() // Insert a batch of entries, all the nodes should be marked as inserted @@ -65,8 +67,11 @@ func TestTrieTracer(t *testing.T) { t.Fatalf("Unexpected deleted node tracked %d", len(deleted)) } - // Commit the changes - trie.Commit(nil) + // Commit the changes and re-create with new root + root, nodes, _ := trie.Commit(false) + db.Update(NewWithNodeSet(nodes)) + trie, _ = New(common.Hash{}, root, db) + trie.tracer = newTracer() // Delete all the elements, check deletion set for _, val := range vals { From f67e54c92fb7edcc44fa3739dfbc5d59790cc6f8 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Thu, 4 Aug 2022 16:13:18 +0200 Subject: [PATCH 25/59] core: use TryGetAccount to read what TryUpdateAccount has written (#25458) * core: use TryGetAccount to read where TryUpdateAccount has been used to write * Gary's review feedback * implement Gary's suggestion * fix bug + rename NewSecure into NewStateTrie * trie: add backwards-compatibility aliases for SecureTrie * Update database.go * make the linter happy Co-authored-by: Felix Lange Co-authored-by: rjl493456442 --- cmd/geth/snapshot.go | 8 +-- core/blockchain.go | 2 +- core/state/database.go | 15 +++-- core/state/pruner/pruner.go | 4 +- core/state/snapshot/generate_test.go | 6 +- core/state/statedb.go | 12 ++-- core/state/trie_prefetcher.go | 6 +- eth/api.go | 4 +- eth/protocols/snap/handler.go | 14 ++--- eth/protocols/snap/sync_test.go | 2 +- les/downloader/downloader_test.go | 2 +- light/trie.go | 16 +++++ trie/iterator_test.go | 4 +- trie/proof.go | 2 +- trie/secure_trie.go | 93 ++++++++++++++++++++-------- trie/secure_trie_test.go | 16 ++--- trie/sync_test.go | 8 +-- trie/trie.go | 16 ++--- 18 files changed, 142 insertions(+), 88 deletions(-) diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index a218ae9cd..39bef1f2d 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -271,7 +271,7 @@ func traverseState(ctx *cli.Context) error { log.Info("Start traversing the state", "root", root, "number", headBlock.NumberU64()) } triedb := trie.NewDatabase(chaindb) - t, err := trie.NewSecure(common.Hash{}, root, triedb) + t, err := trie.NewStateTrie(common.Hash{}, root, triedb) if err != nil { log.Error("Failed to open trie", "root", root, "err", err) return err @@ -292,7 +292,7 @@ func traverseState(ctx *cli.Context) error { return err } if acc.Root != emptyRoot { - storageTrie, err := trie.NewSecure(common.BytesToHash(accIter.Key), acc.Root, triedb) + storageTrie, err := trie.NewStateTrie(common.BytesToHash(accIter.Key), acc.Root, triedb) if err != nil { log.Error("Failed to open storage trie", "root", acc.Root, "err", err) return err @@ -360,7 +360,7 @@ func traverseRawState(ctx *cli.Context) error { log.Info("Start traversing the state", "root", root, "number", headBlock.NumberU64()) } triedb := trie.NewDatabase(chaindb) - t, err := trie.NewSecure(common.Hash{}, root, triedb) + t, err := trie.NewStateTrie(common.Hash{}, root, triedb) if err != nil { log.Error("Failed to open trie", "root", root, "err", err) return err @@ -406,7 +406,7 @@ func traverseRawState(ctx *cli.Context) error { return errors.New("invalid account") } if acc.Root != emptyRoot { - storageTrie, err := trie.NewSecure(common.BytesToHash(accIter.LeafKey()), acc.Root, triedb) + storageTrie, err := trie.NewStateTrie(common.BytesToHash(accIter.LeafKey()), acc.Root, triedb) if err != nil { log.Error("Failed to open storage trie", "root", acc.Root, "err", err) return errors.New("missing storage trie") diff --git a/core/blockchain.go b/core/blockchain.go index 3638a1dce..f35de4046 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -706,7 +706,7 @@ func (bc *BlockChain) SnapSyncCommitHead(hash common.Hash) error { if block == nil { return fmt.Errorf("non existent block [%x..]", hash[:4]) } - if _, err := trie.NewSecure(common.Hash{}, block.Root(), bc.stateCache.TrieDB()); err != nil { + if _, err := trie.NewStateTrie(common.Hash{}, block.Root(), bc.stateCache.TrieDB()); err != nil { return err } diff --git a/core/state/database.go b/core/state/database.go index 8f662ecd3..edbf78ae3 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -63,7 +63,7 @@ type Trie interface { // GetKey returns the sha3 preimage of a hashed key that was previously used // to store a value. // - // TODO(fjl): remove this when SecureTrie is removed + // TODO(fjl): remove this when StateTrie is removed GetKey([]byte) []byte // TryGet returns the value for key stored in the trie. The value bytes must @@ -71,8 +71,8 @@ type Trie interface { // trie.MissingNodeError is returned. TryGet(key []byte) ([]byte, error) - // TryUpdateAccount abstract an account write in the trie. - TryUpdateAccount(key []byte, account *types.StateAccount) error + // TryGetAccount abstract an account read from the trie. + TryGetAccount(key []byte) (*types.StateAccount, error) // TryUpdate associates key with value in the trie. If value has length zero, any // existing value is deleted from the trie. The value bytes must not be modified @@ -80,6 +80,9 @@ type Trie interface { // database, a trie.MissingNodeError is returned. TryUpdate(key, value []byte) error + // TryUpdateAccount abstract an account write to the trie. + TryUpdateAccount(key []byte, account *types.StateAccount) error + // TryDelete removes any existing value for key from the trie. If a node was not // found in the database, a trie.MissingNodeError is returned. TryDelete(key []byte) error @@ -137,7 +140,7 @@ type cachingDB struct { // OpenTrie opens the main account trie at a specific root hash. func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) { - tr, err := trie.NewSecure(common.Hash{}, root, db.db) + tr, err := trie.NewStateTrie(common.Hash{}, root, db.db) if err != nil { return nil, err } @@ -146,7 +149,7 @@ func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) { // OpenStorageTrie opens the storage trie of an account. func (db *cachingDB) OpenStorageTrie(addrHash, root common.Hash) (Trie, error) { - tr, err := trie.NewSecure(addrHash, root, db.db) + tr, err := trie.NewStateTrie(addrHash, root, db.db) if err != nil { return nil, err } @@ -156,7 +159,7 @@ func (db *cachingDB) OpenStorageTrie(addrHash, root common.Hash) (Trie, error) { // CopyTrie returns an independent copy of the given trie. func (db *cachingDB) CopyTrie(t Trie) Trie { switch t := t.(type) { - case *trie.SecureTrie: + case *trie.StateTrie: return t.Copy() default: panic(fmt.Errorf("unknown trie type %T", t)) diff --git a/core/state/pruner/pruner.go b/core/state/pruner/pruner.go index 2f4b068d8..2da2eda8b 100644 --- a/core/state/pruner/pruner.go +++ b/core/state/pruner/pruner.go @@ -410,7 +410,7 @@ func extractGenesis(db ethdb.Database, stateBloom *stateBloom) error { if genesis == nil { return errors.New("missing genesis block") } - t, err := trie.NewSecure(common.Hash{}, genesis.Root(), trie.NewDatabase(db)) + t, err := trie.NewStateTrie(common.Hash{}, genesis.Root(), trie.NewDatabase(db)) if err != nil { return err } @@ -430,7 +430,7 @@ func extractGenesis(db ethdb.Database, stateBloom *stateBloom) error { return err } if acc.Root != emptyRoot { - storageTrie, err := trie.NewSecure(common.BytesToHash(accIter.LeafKey()), acc.Root, trie.NewDatabase(db)) + storageTrie, err := trie.NewStateTrie(common.BytesToHash(accIter.LeafKey()), acc.Root, trie.NewDatabase(db)) if err != nil { return err } diff --git a/core/state/snapshot/generate_test.go b/core/state/snapshot/generate_test.go index 8d89ca59a..5e5ded61e 100644 --- a/core/state/snapshot/generate_test.go +++ b/core/state/snapshot/generate_test.go @@ -142,14 +142,14 @@ func checkSnapRoot(t *testing.T, snap *diskLayer, trieRoot common.Hash) { type testHelper struct { diskdb ethdb.Database triedb *trie.Database - accTrie *trie.SecureTrie + accTrie *trie.StateTrie nodes *trie.MergedNodeSet } func newHelper() *testHelper { diskdb := rawdb.NewMemoryDatabase() triedb := trie.NewDatabase(diskdb) - accTrie, _ := trie.NewSecure(common.Hash{}, common.Hash{}, triedb) + accTrie, _ := trie.NewStateTrie(common.Hash{}, common.Hash{}, triedb) return &testHelper{ diskdb: diskdb, triedb: triedb, @@ -182,7 +182,7 @@ func (t *testHelper) addSnapStorage(accKey string, keys []string, vals []string) } func (t *testHelper) makeStorageTrie(stateRoot, owner common.Hash, keys []string, vals []string, commit bool) []byte { - stTrie, _ := trie.NewSecure(owner, common.Hash{}, t.triedb) + stTrie, _ := trie.NewStateTrie(owner, common.Hash{}, t.triedb) for i, k := range keys { stTrie.Update([]byte(k), []byte(vals[i])) } diff --git a/core/state/statedb.go b/core/state/statedb.go index 322bc540b..cd388d6a3 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -537,20 +537,16 @@ func (s *StateDB) getDeletedStateObject(addr common.Address) *stateObject { // If snapshot unavailable or reading from it failed, load from the database if data == nil { start := time.Now() - enc, err := s.trie.TryGet(addr.Bytes()) + var err error + data, err = s.trie.TryGetAccount(addr.Bytes()) if metrics.EnabledExpensive { s.AccountReads += time.Since(start) } if err != nil { - s.setError(fmt.Errorf("getDeleteStateObject (%x) error: %v", addr.Bytes(), err)) + s.setError(fmt.Errorf("getDeleteStateObject (%x) error: %w", addr.Bytes(), err)) return nil } - if len(enc) == 0 { - return nil - } - data = new(types.StateAccount) - if err := rlp.DecodeBytes(enc, data); err != nil { - log.Error("Failed to decode state object", "addr", addr, "err", err) + if data == nil { return nil } } diff --git a/core/state/trie_prefetcher.go b/core/state/trie_prefetcher.go index 4c817b1bc..0f6bce3b8 100644 --- a/core/state/trie_prefetcher.go +++ b/core/state/trie_prefetcher.go @@ -331,7 +331,11 @@ func (sf *subfetcher) loop() { if _, ok := sf.seen[string(task)]; ok { sf.dups++ } else { - sf.trie.TryGet(task) + if len(task) == len(common.Address{}) { + sf.trie.TryGetAccount(task) + } else { + sf.trie.TryGet(task) + } sf.seen[string(task)] = struct{}{} } } diff --git a/eth/api.go b/eth/api.go index 5d9a3cc3a..3ce8a2d1b 100644 --- a/eth/api.go +++ b/eth/api.go @@ -506,11 +506,11 @@ func (api *DebugAPI) getModifiedAccounts(startBlock, endBlock *types.Block) ([]c } triedb := api.eth.BlockChain().StateCache().TrieDB() - oldTrie, err := trie.NewSecure(common.Hash{}, startBlock.Root(), triedb) + oldTrie, err := trie.NewStateTrie(common.Hash{}, startBlock.Root(), triedb) if err != nil { return nil, err } - newTrie, err := trie.NewSecure(common.Hash{}, endBlock.Root(), triedb) + newTrie, err := trie.NewStateTrie(common.Hash{}, endBlock.Root(), triedb) if err != nil { return nil, err } diff --git a/eth/protocols/snap/handler.go b/eth/protocols/snap/handler.go index 7ecf041e9..77bd96f46 100644 --- a/eth/protocols/snap/handler.go +++ b/eth/protocols/snap/handler.go @@ -23,14 +23,12 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/light" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/enr" - "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" ) @@ -415,15 +413,15 @@ func ServiceGetStorageRangesQuery(chain *core.BlockChain, req *GetStorageRangesP if origin != (common.Hash{}) || (abort && len(storage) > 0) { // Request started at a non-zero hash or was capped prematurely, add // the endpoint Merkle proofs - accTrie, err := trie.New(common.Hash{}, req.Root, chain.StateCache().TrieDB()) + accTrie, err := trie.NewStateTrie(common.Hash{}, req.Root, chain.StateCache().TrieDB()) if err != nil { return nil, nil } - var acc types.StateAccount - if err := rlp.DecodeBytes(accTrie.Get(account[:]), &acc); err != nil { + acc, err := accTrie.TryGetAccountWithPreHashedKey(account[:]) + if err != nil || acc == nil { return nil, nil } - stTrie, err := trie.New(account, acc.Root, chain.StateCache().TrieDB()) + stTrie, err := trie.NewStateTrie(account, acc.Root, chain.StateCache().TrieDB()) if err != nil { return nil, nil } @@ -489,7 +487,7 @@ func ServiceGetTrieNodesQuery(chain *core.BlockChain, req *GetTrieNodesPacket, s // Make sure we have the state associated with the request triedb := chain.StateCache().TrieDB() - accTrie, err := trie.NewSecure(common.Hash{}, req.Root, triedb) + accTrie, err := trie.NewStateTrie(common.Hash{}, req.Root, triedb) if err != nil { // We don't have the requested state available, bail out return nil, nil @@ -531,7 +529,7 @@ func ServiceGetTrieNodesQuery(chain *core.BlockChain, req *GetTrieNodesPacket, s if err != nil || account == nil { break } - stTrie, err := trie.NewSecure(common.BytesToHash(pathset[0]), common.BytesToHash(account.Root), triedb) + stTrie, err := trie.NewStateTrie(common.BytesToHash(pathset[0]), common.BytesToHash(account.Root), triedb) loads++ // always account database reads, even for failures if err != nil { break diff --git a/eth/protocols/snap/sync_test.go b/eth/protocols/snap/sync_test.go index a13e8d308..4d9f631b5 100644 --- a/eth/protocols/snap/sync_test.go +++ b/eth/protocols/snap/sync_test.go @@ -1658,7 +1658,7 @@ func verifyTrie(db ethdb.KeyValueStore, root common.Hash, t *testing.T) { } accounts++ if acc.Root != emptyRoot { - storeTrie, err := trie.NewSecure(common.BytesToHash(accIt.Key), acc.Root, triedb) + storeTrie, err := trie.NewStateTrie(common.BytesToHash(accIt.Key), acc.Root, triedb) if err != nil { t.Fatal(err) } diff --git a/les/downloader/downloader_test.go b/les/downloader/downloader_test.go index 792fc2846..5eea49877 100644 --- a/les/downloader/downloader_test.go +++ b/les/downloader/downloader_test.go @@ -229,7 +229,7 @@ func (dl *downloadTester) CurrentFastBlock() *types.Block { func (dl *downloadTester) FastSyncCommitHead(hash common.Hash) error { // For now only check that the state trie is correct if block := dl.GetBlockByHash(hash); block != nil { - _, err := trie.NewSecure(common.Hash{}, block.Root(), trie.NewDatabase(dl.stateDb)) + _, err := trie.NewStateTrie(common.Hash{}, block.Root(), trie.NewDatabase(dl.stateDb)) return err } return fmt.Errorf("non existent block: %x", hash[:4]) diff --git a/light/trie.go b/light/trie.go index a2ef8ebff..5755e2cc1 100644 --- a/light/trie.go +++ b/light/trie.go @@ -112,6 +112,22 @@ func (t *odrTrie) TryGet(key []byte) ([]byte, error) { return res, err } +func (t *odrTrie) TryGetAccount(key []byte) (*types.StateAccount, error) { + key = crypto.Keccak256(key) + var res types.StateAccount + err := t.do(key, func() (err error) { + value, err := t.trie.TryGet(key) + if err != nil { + return err + } + if value == nil { + return nil + } + return rlp.DecodeBytes(value, &res) + }) + return &res, err +} + func (t *odrTrie) TryUpdateAccount(key []byte, acc *types.StateAccount) error { key = crypto.Keccak256(key) value, err := rlp.EncodeToBytes(acc) diff --git a/trie/iterator_test.go b/trie/iterator_test.go index 0a4c04c8b..e9d822a9a 100644 --- a/trie/iterator_test.go +++ b/trie/iterator_test.go @@ -529,11 +529,11 @@ func (l *loggingDb) Close() error { } // makeLargeTestTrie create a sample test trie -func makeLargeTestTrie() (*Database, *SecureTrie, *loggingDb) { +func makeLargeTestTrie() (*Database, *StateTrie, *loggingDb) { // Create an empty trie logDb := &loggingDb{0, memorydb.New()} triedb := NewDatabase(logDb) - trie, _ := NewSecure(common.Hash{}, common.Hash{}, triedb) + trie, _ := NewStateTrie(common.Hash{}, common.Hash{}, triedb) // Fill it with some arbitrary data for i := 0; i < 10000; i++ { diff --git a/trie/proof.go b/trie/proof.go index fe3662c51..fa8361eef 100644 --- a/trie/proof.go +++ b/trie/proof.go @@ -100,7 +100,7 @@ func (t *Trie) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) e // If the trie does not contain a value for key, the returned proof contains all // nodes of the longest existing prefix of the key (at least the root node), ending // with the node that proves the absence of the key. -func (t *SecureTrie) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) error { +func (t *StateTrie) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) error { return t.trie.Prove(key, fromLevel, proofDb) } diff --git a/trie/secure_trie.go b/trie/secure_trie.go index 59772815e..28b3473c0 100644 --- a/trie/secure_trie.go +++ b/trie/secure_trie.go @@ -25,25 +25,35 @@ import ( "github.com/ethereum/go-ethereum/rlp" ) -// SecureTrie wraps a trie with key hashing. In a secure trie, all +// SecureTrie is the old name of StateTrie. +// Deprecated: use StateTrie. +type SecureTrie = StateTrie + +// NewSecure creates a new StateTrie. +// Deprecated: use NewStateTrie. +func NewSecure(owner common.Hash, root common.Hash, db *Database) (*SecureTrie, error) { + return NewStateTrie(owner, root, db) +} + +// StateTrie wraps a trie with key hashing. In a secure trie, all // access operations hash the key using keccak256. This prevents // calling code from creating long chains of nodes that // increase the access time. // -// Contrary to a regular trie, a SecureTrie can only be created with +// Contrary to a regular trie, a StateTrie can only be created with // New and must have an attached database. The database also stores // the preimage of each key. // -// SecureTrie is not safe for concurrent use. -type SecureTrie struct { +// StateTrie is not safe for concurrent use. +type StateTrie struct { trie Trie preimages *preimageStore hashKeyBuf [common.HashLength]byte secKeyCache map[string][]byte - secKeyCacheOwner *SecureTrie // Pointer to self, replace the key cache on mismatch + secKeyCacheOwner *StateTrie // Pointer to self, replace the key cache on mismatch } -// NewSecure creates a trie with an existing root node from a backing database +// NewStateTrie creates a trie with an existing root node from a backing database // and optional intermediate in-memory node pool. // // If root is the zero hash or the sha3 hash of an empty string, the @@ -54,7 +64,7 @@ type SecureTrie struct { // Loaded nodes are kept around until their 'cache generation' expires. // A new cache generation is created by each call to Commit. // cachelimit sets the number of past cache generations to keep. -func NewSecure(owner common.Hash, root common.Hash, db *Database) (*SecureTrie, error) { +func NewStateTrie(owner common.Hash, root common.Hash, db *Database) (*StateTrie, error) { if db == nil { panic("trie.NewSecure called without a database") } @@ -62,12 +72,12 @@ func NewSecure(owner common.Hash, root common.Hash, db *Database) (*SecureTrie, if err != nil { return nil, err } - return &SecureTrie{trie: *trie, preimages: db.preimages}, nil + return &StateTrie{trie: *trie, preimages: db.preimages}, nil } // Get returns the value for key stored in the trie. // The value bytes must not be modified by the caller. -func (t *SecureTrie) Get(key []byte) []byte { +func (t *StateTrie) Get(key []byte) []byte { res, err := t.TryGet(key) if err != nil { log.Error(fmt.Sprintf("Unhandled trie error: %v", err)) @@ -78,19 +88,50 @@ func (t *SecureTrie) Get(key []byte) []byte { // TryGet returns the value for key stored in the trie. // The value bytes must not be modified by the caller. // If a node was not found in the database, a MissingNodeError is returned. -func (t *SecureTrie) TryGet(key []byte) ([]byte, error) { +func (t *StateTrie) TryGet(key []byte) ([]byte, error) { return t.trie.TryGet(t.hashKey(key)) } +func (t *StateTrie) TryGetAccount(key []byte) (*types.StateAccount, error) { + var ret types.StateAccount + res, err := t.trie.TryGet(t.hashKey(key)) + if err != nil { + log.Error(fmt.Sprintf("Unhandled trie error: %v", err)) + return &ret, err + } + if res == nil { + return nil, nil + } + err = rlp.DecodeBytes(res, &ret) + return &ret, err +} + +// TryGetAccountWithPreHashedKey does the same thing as TryGetAccount, however +// it expects a key that is already hashed. This constitutes an abstraction leak, +// since the client code needs to know the key format. +func (t *StateTrie) TryGetAccountWithPreHashedKey(key []byte) (*types.StateAccount, error) { + var ret types.StateAccount + res, err := t.trie.TryGet(key) + if err != nil { + log.Error(fmt.Sprintf("Unhandled trie error: %v", err)) + return &ret, err + } + if res == nil { + return nil, nil + } + err = rlp.DecodeBytes(res, &ret) + return &ret, err +} + // TryGetNode attempts to retrieve a trie node by compact-encoded path. It is not // possible to use keybyte-encoding as the path might contain odd nibbles. -func (t *SecureTrie) TryGetNode(path []byte) ([]byte, int, error) { +func (t *StateTrie) TryGetNode(path []byte) ([]byte, int, error) { return t.trie.TryGetNode(path) } // TryUpdateAccount account will abstract the write of an account to the // secure trie. -func (t *SecureTrie) TryUpdateAccount(key []byte, acc *types.StateAccount) error { +func (t *StateTrie) TryUpdateAccount(key []byte, acc *types.StateAccount) error { hk := t.hashKey(key) data, err := rlp.EncodeToBytes(acc) if err != nil { @@ -109,7 +150,7 @@ func (t *SecureTrie) TryUpdateAccount(key []byte, acc *types.StateAccount) error // // The value bytes must not be modified by the caller while they are // stored in the trie. -func (t *SecureTrie) Update(key, value []byte) { +func (t *StateTrie) Update(key, value []byte) { if err := t.TryUpdate(key, value); err != nil { log.Error(fmt.Sprintf("Unhandled trie error: %v", err)) } @@ -123,7 +164,7 @@ func (t *SecureTrie) Update(key, value []byte) { // stored in the trie. // // If a node was not found in the database, a MissingNodeError is returned. -func (t *SecureTrie) TryUpdate(key, value []byte) error { +func (t *StateTrie) TryUpdate(key, value []byte) error { hk := t.hashKey(key) err := t.trie.TryUpdate(hk, value) if err != nil { @@ -134,7 +175,7 @@ func (t *SecureTrie) TryUpdate(key, value []byte) error { } // Delete removes any existing value for key from the trie. -func (t *SecureTrie) Delete(key []byte) { +func (t *StateTrie) Delete(key []byte) { if err := t.TryDelete(key); err != nil { log.Error(fmt.Sprintf("Unhandled trie error: %v", err)) } @@ -142,7 +183,7 @@ func (t *SecureTrie) Delete(key []byte) { // TryDelete removes any existing value for key from the trie. // If a node was not found in the database, a MissingNodeError is returned. -func (t *SecureTrie) TryDelete(key []byte) error { +func (t *StateTrie) TryDelete(key []byte) error { hk := t.hashKey(key) delete(t.getSecKeyCache(), string(hk)) return t.trie.TryDelete(hk) @@ -150,7 +191,7 @@ func (t *SecureTrie) TryDelete(key []byte) error { // GetKey returns the sha3 preimage of a hashed key that was // previously used to store a value. -func (t *SecureTrie) GetKey(shaKey []byte) []byte { +func (t *StateTrie) GetKey(shaKey []byte) []byte { if key, ok := t.getSecKeyCache()[string(shaKey)]; ok { return key } @@ -167,7 +208,7 @@ func (t *SecureTrie) GetKey(shaKey []byte) []byte { // All cached preimages will be also flushed if preimages recording is enabled. // Once the trie is committed, it's not usable anymore. A new trie must // be created with new root and updated trie database for following usage -func (t *SecureTrie) Commit(collectLeaf bool) (common.Hash, *NodeSet, error) { +func (t *StateTrie) Commit(collectLeaf bool) (common.Hash, *NodeSet, error) { // Write all the pre-images to the actual disk database if len(t.getSecKeyCache()) > 0 { if t.preimages != nil { @@ -183,15 +224,15 @@ func (t *SecureTrie) Commit(collectLeaf bool) (common.Hash, *NodeSet, error) { return t.trie.Commit(collectLeaf) } -// Hash returns the root hash of SecureTrie. It does not write to the +// Hash returns the root hash of StateTrie. It does not write to the // database and can be used even if the trie doesn't have one. -func (t *SecureTrie) Hash() common.Hash { +func (t *StateTrie) Hash() common.Hash { return t.trie.Hash() } -// Copy returns a copy of SecureTrie. -func (t *SecureTrie) Copy() *SecureTrie { - return &SecureTrie{ +// Copy returns a copy of StateTrie. +func (t *StateTrie) Copy() *StateTrie { + return &StateTrie{ trie: *t.trie.Copy(), preimages: t.preimages, secKeyCache: t.secKeyCache, @@ -200,14 +241,14 @@ func (t *SecureTrie) Copy() *SecureTrie { // NodeIterator returns an iterator that returns nodes of the underlying trie. Iteration // starts at the key after the given start key. -func (t *SecureTrie) NodeIterator(start []byte) NodeIterator { +func (t *StateTrie) NodeIterator(start []byte) NodeIterator { return t.trie.NodeIterator(start) } // hashKey returns the hash of key as an ephemeral buffer. // The caller must not hold onto the return value because it will become // invalid on the next call to hashKey or secKey. -func (t *SecureTrie) hashKey(key []byte) []byte { +func (t *StateTrie) hashKey(key []byte) []byte { h := newHasher(false) h.sha.Reset() h.sha.Write(key) @@ -219,7 +260,7 @@ func (t *SecureTrie) hashKey(key []byte) []byte { // getSecKeyCache returns the current secure key cache, creating a new one if // ownership changed (i.e. the current secure trie is a copy of another owning // the actual cache). -func (t *SecureTrie) getSecKeyCache() map[string][]byte { +func (t *StateTrie) getSecKeyCache() map[string][]byte { if t != t.secKeyCacheOwner { t.secKeyCacheOwner = t t.secKeyCache = make(map[string][]byte) diff --git a/trie/secure_trie_test.go b/trie/secure_trie_test.go index 524d1db94..05bddb21b 100644 --- a/trie/secure_trie_test.go +++ b/trie/secure_trie_test.go @@ -28,16 +28,16 @@ import ( "github.com/ethereum/go-ethereum/ethdb/memorydb" ) -func newEmptySecure() *SecureTrie { - trie, _ := NewSecure(common.Hash{}, common.Hash{}, NewDatabase(memorydb.New())) +func newEmptySecure() *StateTrie { + trie, _ := NewStateTrie(common.Hash{}, common.Hash{}, NewDatabase(memorydb.New())) return trie } -// makeTestSecureTrie creates a large enough secure trie for testing. -func makeTestSecureTrie() (*Database, *SecureTrie, map[string][]byte) { +// makeTestStateTrie creates a large enough secure trie for testing. +func makeTestStateTrie() (*Database, *StateTrie, map[string][]byte) { // Create an empty trie triedb := NewDatabase(memorydb.New()) - trie, _ := NewSecure(common.Hash{}, common.Hash{}, triedb) + trie, _ := NewStateTrie(common.Hash{}, common.Hash{}, triedb) // Fill it with some arbitrary data content := make(map[string][]byte) @@ -112,12 +112,12 @@ func TestSecureGetKey(t *testing.T) { } } -func TestSecureTrieConcurrency(t *testing.T) { +func TestStateTrieConcurrency(t *testing.T) { // Create an initial trie and copy if for concurrent access - _, trie, _ := makeTestSecureTrie() + _, trie, _ := makeTestStateTrie() threads := runtime.NumCPU() - tries := make([]*SecureTrie, threads) + tries := make([]*StateTrie, threads) for i := 0; i < threads; i++ { tries[i] = trie.Copy() } diff --git a/trie/sync_test.go b/trie/sync_test.go index c2ddb98c7..9fd1d636c 100644 --- a/trie/sync_test.go +++ b/trie/sync_test.go @@ -27,10 +27,10 @@ import ( ) // makeTestTrie create a sample test trie to test node-wise reconstruction. -func makeTestTrie() (*Database, *SecureTrie, map[string][]byte) { +func makeTestTrie() (*Database, *StateTrie, map[string][]byte) { // Create an empty trie triedb := NewDatabase(memorydb.New()) - trie, _ := NewSecure(common.Hash{}, common.Hash{}, triedb) + trie, _ := NewStateTrie(common.Hash{}, common.Hash{}, triedb) // Fill it with some arbitrary data content := make(map[string][]byte) @@ -67,7 +67,7 @@ func makeTestTrie() (*Database, *SecureTrie, map[string][]byte) { // content map. func checkTrieContents(t *testing.T, db *Database, root []byte, content map[string][]byte) { // Check root availability and trie contents - trie, err := NewSecure(common.Hash{}, common.BytesToHash(root), db) + trie, err := NewStateTrie(common.Hash{}, common.BytesToHash(root), db) if err != nil { t.Fatalf("failed to create trie at %x: %v", root, err) } @@ -84,7 +84,7 @@ func checkTrieContents(t *testing.T, db *Database, root []byte, content map[stri // checkTrieConsistency checks that all nodes in a trie are indeed present. func checkTrieConsistency(db *Database, root common.Hash) error { // Create and iterate a trie rooted in a subnode - trie, err := NewSecure(common.Hash{}, root, db) + trie, err := NewStateTrie(common.Hash{}, root, db) if err != nil { return nil // Consider a non existent state consistent } diff --git a/trie/trie.go b/trie/trie.go index 08f2480db..9274d8838 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -23,10 +23,8 @@ import ( "fmt" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/rlp" ) var ( @@ -275,14 +273,6 @@ func (t *Trie) Update(key, value []byte) { } } -func (t *Trie) TryUpdateAccount(key []byte, acc *types.StateAccount) error { - data, err := rlp.EncodeToBytes(acc) - if err != nil { - return fmt.Errorf("can't encode object at %x: %w", key[:], err) - } - return t.TryUpdate(key, data) -} - // TryUpdate associates key with value in the trie. Subsequent calls to // Get will return value. If value has length zero, any existing value // is deleted from the trie and calls to Get will return nil. @@ -292,6 +282,12 @@ func (t *Trie) TryUpdateAccount(key []byte, acc *types.StateAccount) error { // // If a node was not found in the database, a MissingNodeError is returned. func (t *Trie) TryUpdate(key, value []byte) error { + return t.tryUpdate(key, value) +} + +// tryUpdate expects an RLP-encoded value and performs the core function +// for TryUpdate and TryUpdateAccount. +func (t *Trie) tryUpdate(key, value []byte) error { t.unhashed++ k := keybytesToHex(key) if len(value) != 0 { From e44d6551c3c872584722c366c863381f7e91df91 Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Mon, 8 Aug 2022 17:08:36 +0800 Subject: [PATCH 26/59] cmd, core, ethdb, node: move chain freezer one folder deeper (#25487) * cmd, core, ethdb, node: create chain freezer in a sub folder * core/rawdb: remove unused code * core, ethdb, node: add AncientDatadir API back * cmd, core: extend freezer info dump for sub-ancient-store * core/rawdb: rework freezer inspector * core/rawdb: address comments from Peter * core/rawdb: fix build issue --- cmd/geth/dbcmd.go | 53 ++++++++------------ cmd/utils/flags.go | 2 +- core/rawdb/accessors_chain.go | 26 +++++----- core/rawdb/ancient_scheme.go | 86 ++++++++++++++++++++++++++++++++ core/rawdb/chain_freezer.go | 10 ++-- core/rawdb/chain_iterator.go | 2 +- core/rawdb/database.go | 50 ++++++++++++++++--- core/rawdb/freezer.go | 11 +--- core/rawdb/freezer_table.go | 8 ++- core/rawdb/freezer_table_test.go | 2 +- core/rawdb/schema.go | 27 ---------- ethdb/database.go | 5 +- node/node.go | 22 ++++---- 13 files changed, 191 insertions(+), 113 deletions(-) create mode 100644 core/rawdb/ancient_scheme.go diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index 27661d2c9..ab7427712 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -22,7 +22,6 @@ import ( "os" "os/signal" "path/filepath" - "sort" "strconv" "strings" "syscall" @@ -160,8 +159,8 @@ WARNING: This is a low-level operation which may cause database corruption!`, dbDumpFreezerIndex = &cli.Command{ Action: freezerInspect, Name: "freezer-index", - Usage: "Dump out the index of a given freezer type", - ArgsUsage: " ", + Usage: "Dump out the index of a specific freezer table", + ArgsUsage: " ", Flags: flags.Merge([]cli.Flag{ utils.SyncModeFlag, }, utils.NetworkFlags, utils.DatabasePathFlags), @@ -275,7 +274,7 @@ func inspect(ctx *cli.Context) error { start []byte ) if ctx.NArg() > 2 { - return fmt.Errorf("Max 2 arguments: %v", ctx.Command.ArgsUsage) + return fmt.Errorf("max 2 arguments: %v", ctx.Command.ArgsUsage) } if ctx.NArg() >= 1 { if d, err := hexutil.Decode(ctx.Args().Get(0)); err != nil { @@ -536,43 +535,35 @@ func dbDumpTrie(ctx *cli.Context) error { } func freezerInspect(ctx *cli.Context) error { - var ( - start, end int64 - disableSnappy bool - err error - ) - if ctx.NArg() < 3 { + if ctx.NArg() < 4 { return fmt.Errorf("required arguments: %v", ctx.Command.ArgsUsage) } - kind := ctx.Args().Get(0) - if noSnap, ok := rawdb.FreezerNoSnappy[kind]; !ok { - var options []string - for opt := range rawdb.FreezerNoSnappy { - options = append(options, opt) - } - sort.Strings(options) - return fmt.Errorf("Could read freezer-type '%v'. Available options: %v", kind, options) - } else { - disableSnappy = noSnap - } - if start, err = strconv.ParseInt(ctx.Args().Get(1), 10, 64); err != nil { - log.Info("Could read start-param", "error", err) + var ( + freezer = ctx.Args().Get(0) + table = ctx.Args().Get(1) + ) + start, err := strconv.ParseInt(ctx.Args().Get(2), 10, 64) + if err != nil { + log.Info("Could not read start-param", "err", err) return err } - if end, err = strconv.ParseInt(ctx.Args().Get(2), 10, 64); err != nil { - log.Info("Could read count param", "error", err) + end, err := strconv.ParseInt(ctx.Args().Get(3), 10, 64) + if err != nil { + log.Info("Could not read count param", "err", err) return err } stack, _ := makeConfigNode(ctx) defer stack.Close() - path := filepath.Join(stack.ResolvePath("chaindata"), "ancient") - log.Info("Opening freezer", "location", path, "name", kind) - if f, err := rawdb.NewFreezerTable(path, kind, disableSnappy, true); err != nil { + + db := utils.MakeChainDatabase(ctx, stack, true) + defer db.Close() + + ancient, err := db.AncientDatadir() + if err != nil { + log.Info("Failed to retrieve ancient root", "err", err) return err - } else { - f.DumpIndex(start, end) } - return nil + return rawdb.InspectFreezerTable(ancient, freezer, table, start, end) } func importLDBdata(ctx *cli.Context) error { diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index ff85e259a..efea93492 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -91,7 +91,7 @@ var ( } AncientFlag = &flags.DirectoryFlag{ Name: "datadir.ancient", - Usage: "Data directory for ancient chain segments (default = inside chaindata)", + Usage: "Root directory for ancient data (default = inside chaindata)", Category: flags.EthCategory, } MinFreeDiskSpaceFlag = &flags.DirectoryFlag{ diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index 8ea2e2ca7..aeba3690d 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -37,7 +37,7 @@ import ( func ReadCanonicalHash(db ethdb.Reader, number uint64) common.Hash { var data []byte db.ReadAncients(func(reader ethdb.AncientReaderOp) error { - data, _ = reader.Ancient(freezerHashTable, number) + data, _ = reader.Ancient(chainFreezerHashTable, number) if len(data) == 0 { // Get it by hash from leveldb data, _ = db.Get(headerHashKey(number)) @@ -335,7 +335,7 @@ func ReadHeaderRange(db ethdb.Reader, number uint64, count uint64) []rlp.RawValu } // read remaining from ancients max := count * 700 - data, err := db.AncientRange(freezerHeaderTable, i+1-count, count, max) + data, err := db.AncientRange(chainFreezerHeaderTable, i+1-count, count, max) if err == nil && uint64(len(data)) == count { // the data is on the order [h, h+1, .., n] -- reordering needed for i := range data { @@ -352,7 +352,7 @@ func ReadHeaderRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValu // First try to look up the data in ancient database. Extra hash // comparison is necessary since ancient database only maintains // the canonical data. - data, _ = reader.Ancient(freezerHeaderTable, number) + data, _ = reader.Ancient(chainFreezerHeaderTable, number) if len(data) > 0 && crypto.Keccak256Hash(data) == hash { return nil } @@ -428,7 +428,7 @@ func deleteHeaderWithoutNumber(db ethdb.KeyValueWriter, hash common.Hash, number // isCanon is an internal utility method, to check whether the given number/hash // is part of the ancient (canon) set. func isCanon(reader ethdb.AncientReaderOp, number uint64, hash common.Hash) bool { - h, err := reader.Ancient(freezerHashTable, number) + h, err := reader.Ancient(chainFreezerHashTable, number) if err != nil { return false } @@ -444,7 +444,7 @@ func ReadBodyRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue db.ReadAncients(func(reader ethdb.AncientReaderOp) error { // Check if the data is in ancients if isCanon(reader, number, hash) { - data, _ = reader.Ancient(freezerBodiesTable, number) + data, _ = reader.Ancient(chainFreezerBodiesTable, number) return nil } // If not, try reading from leveldb @@ -459,7 +459,7 @@ func ReadBodyRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue func ReadCanonicalBodyRLP(db ethdb.Reader, number uint64) rlp.RawValue { var data []byte db.ReadAncients(func(reader ethdb.AncientReaderOp) error { - data, _ = reader.Ancient(freezerBodiesTable, number) + data, _ = reader.Ancient(chainFreezerBodiesTable, number) if len(data) > 0 { return nil } @@ -527,7 +527,7 @@ func ReadTdRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawValue { db.ReadAncients(func(reader ethdb.AncientReaderOp) error { // Check if the data is in ancients if isCanon(reader, number, hash) { - data, _ = reader.Ancient(freezerDifficultyTable, number) + data, _ = reader.Ancient(chainFreezerDifficultyTable, number) return nil } // If not, try reading from leveldb @@ -587,7 +587,7 @@ func ReadReceiptsRLP(db ethdb.Reader, hash common.Hash, number uint64) rlp.RawVa db.ReadAncients(func(reader ethdb.AncientReaderOp) error { // Check if the data is in ancients if isCanon(reader, number, hash) { - data, _ = reader.Ancient(freezerReceiptTable, number) + data, _ = reader.Ancient(chainFreezerReceiptTable, number) return nil } // If not, try reading from leveldb @@ -819,19 +819,19 @@ func WriteAncientBlocks(db ethdb.AncientWriter, blocks []*types.Block, receipts func writeAncientBlock(op ethdb.AncientWriteOp, block *types.Block, header *types.Header, receipts []*types.ReceiptForStorage, td *big.Int) error { num := block.NumberU64() - if err := op.AppendRaw(freezerHashTable, num, block.Hash().Bytes()); err != nil { + if err := op.AppendRaw(chainFreezerHashTable, num, block.Hash().Bytes()); err != nil { return fmt.Errorf("can't add block %d hash: %v", num, err) } - if err := op.Append(freezerHeaderTable, num, header); err != nil { + if err := op.Append(chainFreezerHeaderTable, num, header); err != nil { return fmt.Errorf("can't append block header %d: %v", num, err) } - if err := op.Append(freezerBodiesTable, num, block.Body()); err != nil { + if err := op.Append(chainFreezerBodiesTable, num, block.Body()); err != nil { return fmt.Errorf("can't append block body %d: %v", num, err) } - if err := op.Append(freezerReceiptTable, num, receipts); err != nil { + if err := op.Append(chainFreezerReceiptTable, num, receipts); err != nil { return fmt.Errorf("can't append block %d receipts: %v", num, err) } - if err := op.Append(freezerDifficultyTable, num, td); err != nil { + if err := op.Append(chainFreezerDifficultyTable, num, td); err != nil { return fmt.Errorf("can't append block %d total difficulty: %v", num, err) } return nil diff --git a/core/rawdb/ancient_scheme.go b/core/rawdb/ancient_scheme.go new file mode 100644 index 000000000..3da061cbd --- /dev/null +++ b/core/rawdb/ancient_scheme.go @@ -0,0 +1,86 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rawdb + +import "fmt" + +// The list of table names of chain freezer. +const ( + // chainFreezerHeaderTable indicates the name of the freezer header table. + chainFreezerHeaderTable = "headers" + + // chainFreezerHashTable indicates the name of the freezer canonical hash table. + chainFreezerHashTable = "hashes" + + // chainFreezerBodiesTable indicates the name of the freezer block body table. + chainFreezerBodiesTable = "bodies" + + // chainFreezerReceiptTable indicates the name of the freezer receipts table. + chainFreezerReceiptTable = "receipts" + + // chainFreezerDifficultyTable indicates the name of the freezer total difficulty table. + chainFreezerDifficultyTable = "diffs" +) + +// chainFreezerNoSnappy configures whether compression is disabled for the ancient-tables. +// Hashes and difficulties don't compress well. +var chainFreezerNoSnappy = map[string]bool{ + chainFreezerHeaderTable: false, + chainFreezerHashTable: true, + chainFreezerBodiesTable: false, + chainFreezerReceiptTable: false, + chainFreezerDifficultyTable: true, +} + +// The list of identifiers of ancient stores. +var ( + chainFreezerName = "chain" // the folder name of chain segment ancient store. +) + +// freezers the collections of all builtin freezers. +var freezers = []string{chainFreezerName} + +// InspectFreezerTable dumps out the index of a specific freezer table. The passed +// ancient indicates the path of root ancient directory where the chain freezer can +// be opened. Start and end specify the range for dumping out indexes. +// Note this function can only be used for debugging purposes. +func InspectFreezerTable(ancient string, freezerName string, tableName string, start, end int64) error { + var ( + path string + tables map[string]bool + ) + switch freezerName { + case chainFreezerName: + path, tables = resolveChainFreezerDir(ancient), chainFreezerNoSnappy + default: + return fmt.Errorf("unknown freezer, supported ones: %v", freezers) + } + noSnappy, exist := tables[tableName] + if !exist { + var names []string + for name := range tables { + names = append(names, name) + } + return fmt.Errorf("unknown table, supported ones: %v", names) + } + table, err := newFreezerTable(path, tableName, noSnappy, true) + if err != nil { + return err + } + table.dumpIndexStdout(start, end) + return nil +} diff --git a/core/rawdb/chain_freezer.go b/core/rawdb/chain_freezer.go index ec39b7b59..7d9c9c015 100644 --- a/core/rawdb/chain_freezer.go +++ b/core/rawdb/chain_freezer.go @@ -278,19 +278,19 @@ func (f *chainFreezer) freezeRange(nfdb *nofreezedb, number, limit uint64) (hash } // Write to the batch. - if err := op.AppendRaw(freezerHashTable, number, hash[:]); err != nil { + if err := op.AppendRaw(chainFreezerHashTable, number, hash[:]); err != nil { return fmt.Errorf("can't write hash to Freezer: %v", err) } - if err := op.AppendRaw(freezerHeaderTable, number, header); err != nil { + if err := op.AppendRaw(chainFreezerHeaderTable, number, header); err != nil { return fmt.Errorf("can't write header to Freezer: %v", err) } - if err := op.AppendRaw(freezerBodiesTable, number, body); err != nil { + if err := op.AppendRaw(chainFreezerBodiesTable, number, body); err != nil { return fmt.Errorf("can't write body to Freezer: %v", err) } - if err := op.AppendRaw(freezerReceiptTable, number, receipts); err != nil { + if err := op.AppendRaw(chainFreezerReceiptTable, number, receipts); err != nil { return fmt.Errorf("can't write receipts to Freezer: %v", err) } - if err := op.AppendRaw(freezerDifficultyTable, number, td); err != nil { + if err := op.AppendRaw(chainFreezerDifficultyTable, number, td); err != nil { return fmt.Errorf("can't write td to Freezer: %v", err) } diff --git a/core/rawdb/chain_iterator.go b/core/rawdb/chain_iterator.go index 21e42f42d..867fed63a 100644 --- a/core/rawdb/chain_iterator.go +++ b/core/rawdb/chain_iterator.go @@ -50,7 +50,7 @@ func InitDatabaseFromFreezer(db ethdb.Database) { if i+count > frozen { count = frozen - i } - data, err := db.AncientRange(freezerHashTable, i, count, 32*count) + data, err := db.AncientRange(chainFreezerHashTable, i, count, 32*count) if err != nil { log.Crit("Failed to init database from freezer", "err", err) } diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 3fbee41da..831ca69c4 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -21,6 +21,7 @@ import ( "errors" "fmt" "os" + "path" "sync/atomic" "time" @@ -34,10 +35,16 @@ import ( // freezerdb is a database wrapper that enabled freezer data retrievals. type freezerdb struct { + ancientRoot string ethdb.KeyValueStore ethdb.AncientStore } +// AncientDatadir returns the path of root ancient directory. +func (frdb *freezerdb) AncientDatadir() (string, error) { + return frdb.ancientRoot, nil +} + // Close implements io.Closer, closing both the fast key-value store as well as // the slow ancient tables. func (frdb *freezerdb) Close() error { @@ -162,12 +169,36 @@ func NewDatabase(db ethdb.KeyValueStore) ethdb.Database { return &nofreezedb{KeyValueStore: db} } +// resolveChainFreezerDir is a helper function which resolves the absolute path +// of chain freezer by considering backward compatibility. +func resolveChainFreezerDir(ancient string) string { + // Check if the chain freezer is already present in the specified + // sub folder, if not then two possibilities: + // - chain freezer is not initialized + // - chain freezer exists in legacy location (root ancient folder) + freezer := path.Join(ancient, chainFreezerName) + if !common.FileExist(freezer) { + if !common.FileExist(ancient) { + // The entire ancient store is not initialized, still use the sub + // folder for initialization. + } else { + // Ancient root is already initialized, then we hold the assumption + // that chain freezer is also initialized and located in root folder. + // In this case fallback to legacy location. + freezer = ancient + log.Info("Found legacy ancient chain path", "location", ancient) + } + } + return freezer +} + // NewDatabaseWithFreezer creates a high level database on top of a given key- // value data store with a freezer moving immutable chain segments into cold -// storage. -func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace string, readonly bool) (ethdb.Database, error) { +// storage. The passed ancient indicates the path of root ancient directory +// where the chain freezer can be opened. +func NewDatabaseWithFreezer(db ethdb.KeyValueStore, ancient string, namespace string, readonly bool) (ethdb.Database, error) { // Create the idle freezer instance - frdb, err := newChainFreezer(freezer, namespace, readonly, freezerTableSize, FreezerNoSnappy) + frdb, err := newChainFreezer(resolveChainFreezerDir(ancient), namespace, readonly, freezerTableSize, chainFreezerNoSnappy) if err != nil { return nil, err } @@ -198,7 +229,7 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace st // If the freezer already contains something, ensure that the genesis blocks // match, otherwise we might mix up freezers across chains and destroy both // the freezer and the key-value store. - frgenesis, err := frdb.Ancient(freezerHashTable, 0) + frgenesis, err := frdb.Ancient(chainFreezerHashTable, 0) if err != nil { return nil, fmt.Errorf("failed to retrieve genesis from ancient %v", err) } else if !bytes.Equal(kvgenesis, frgenesis) { @@ -244,6 +275,7 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace st }() } return &freezerdb{ + ancientRoot: ancient, KeyValueStore: db, AncientStore: frdb, }, nil @@ -273,13 +305,15 @@ func NewLevelDBDatabase(file string, cache int, handles int, namespace string, r } // NewLevelDBDatabaseWithFreezer creates a persistent key-value database with a -// freezer moving immutable chain segments into cold storage. -func NewLevelDBDatabaseWithFreezer(file string, cache int, handles int, freezer string, namespace string, readonly bool) (ethdb.Database, error) { +// freezer moving immutable chain segments into cold storage. The passed ancient +// indicates the path of root ancient directory where the chain freezer can be +// opened. +func NewLevelDBDatabaseWithFreezer(file string, cache int, handles int, ancient string, namespace string, readonly bool) (ethdb.Database, error) { kvdb, err := leveldb.New(file, cache, handles, namespace, readonly) if err != nil { return nil, err } - frdb, err := NewDatabaseWithFreezer(kvdb, freezer, namespace, readonly) + frdb, err := NewDatabaseWithFreezer(kvdb, ancient, namespace, readonly) if err != nil { kvdb.Close() return nil, err @@ -441,7 +475,7 @@ func InspectDatabase(db ethdb.Database, keyPrefix, keyStart []byte) error { } // Inspect append-only file store then. ancientSizes := []*common.StorageSize{&ancientHeadersSize, &ancientBodiesSize, &ancientReceiptsSize, &ancientHashesSize, &ancientTdsSize} - for i, category := range []string{freezerHeaderTable, freezerBodiesTable, freezerReceiptTable, freezerHashTable, freezerDifficultyTable} { + for i, category := range []string{chainFreezerHeaderTable, chainFreezerBodiesTable, chainFreezerReceiptTable, chainFreezerHashTable, chainFreezerDifficultyTable} { if size, err := db.AncientSize(category); err == nil { *ancientSizes[i] += common.StorageSize(size) total += common.StorageSize(size) diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index 63fd8cdcf..6dea98c3d 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -68,8 +68,6 @@ type Freezer struct { frozen uint64 // Number of blocks already frozen tail uint64 // Number of the first stored item in the freezer - datadir string // Path of root directory of ancient store - // This lock synchronizes writers and the truncate operation, as well as // the "atomic" (batched) read operations. writeLock sync.RWMutex @@ -111,7 +109,6 @@ func NewFreezer(datadir string, namespace string, readonly bool, maxTableSize ui readonly: readonly, tables: make(map[string]*freezerTable), instanceLock: lock, - datadir: datadir, } // Create the tables. @@ -429,7 +426,7 @@ func (f *Freezer) MigrateTable(kind string, convert convertLegacyFn) error { // Set up new dir for the migrated table, the content of which // we'll at the end move over to the ancients dir. migrationPath := filepath.Join(ancientsPath, "migration") - newTable, err := NewFreezerTable(migrationPath, kind, table.noCompression, false) + newTable, err := newFreezerTable(migrationPath, kind, table.noCompression, false) if err != nil { return err } @@ -486,11 +483,5 @@ func (f *Freezer) MigrateTable(kind string, convert convertLegacyFn) error { if err := os.Remove(migrationPath); err != nil { return err } - return nil } - -// AncientDatadir returns the root directory path of the ancient store. -func (f *Freezer) AncientDatadir() (string, error) { - return f.datadir, nil -} diff --git a/core/rawdb/freezer_table.go b/core/rawdb/freezer_table.go index dd4a80efc..51d7d1930 100644 --- a/core/rawdb/freezer_table.go +++ b/core/rawdb/freezer_table.go @@ -123,8 +123,8 @@ type freezerTable struct { lock sync.RWMutex // Mutex protecting the data file descriptors } -// NewFreezerTable opens the given path as a freezer table. -func NewFreezerTable(path, name string, disableSnappy, readonly bool) (*freezerTable, error) { +// newFreezerTable opens the given path as a freezer table. +func newFreezerTable(path, name string, disableSnappy, readonly bool) (*freezerTable, error) { return newTable(path, name, metrics.NilMeter{}, metrics.NilMeter{}, metrics.NilGauge{}, freezerTableSize, disableSnappy, readonly) } @@ -884,9 +884,7 @@ func (t *freezerTable) Sync() error { return t.head.Sync() } -// DumpIndex is a debug print utility function, mainly for testing. It can also -// be used to analyse a live freezer table index. -func (t *freezerTable) DumpIndex(start, stop int64) { +func (t *freezerTable) dumpIndexStdout(start, stop int64) { t.dumpIndex(os.Stdout, start, stop) } diff --git a/core/rawdb/freezer_table_test.go b/core/rawdb/freezer_table_test.go index 0bddcf721..ea28e7175 100644 --- a/core/rawdb/freezer_table_test.go +++ b/core/rawdb/freezer_table_test.go @@ -902,7 +902,7 @@ func TestSequentialRead(t *testing.T) { } // Write 15 bytes 30 times writeChunks(t, f, 30, 15) - f.DumpIndex(0, 30) + f.dumpIndexStdout(0, 30) f.Close() } { // Open it, iterate, verify iteration diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index 041c9f044..a55ebdff7 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -111,33 +111,6 @@ var ( preimageHitCounter = metrics.NewRegisteredCounter("db/preimage/hits", nil) ) -const ( - // freezerHeaderTable indicates the name of the freezer header table. - freezerHeaderTable = "headers" - - // freezerHashTable indicates the name of the freezer canonical hash table. - freezerHashTable = "hashes" - - // freezerBodiesTable indicates the name of the freezer block body table. - freezerBodiesTable = "bodies" - - // freezerReceiptTable indicates the name of the freezer receipts table. - freezerReceiptTable = "receipts" - - // freezerDifficultyTable indicates the name of the freezer total difficulty table. - freezerDifficultyTable = "diffs" -) - -// FreezerNoSnappy configures whether compression is disabled for the ancient-tables. -// Hashes and difficulties don't compress well. -var FreezerNoSnappy = map[string]bool{ - freezerHeaderTable: false, - freezerHashTable: true, - freezerBodiesTable: false, - freezerReceiptTable: false, - freezerDifficultyTable: true, -} - // LegacyTxLookupEntry is the legacy TxLookupEntry definition with some unnecessary // fields. type LegacyTxLookupEntry struct { diff --git a/ethdb/database.go b/ethdb/database.go index e8faa2d86..361218f24 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -142,7 +142,9 @@ type AncientWriteOp interface { // AncientStater wraps the Stat method of a backing data store. type AncientStater interface { - // AncientDatadir returns the root directory path of the ancient store. + // AncientDatadir returns the path of root ancient directory. Empty string + // will be returned if ancient store is not enabled at all. The returned + // path can be used to construct the path of other freezers. AncientDatadir() (string, error) } @@ -172,7 +174,6 @@ type Stater interface { type AncientStore interface { AncientReader AncientWriter - AncientStater io.Closer } diff --git a/node/node.go b/node/node.go index 0a2b9eb83..b60e32f22 100644 --- a/node/node.go +++ b/node/node.go @@ -703,7 +703,7 @@ func (n *Node) OpenDatabase(name string, cache, handles int, namespace string, r // also attaching a chain freezer to it that moves ancient chain data from the // database to immutable append-only files. If the node is an ephemeral one, a // memory database is returned. -func (n *Node) OpenDatabaseWithFreezer(name string, cache, handles int, freezer, namespace string, readonly bool) (ethdb.Database, error) { +func (n *Node) OpenDatabaseWithFreezer(name string, cache, handles int, ancient string, namespace string, readonly bool) (ethdb.Database, error) { n.lock.Lock() defer n.lock.Unlock() if n.state == closedState { @@ -715,14 +715,7 @@ func (n *Node) OpenDatabaseWithFreezer(name string, cache, handles int, freezer, if n.config.DataDir == "" { db = rawdb.NewMemoryDatabase() } else { - root := n.ResolvePath(name) - switch { - case freezer == "": - freezer = filepath.Join(root, "ancient") - case !filepath.IsAbs(freezer): - freezer = n.ResolvePath(freezer) - } - db, err = rawdb.NewLevelDBDatabaseWithFreezer(root, cache, handles, freezer, namespace, readonly) + db, err = rawdb.NewLevelDBDatabaseWithFreezer(n.ResolvePath(name), cache, handles, n.ResolveAncient(name, ancient), namespace, readonly) } if err == nil { @@ -736,6 +729,17 @@ func (n *Node) ResolvePath(x string) string { return n.config.ResolvePath(x) } +// ResolveAncient returns the absolute path of the root ancient directory. +func (n *Node) ResolveAncient(name string, ancient string) string { + switch { + case ancient == "": + ancient = filepath.Join(n.ResolvePath(name), "ancient") + case !filepath.IsAbs(ancient): + ancient = n.ResolvePath(ancient) + } + return ancient +} + // closeTrackingDB wraps the Close method of a database. When the database is closed by the // service, the wrapper removes it from the node's database map. This ensures that Node // won't auto-close the database if it is closed by the service that opened it. From e93442c6cfca34aec3f83a03b35a97592571cce4 Mon Sep 17 00:00:00 2001 From: int88 <106391185+int88@users.noreply.github.com> Date: Mon, 8 Aug 2022 21:07:54 +0800 Subject: [PATCH 27/59] eth/downloader: fix log errors of queue_test.go (#25494) --- eth/downloader/queue_test.go | 4 ++-- les/downloader/queue_test.go | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/eth/downloader/queue_test.go b/eth/downloader/queue_test.go index e2e9654ea..d2dfbceff 100644 --- a/eth/downloader/queue_test.go +++ b/eth/downloader/queue_test.go @@ -156,7 +156,7 @@ func TestBasics(t *testing.T) { // The second peer should hit throttling if !throttle { - t.Fatalf("should not throttle") + t.Fatalf("should throttle") } // And not get any fetches at all, since it was throttled to begin with if fetchReq != nil { @@ -251,7 +251,7 @@ func TestEmptyBlocks(t *testing.T) { // there should be nothing to fetch, blocks are empty if fetchReq != nil { - t.Fatal("there should be no body fetch tasks remaining") + t.Fatal("there should be no receipt fetch tasks remaining") } } if q.blockTaskQueue.Size() != numOfBlocks-10 { diff --git a/les/downloader/queue_test.go b/les/downloader/queue_test.go index 2da8e4958..848979479 100644 --- a/les/downloader/queue_test.go +++ b/les/downloader/queue_test.go @@ -150,7 +150,7 @@ func TestBasics(t *testing.T) { // The second peer should hit throttling if !throttle { - t.Fatalf("should not throttle") + t.Fatalf("should throttle") } // And not get any fetches at all, since it was throttled to begin with if fetchReq != nil { @@ -239,7 +239,7 @@ func TestEmptyBlocks(t *testing.T) { // there should be nothing to fetch, blocks are empty if fetchReq != nil { - t.Fatal("there should be no body fetch tasks remaining") + t.Fatal("there should be no receipt fetch tasks remaining") } } if q.blockTaskQueue.Size() != numOfBlocks-10 { From e4b3bd6f26493edb379271787468b3cc36d3af16 Mon Sep 17 00:00:00 2001 From: int88 <106391185+int88@users.noreply.github.com> Date: Tue, 9 Aug 2022 06:20:46 +0800 Subject: [PATCH 28/59] core: fix uncle creation in TestFastVsFullChains (#25476) Co-authored-by: Felix Lange --- core/blockchain_test.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/blockchain_test.go b/core/blockchain_test.go index 8e9486536..517cb5e1d 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -759,9 +759,9 @@ func TestFastVsFullChains(t *testing.T) { block.AddTx(tx) } } - // If the block number is a multiple of 5, add a few bonus uncles to the block - if i%5 == 5 { - block.AddUncle(&types.Header{ParentHash: block.PrevBlock(i - 1).Hash(), Number: big.NewInt(int64(i - 1))}) + // If the block number is a multiple of 5, add an uncle to the block + if i%5 == 4 { + block.AddUncle(&types.Header{ParentHash: block.PrevBlock(i - 2).Hash(), Number: big.NewInt(int64(i))}) } }) // Import the chain as an archive node for the comparison baseline From 759d795c56cc158c4c296a0767a51cc262259fbc Mon Sep 17 00:00:00 2001 From: aaronbuchwald Date: Tue, 9 Aug 2022 04:23:41 -0400 Subject: [PATCH 29/59] eth: formatted error nit (#25499) --- eth/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth/api.go b/eth/api.go index 3ce8a2d1b..5642ef4c3 100644 --- a/eth/api.go +++ b/eth/api.go @@ -586,5 +586,5 @@ func (api *DebugAPI) GetAccessibleState(from, to rpc.BlockNumber) (uint64, error return uint64(i), nil } } - return 0, fmt.Errorf("no state found") + return 0, errors.New("no state found") } From 86de2e516e5a4a2bbe1d29b46a0f460fbdde8303 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi <1591639+s1na@users.noreply.github.com> Date: Tue, 9 Aug 2022 11:04:57 +0200 Subject: [PATCH 30/59] eth/tracers: add onlyTopCall option to callTracer (#25430) This PR allows users to pass in a config object directly to the tracers. Previously only the struct logger was configurable. It also adds an option to the call tracer which if enabled makes it ignore any subcall and collect only information about the top-level call. See #25419 for discussion. The tracers will silently ignore if they are passed a config they don't care about. --- core/vm/runtime/runtime_test.go | 6 +- eth/tracers/api.go | 11 +-- .../internal/tracetest/calltrace_test.go | 15 ++-- .../testdata/call_tracer/simple_onlytop.json | 72 +++++++++++++++++++ eth/tracers/js/goja.go | 13 +++- eth/tracers/js/tracer_test.go | 48 ++++++++++--- eth/tracers/native/4byte.go | 4 +- eth/tracers/native/call.go | 21 +++++- eth/tracers/native/noop.go | 4 +- eth/tracers/native/prestate.go | 4 +- eth/tracers/native/revertreason.go | 4 +- eth/tracers/native/tracer.go | 7 +- eth/tracers/tracers.go | 6 +- 13 files changed, 174 insertions(+), 41 deletions(-) create mode 100644 eth/tracers/internal/tracetest/testdata/call_tracer/simple_onlytop.json diff --git a/core/vm/runtime/runtime_test.go b/core/vm/runtime/runtime_test.go index 8864219ff..0fb287292 100644 --- a/core/vm/runtime/runtime_test.go +++ b/core/vm/runtime/runtime_test.go @@ -333,7 +333,7 @@ func benchmarkNonModifyingCode(gas uint64, code []byte, name string, tracerCode cfg.State, _ = state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) cfg.GasLimit = gas if len(tracerCode) > 0 { - tracer, err := tracers.New(tracerCode, new(tracers.Context)) + tracer, err := tracers.New(tracerCode, new(tracers.Context), nil) if err != nil { b.Fatal(err) } @@ -832,7 +832,7 @@ func TestRuntimeJSTracer(t *testing.T) { statedb.SetCode(common.HexToAddress("0xee"), calleeCode) statedb.SetCode(common.HexToAddress("0xff"), depressedCode) - tracer, err := tracers.New(jsTracer, new(tracers.Context)) + tracer, err := tracers.New(jsTracer, new(tracers.Context), nil) if err != nil { t.Fatal(err) } @@ -868,7 +868,7 @@ func TestJSTracerCreateTx(t *testing.T) { code := []byte{byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, byte(vm.RETURN)} statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) - tracer, err := tracers.New(jsTracer, new(tracers.Context)) + tracer, err := tracers.New(jsTracer, new(tracers.Context), nil) if err != nil { t.Fatal(err) } diff --git a/eth/tracers/api.go b/eth/tracers/api.go index 4041b1770..014e2f6ad 100644 --- a/eth/tracers/api.go +++ b/eth/tracers/api.go @@ -20,6 +20,7 @@ import ( "bufio" "bytes" "context" + "encoding/json" "errors" "fmt" "os" @@ -169,15 +170,15 @@ type TraceConfig struct { Tracer *string Timeout *string Reexec *uint64 + // Config specific to given tracer. Note struct logger + // config are historically embedded in main object. + TracerConfig json.RawMessage } // TraceCallConfig is the config for traceCall API. It holds one more // field to override the state for tracing. type TraceCallConfig struct { - *logger.Config - Tracer *string - Timeout *string - Reexec *uint64 + TraceConfig StateOverrides *ethapi.StateOverride BlockOverrides *ethapi.BlockOverrides } @@ -882,7 +883,7 @@ func (api *API) traceTx(ctx context.Context, message core.Message, txctx *Contex // Default tracer is the struct logger tracer = logger.NewStructLogger(config.Config) if config.Tracer != nil { - tracer, err = New(*config.Tracer, txctx) + tracer, err = New(*config.Tracer, txctx, config.TracerConfig) if err != nil { return nil, err } diff --git a/eth/tracers/internal/tracetest/calltrace_test.go b/eth/tracers/internal/tracetest/calltrace_test.go index cbf20ed00..cabddac49 100644 --- a/eth/tracers/internal/tracetest/calltrace_test.go +++ b/eth/tracers/internal/tracetest/calltrace_test.go @@ -118,10 +118,11 @@ type callTrace struct { // callTracerTest defines a single test to check the call tracer against. type callTracerTest struct { - Genesis *core.Genesis `json:"genesis"` - Context *callContext `json:"context"` - Input string `json:"input"` - Result *callTrace `json:"result"` + Genesis *core.Genesis `json:"genesis"` + Context *callContext `json:"context"` + Input string `json:"input"` + TracerConfig json.RawMessage `json:"tracerConfig"` + Result *callTrace `json:"result"` } // Iterates over all the input-output datasets in the tracer test harness and @@ -179,7 +180,7 @@ func testCallTracer(tracerName string, dirPath string, t *testing.T) { } _, statedb = tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false) ) - tracer, err := tracers.New(tracerName, new(tracers.Context)) + tracer, err := tracers.New(tracerName, new(tracers.Context), test.TracerConfig) if err != nil { t.Fatalf("failed to create call tracer: %v", err) } @@ -293,7 +294,7 @@ func benchTracer(tracerName string, test *callTracerTest, b *testing.B) { b.ReportAllocs() b.ResetTimer() for i := 0; i < b.N; i++ { - tracer, err := tracers.New(tracerName, new(tracers.Context)) + tracer, err := tracers.New(tracerName, new(tracers.Context), nil) if err != nil { b.Fatalf("failed to create call tracer: %v", err) } @@ -359,7 +360,7 @@ func TestZeroValueToNotExitCall(t *testing.T) { } _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), alloc, false) // Create the tracer, the EVM environment and run it - tracer, err := tracers.New("callTracer", nil) + tracer, err := tracers.New("callTracer", nil, nil) if err != nil { t.Fatalf("failed to create call tracer: %v", err) } diff --git a/eth/tracers/internal/tracetest/testdata/call_tracer/simple_onlytop.json b/eth/tracers/internal/tracetest/testdata/call_tracer/simple_onlytop.json new file mode 100644 index 000000000..ac1fef440 --- /dev/null +++ b/eth/tracers/internal/tracetest/testdata/call_tracer/simple_onlytop.json @@ -0,0 +1,72 @@ +{ + "context": { + "difficulty": "3502894804", + "gasLimit": "4722976", + "miner": "0x1585936b53834b021f68cc13eeefdec2efc8e724", + "number": "2289806", + "timestamp": "1513601314" + }, + "genesis": { + "alloc": { + "0x0024f658a46fbb89d8ac105e98d7ac7cbbaf27c5": { + "balance": "0x0", + "code": "0x", + "nonce": "22", + "storage": {} + }, + "0x3b873a919aa0512d5a0f09e6dcceaa4a6727fafe": { + "balance": "0x4d87094125a369d9bd5", + "code": "0x606060405236156100935763ffffffff60e060020a60003504166311ee8382811461009c57806313af4035146100be5780631f5e8f4c146100ee57806324daddc5146101125780634921a91a1461013b57806363e4bff414610157578063764978f91461017f578063893d20e8146101a1578063ba40aaa1146101cd578063cebc9a82146101f4578063e177246e14610216575b61009a5b5b565b005b34156100a457fe5b6100ac61023d565b60408051918252519081900360200190f35b34156100c657fe5b6100da600160a060020a0360043516610244565b604080519115158252519081900360200190f35b34156100f657fe5b6100da610307565b604080519115158252519081900360200190f35b341561011a57fe5b6100da6004351515610318565b604080519115158252519081900360200190f35b6100da6103d6565b604080519115158252519081900360200190f35b6100da600160a060020a0360043516610420565b604080519115158252519081900360200190f35b341561018757fe5b6100ac61046c565b60408051918252519081900360200190f35b34156101a957fe5b6101b1610473565b60408051600160a060020a039092168252519081900360200190f35b34156101d557fe5b6100da600435610483565b604080519115158252519081900360200190f35b34156101fc57fe5b6100ac61050d565b60408051918252519081900360200190f35b341561021e57fe5b6100da600435610514565b604080519115158252519081900360200190f35b6003545b90565b60006000610250610473565b600160a060020a031633600160a060020a03161415156102705760006000fd5b600160a060020a03831615156102865760006000fd5b50600054600160a060020a0390811690831681146102fb57604051600160a060020a0380851691908316907ffcf23a92150d56e85e3a3d33b357493246e55783095eb6a733eb8439ffc752c890600090a360008054600160a060020a031916600160a060020a03851617905560019150610300565b600091505b5b50919050565b60005460a060020a900460ff165b90565b60006000610324610473565b600160a060020a031633600160a060020a03161415156103445760006000fd5b5060005460a060020a900460ff16801515831515146102fb576000546040805160a060020a90920460ff1615158252841515602083015280517fe6cd46a119083b86efc6884b970bfa30c1708f53ba57b86716f15b2f4551a9539281900390910190a16000805460a060020a60ff02191660a060020a8515150217905560019150610300565b600091505b5b50919050565b60006103e0610307565b801561040557506103ef610473565b600160a060020a031633600160a060020a031614155b156104105760006000fd5b610419336105a0565b90505b5b90565b600061042a610307565b801561044f5750610439610473565b600160a060020a031633600160a060020a031614155b1561045a5760006000fd5b610463826105a0565b90505b5b919050565b6001545b90565b600054600160a060020a03165b90565b6000600061048f610473565b600160a060020a031633600160a060020a03161415156104af5760006000fd5b506001548281146102fb57604080518281526020810185905281517f79a3746dde45672c9e8ab3644b8bb9c399a103da2dc94b56ba09777330a83509929181900390910190a160018381559150610300565b600091505b5b50919050565b6002545b90565b60006000610520610473565b600160a060020a031633600160a060020a03161415156105405760006000fd5b506002548281146102fb57604080518281526020810185905281517ff6991a728965fedd6e927fdf16bdad42d8995970b4b31b8a2bf88767516e2494929181900390910190a1600283905560019150610300565b600091505b5b50919050565b60006000426105ad61023d565b116102fb576105c46105bd61050d565b4201610652565b6105cc61046c565b604051909150600160a060020a038416908290600081818185876187965a03f1925050501561063d57604080518281529051600160a060020a038516917f9bca65ce52fdef8a470977b51f247a2295123a4807dfa9e502edf0d30722da3b919081900360200190a260019150610300565b6102fb42610652565b5b600091505b50919050565b60038190555b505600a165627a7a72305820f3c973c8b7ed1f62000b6701bd5b708469e19d0f1d73fde378a56c07fd0b19090029", + "nonce": "1", + "storage": { + "0x0000000000000000000000000000000000000000000000000000000000000000": "0x000000000000000000000001b436ba50d378d4bbc8660d312a13df6af6e89dfb", + "0x0000000000000000000000000000000000000000000000000000000000000001": "0x00000000000000000000000000000000000000000000000006f05b59d3b20000", + "0x0000000000000000000000000000000000000000000000000000000000000002": "0x000000000000000000000000000000000000000000000000000000000000003c", + "0x0000000000000000000000000000000000000000000000000000000000000003": "0x000000000000000000000000000000000000000000000000000000005a37b834" + } + }, + "0xb436ba50d378d4bbc8660d312a13df6af6e89dfb": { + "balance": "0x1780d77678137ac1b775", + "code": "0x", + "nonce": "29072", + "storage": {} + } + }, + "config": { + "byzantiumBlock": 1700000, + "chainId": 3, + "daoForkSupport": true, + "eip150Block": 0, + "eip150Hash": "0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d", + "eip155Block": 10, + "eip158Block": 10, + "ethash": {}, + "homesteadBlock": 0 + }, + "difficulty": "3509749784", + "extraData": "0x4554482e45544846414e532e4f52472d4641313738394444", + "gasLimit": "4727564", + "hash": "0x609948ac3bd3c00b7736b933248891d6c901ee28f066241bddb28f4e00a9f440", + "miner": "0xbbf5029fd710d227630c8b7d338051b8e76d50b3", + "mixHash": "0xb131e4507c93c7377de00e7c271bf409ec7492767142ff0f45c882f8068c2ada", + "nonce": "0x4eb12e19c16d43da", + "number": "2289805", + "stateRoot": "0xc7f10f352bff82fac3c2999d3085093d12652e19c7fd32591de49dc5d91b4f1f", + "timestamp": "1513601261", + "totalDifficulty": "7143276353481064" + }, + "input": "0xf88b8271908506fc23ac0083015f90943b873a919aa0512d5a0f09e6dcceaa4a6727fafe80a463e4bff40000000000000000000000000024f658a46fbb89d8ac105e98d7ac7cbbaf27c52aa0bdce0b59e8761854e857fe64015f06dd08a4fbb7624f6094893a79a72e6ad6bea01d9dde033cff7bb235a3163f348a6d7ab8d6b52bc0963a95b91612e40ca766a4", + "tracerConfig": { + "onlyTopCall": true + }, + "result": { + "from": "0xb436ba50d378d4bbc8660d312a13df6af6e89dfb", + "gas": "0x10738", + "gasUsed": "0x3ef9", + "input": "0x63e4bff40000000000000000000000000024f658a46fbb89d8ac105e98d7ac7cbbaf27c5", + "output": "0x0000000000000000000000000000000000000000000000000000000000000001", + "to": "0x3b873a919aa0512d5a0f09e6dcceaa4a6727fafe", + "type": "CALL", + "value": "0x0" + } +} diff --git a/eth/tracers/js/goja.go b/eth/tracers/js/goja.go index a076168f7..8238bb603 100644 --- a/eth/tracers/js/goja.go +++ b/eth/tracers/js/goja.go @@ -125,7 +125,7 @@ type jsTracer struct { // The methods `result` and `fault` are required to be present. // The methods `step`, `enter`, and `exit` are optional, but note that // `enter` and `exit` always go together. -func newJsTracer(code string, ctx *tracers.Context) (tracers.Tracer, error) { +func newJsTracer(code string, ctx *tracers.Context, cfg json.RawMessage) (tracers.Tracer, error) { if c, ok := assetTracers[code]; ok { code = c } @@ -177,6 +177,17 @@ func newJsTracer(code string, ctx *tracers.Context) (tracers.Tracer, error) { t.exit = exit t.result = result t.fault = fault + + // Pass in config + if setup, ok := goja.AssertFunction(obj.Get("setup")); ok { + cfgStr := "{}" + if cfg != nil { + cfgStr = string(cfg) + } + if _, err := setup(obj, vm.ToValue(cfgStr)); err != nil { + return nil, err + } + } // Setup objects carrying data to JS. These are created once and re-used. t.log = &steplog{ vm: vm, diff --git a/eth/tracers/js/tracer_test.go b/eth/tracers/js/tracer_test.go index 0bdda770a..80a002d5a 100644 --- a/eth/tracers/js/tracer_test.go +++ b/eth/tracers/js/tracer_test.go @@ -85,7 +85,7 @@ func runTrace(tracer tracers.Tracer, vmctx *vmContext, chaincfg *params.ChainCon func TestTracer(t *testing.T) { execTracer := func(code string) ([]byte, string) { t.Helper() - tracer, err := newJsTracer(code, nil) + tracer, err := newJsTracer(code, nil, nil) if err != nil { t.Fatal(err) } @@ -149,7 +149,7 @@ func TestTracer(t *testing.T) { func TestHalt(t *testing.T) { timeout := errors.New("stahp") - tracer, err := newJsTracer("{step: function() { while(1); }, result: function() { return null; }, fault: function(){}}", nil) + tracer, err := newJsTracer("{step: function() { while(1); }, result: function() { return null; }, fault: function(){}}", nil, nil) if err != nil { t.Fatal(err) } @@ -163,7 +163,7 @@ func TestHalt(t *testing.T) { } func TestHaltBetweenSteps(t *testing.T) { - tracer, err := newJsTracer("{step: function() {}, fault: function() {}, result: function() { return null; }}", nil) + tracer, err := newJsTracer("{step: function() {}, fault: function() {}, result: function() { return null; }}", nil, nil) if err != nil { t.Fatal(err) } @@ -187,7 +187,7 @@ func TestHaltBetweenSteps(t *testing.T) { func TestNoStepExec(t *testing.T) { execTracer := func(code string) []byte { t.Helper() - tracer, err := newJsTracer(code, nil) + tracer, err := newJsTracer(code, nil, nil) if err != nil { t.Fatal(err) } @@ -221,7 +221,7 @@ func TestIsPrecompile(t *testing.T) { chaincfg.IstanbulBlock = big.NewInt(200) chaincfg.BerlinBlock = big.NewInt(300) txCtx := vm.TxContext{GasPrice: big.NewInt(100000)} - tracer, err := newJsTracer("{addr: toAddress('0000000000000000000000000000000000000009'), res: null, step: function() { this.res = isPrecompiled(this.addr); }, fault: function() {}, result: function() { return this.res; }}", nil) + tracer, err := newJsTracer("{addr: toAddress('0000000000000000000000000000000000000009'), res: null, step: function() { this.res = isPrecompiled(this.addr); }, fault: function() {}, result: function() { return this.res; }}", nil, nil) if err != nil { t.Fatal(err) } @@ -235,7 +235,7 @@ func TestIsPrecompile(t *testing.T) { t.Errorf("tracer should not consider blake2f as precompile in byzantium") } - tracer, _ = newJsTracer("{addr: toAddress('0000000000000000000000000000000000000009'), res: null, step: function() { this.res = isPrecompiled(this.addr); }, fault: function() {}, result: function() { return this.res; }}", nil) + tracer, _ = newJsTracer("{addr: toAddress('0000000000000000000000000000000000000009'), res: null, step: function() { this.res = isPrecompiled(this.addr); }, fault: function() {}, result: function() { return this.res; }}", nil, nil) blockCtx = vm.BlockContext{BlockNumber: big.NewInt(250)} res, err = runTrace(tracer, &vmContext{blockCtx, txCtx}, chaincfg) if err != nil { @@ -248,14 +248,14 @@ func TestIsPrecompile(t *testing.T) { func TestEnterExit(t *testing.T) { // test that either both or none of enter() and exit() are defined - if _, err := newJsTracer("{step: function() {}, fault: function() {}, result: function() { return null; }, enter: function() {}}", new(tracers.Context)); err == nil { + if _, err := newJsTracer("{step: function() {}, fault: function() {}, result: function() { return null; }, enter: function() {}}", new(tracers.Context), nil); err == nil { t.Fatal("tracer creation should've failed without exit() definition") } - if _, err := newJsTracer("{step: function() {}, fault: function() {}, result: function() { return null; }, enter: function() {}, exit: function() {}}", new(tracers.Context)); err != nil { + if _, err := newJsTracer("{step: function() {}, fault: function() {}, result: function() { return null; }, enter: function() {}, exit: function() {}}", new(tracers.Context), nil); err != nil { t.Fatal(err) } // test that the enter and exit method are correctly invoked and the values passed - tracer, err := newJsTracer("{enters: 0, exits: 0, enterGas: 0, gasUsed: 0, step: function() {}, fault: function() {}, result: function() { return {enters: this.enters, exits: this.exits, enterGas: this.enterGas, gasUsed: this.gasUsed} }, enter: function(frame) { this.enters++; this.enterGas = frame.getGas(); }, exit: function(res) { this.exits++; this.gasUsed = res.getGasUsed(); }}", new(tracers.Context)) + tracer, err := newJsTracer("{enters: 0, exits: 0, enterGas: 0, gasUsed: 0, step: function() {}, fault: function() {}, result: function() { return {enters: this.enters, exits: this.exits, enterGas: this.enterGas, gasUsed: this.gasUsed} }, enter: function(frame) { this.enters++; this.enterGas = frame.getGas(); }, exit: function(res) { this.exits++; this.gasUsed = res.getGasUsed(); }}", new(tracers.Context), nil) if err != nil { t.Fatal(err) } @@ -274,3 +274,33 @@ func TestEnterExit(t *testing.T) { t.Errorf("Number of invocations of enter() and exit() is wrong. Have %s, want %s\n", have, want) } } + +func TestSetup(t *testing.T) { + // Test empty config + _, err := newJsTracer(`{setup: function(cfg) { if (cfg !== "{}") { throw("invalid empty config") } }, fault: function() {}, result: function() {}}`, new(tracers.Context), nil) + if err != nil { + t.Error(err) + } + + cfg, err := json.Marshal(map[string]string{"foo": "bar"}) + if err != nil { + t.Fatal(err) + } + // Test no setup func + _, err = newJsTracer(`{fault: function() {}, result: function() {}}`, new(tracers.Context), cfg) + if err != nil { + t.Fatal(err) + } + // Test config value + tracer, err := newJsTracer("{config: null, setup: function(cfg) { this.config = JSON.parse(cfg) }, step: function() {}, fault: function() {}, result: function() { return this.config.foo }}", new(tracers.Context), cfg) + if err != nil { + t.Fatal(err) + } + have, err := tracer.GetResult() + if err != nil { + t.Fatal(err) + } + if string(have) != `"bar"` { + t.Errorf("tracer returned wrong result. have: %s, want: \"bar\"\n", string(have)) + } +} diff --git a/eth/tracers/native/4byte.go b/eth/tracers/native/4byte.go index 92cc70994..34e608bfd 100644 --- a/eth/tracers/native/4byte.go +++ b/eth/tracers/native/4byte.go @@ -55,11 +55,11 @@ type fourByteTracer struct { // newFourByteTracer returns a native go tracer which collects // 4 byte-identifiers of a tx, and implements vm.EVMLogger. -func newFourByteTracer(ctx *tracers.Context) tracers.Tracer { +func newFourByteTracer(ctx *tracers.Context, _ json.RawMessage) (tracers.Tracer, error) { t := &fourByteTracer{ ids: make(map[string]int), } - return t + return t, nil } // isPrecompiled returns whether the addr is a precompile. Logic borrowed from newJsTracer in eth/tracers/js/tracer.go diff --git a/eth/tracers/native/call.go b/eth/tracers/native/call.go index d334e328a..7af0e658a 100644 --- a/eth/tracers/native/call.go +++ b/eth/tracers/native/call.go @@ -50,16 +50,27 @@ type callFrame struct { type callTracer struct { env *vm.EVM callstack []callFrame + config callTracerConfig interrupt uint32 // Atomic flag to signal execution interruption reason error // Textual reason for the interruption } +type callTracerConfig struct { + OnlyTopCall bool `json:"onlyTopCall"` // If true, call tracer won't collect any subcalls +} + // newCallTracer returns a native go tracer which tracks // call frames of a tx, and implements vm.EVMLogger. -func newCallTracer(ctx *tracers.Context) tracers.Tracer { +func newCallTracer(ctx *tracers.Context, cfg json.RawMessage) (tracers.Tracer, error) { + var config callTracerConfig + if cfg != nil { + if err := json.Unmarshal(cfg, &config); err != nil { + return nil, err + } + } // First callframe contains tx context info // and is populated on start and end. - return &callTracer{callstack: make([]callFrame, 1)} + return &callTracer{callstack: make([]callFrame, 1), config: config}, nil } // CaptureStart implements the EVMLogger interface to initialize the tracing operation. @@ -101,6 +112,9 @@ func (t *callTracer) CaptureFault(pc uint64, op vm.OpCode, gas, cost uint64, _ * // CaptureEnter is called when EVM enters a new scope (via call, create or selfdestruct). func (t *callTracer) CaptureEnter(typ vm.OpCode, from common.Address, to common.Address, input []byte, gas uint64, value *big.Int) { + if t.config.OnlyTopCall { + return + } // Skip if tracing was interrupted if atomic.LoadUint32(&t.interrupt) > 0 { t.env.Cancel() @@ -121,6 +135,9 @@ func (t *callTracer) CaptureEnter(typ vm.OpCode, from common.Address, to common. // CaptureExit is called when EVM exits a scope, even if the scope didn't // execute any code. func (t *callTracer) CaptureExit(output []byte, gasUsed uint64, err error) { + if t.config.OnlyTopCall { + return + } size := len(t.callstack) if size <= 1 { return diff --git a/eth/tracers/native/noop.go b/eth/tracers/native/noop.go index 0849fd74e..c252b2408 100644 --- a/eth/tracers/native/noop.go +++ b/eth/tracers/native/noop.go @@ -35,8 +35,8 @@ func init() { type noopTracer struct{} // newNoopTracer returns a new noop tracer. -func newNoopTracer(ctx *tracers.Context) tracers.Tracer { - return &noopTracer{} +func newNoopTracer(ctx *tracers.Context, _ json.RawMessage) (tracers.Tracer, error) { + return &noopTracer{}, nil } // CaptureStart implements the EVMLogger interface to initialize the tracing operation. diff --git a/eth/tracers/native/prestate.go b/eth/tracers/native/prestate.go index 4d289ca62..b513f383b 100644 --- a/eth/tracers/native/prestate.go +++ b/eth/tracers/native/prestate.go @@ -51,10 +51,10 @@ type prestateTracer struct { reason error // Textual reason for the interruption } -func newPrestateTracer(ctx *tracers.Context) tracers.Tracer { +func newPrestateTracer(ctx *tracers.Context, _ json.RawMessage) (tracers.Tracer, error) { // First callframe contains tx context info // and is populated on start and end. - return &prestateTracer{prestate: prestate{}} + return &prestateTracer{prestate: prestate{}}, nil } // CaptureStart implements the EVMLogger interface to initialize the tracing operation. diff --git a/eth/tracers/native/revertreason.go b/eth/tracers/native/revertreason.go index b402396cb..d09b86100 100644 --- a/eth/tracers/native/revertreason.go +++ b/eth/tracers/native/revertreason.go @@ -46,8 +46,8 @@ type revertReasonTracer struct { } // newRevertReasonTracer returns a new revert reason tracer. -func newRevertReasonTracer(_ *tracers.Context) tracers.Tracer { - return &revertReasonTracer{} +func newRevertReasonTracer(_ *tracers.Context, _ json.RawMessage) (tracers.Tracer, error) { + return &revertReasonTracer{}, nil } // CaptureStart implements the EVMLogger interface to initialize the tracing operation. diff --git a/eth/tracers/native/tracer.go b/eth/tracers/native/tracer.go index 3bab870ea..9587caf19 100644 --- a/eth/tracers/native/tracer.go +++ b/eth/tracers/native/tracer.go @@ -35,6 +35,7 @@ func init() { package native import ( + "encoding/json" "errors" "github.com/ethereum/go-ethereum/eth/tracers" @@ -46,7 +47,7 @@ func init() { } // ctorFn is the constructor signature of a native tracer. -type ctorFn = func(*tracers.Context) tracers.Tracer +type ctorFn = func(*tracers.Context, json.RawMessage) (tracers.Tracer, error) /* ctors is a map of package-local tracer constructors. @@ -71,12 +72,12 @@ func register(name string, ctor ctorFn) { } // lookup returns a tracer, if one can be matched to the given name. -func lookup(name string, ctx *tracers.Context) (tracers.Tracer, error) { +func lookup(name string, ctx *tracers.Context, cfg json.RawMessage) (tracers.Tracer, error) { if ctors == nil { ctors = make(map[string]ctorFn) } if ctor, ok := ctors[name]; ok { - return ctor(ctx), nil + return ctor(ctx, cfg) } return nil, errors.New("no tracer found") } diff --git a/eth/tracers/tracers.go b/eth/tracers/tracers.go index e7073e7d2..3d2d1256c 100644 --- a/eth/tracers/tracers.go +++ b/eth/tracers/tracers.go @@ -42,7 +42,7 @@ type Tracer interface { Stop(err error) } -type lookupFunc func(string, *Context) (Tracer, error) +type lookupFunc func(string, *Context, json.RawMessage) (Tracer, error) var ( lookups []lookupFunc @@ -62,9 +62,9 @@ func RegisterLookup(wildcard bool, lookup lookupFunc) { // New returns a new instance of a tracer, by iterating through the // registered lookups. -func New(code string, ctx *Context) (Tracer, error) { +func New(code string, ctx *Context, cfg json.RawMessage) (Tracer, error) { for _, lookup := range lookups { - if tracer, err := lookup(code, ctx); err == nil { + if tracer, err := lookup(code, ctx, cfg); err == nil { return tracer, nil } } From a41ea8a97cd0f9db7a87e2dd15b380d4f1fbc311 Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Tue, 9 Aug 2022 17:44:39 +0800 Subject: [PATCH 31/59] all: cleanup the APIs for initializing genesis (#25473) * all: polish tests * core: apply feedback from Guillaume * core: fix comment --- cmd/devp2p/internal/ethtest/chain.go | 2 +- cmd/evm/runner.go | 2 +- cmd/faucet/faucet.go | 2 +- consensus/clique/snapshot_test.go | 4 +- core/blockchain_test.go | 4 +- core/genesis.go | 78 +++++++++++++------------ core/genesis_test.go | 14 ++--- core/rawdb/accessors_metadata.go | 13 +++-- core/rawdb/schema.go | 4 +- eth/catalyst/api_test.go | 2 +- eth/downloader/downloader_test.go | 6 +- eth/downloader/queue_test.go | 14 ++--- eth/downloader/testchain_test.go | 9 ++- eth/fetcher/block_fetcher_test.go | 12 ++-- eth/filters/filter_test.go | 18 +++++- eth/gasprice/gasprice_test.go | 8 +-- ethclient/ethclient_test.go | 2 +- ethclient/gethclient/gethclient_test.go | 2 +- les/catalyst/api_test.go | 2 +- les/downloader/queue_test.go | 14 ++--- les/downloader/testchain_test.go | 7 ++- les/fetcher/block_fetcher_test.go | 13 +++-- les/peer_test.go | 3 +- miner/stress/beacon/main.go | 2 +- tests/state_test.go | 2 +- tests/state_test_util.go | 2 +- 26 files changed, 133 insertions(+), 108 deletions(-) diff --git a/cmd/devp2p/internal/ethtest/chain.go b/cmd/devp2p/internal/ethtest/chain.go index df724f470..83ceb2a4f 100644 --- a/cmd/devp2p/internal/ethtest/chain.go +++ b/cmd/devp2p/internal/ethtest/chain.go @@ -139,7 +139,7 @@ func loadChain(chainfile string, genesis string) (*Chain, error) { if err != nil { return nil, err } - gblock := gen.ToBlock(nil) + gblock := gen.ToBlock() blocks, err := blocksFromFile(chainfile, gblock) if err != nil { diff --git a/cmd/evm/runner.go b/cmd/evm/runner.go index 05b9ccdeb..9b1975c05 100644 --- a/cmd/evm/runner.go +++ b/cmd/evm/runner.go @@ -138,7 +138,7 @@ func runCmd(ctx *cli.Context) error { gen := readGenesis(ctx.String(GenesisFlag.Name)) genesisConfig = gen db := rawdb.NewMemoryDatabase() - genesis := gen.ToBlock(db) + genesis := gen.MustCommit(db) statedb, _ = state.New(genesis.Root(), state.NewDatabase(db), nil) chainConfig = gen.Config } else { diff --git a/cmd/faucet/faucet.go b/cmd/faucet/faucet.go index 6c8796076..d49b9ed29 100644 --- a/cmd/faucet/faucet.go +++ b/cmd/faucet/faucet.go @@ -248,7 +248,7 @@ func newFaucet(genesis *core.Genesis, port int, enodes []*enode.Node, network ui cfg.SyncMode = downloader.LightSync cfg.NetworkId = network cfg.Genesis = genesis - utils.SetDNSDiscoveryDefaults(&cfg, genesis.ToBlock(nil).Hash()) + utils.SetDNSDiscoveryDefaults(&cfg, genesis.ToBlock().Hash()) lesBackend, err := les.New(stack, &cfg) if err != nil { diff --git a/consensus/clique/snapshot_test.go b/consensus/clique/snapshot_test.go index 094868ca7..b87ad8c23 100644 --- a/consensus/clique/snapshot_test.go +++ b/consensus/clique/snapshot_test.go @@ -403,7 +403,7 @@ func TestClique(t *testing.T) { } // Create a pristine blockchain with the genesis injected db := rawdb.NewMemoryDatabase() - genesis.Commit(db) + genesisBlock := genesis.MustCommit(db) // Assemble a chain of headers from the cast votes config := *params.TestChainConfig @@ -414,7 +414,7 @@ func TestClique(t *testing.T) { engine := New(config.Clique, db) engine.fakeDiff = true - blocks, _ := core.GenerateChain(&config, genesis.ToBlock(db), engine, db, len(tt.votes), func(j int, gen *core.BlockGen) { + blocks, _ := core.GenerateChain(&config, genesisBlock, engine, db, len(tt.votes), func(j int, gen *core.BlockGen) { // Cast the vote contained in this block gen.SetCoinbase(accounts.address(tt.votes[j].voted)) if tt.votes[j].auth { diff --git a/core/blockchain_test.go b/core/blockchain_test.go index 517cb5e1d..44256f618 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -1941,8 +1941,8 @@ func testSideImport(t *testing.T, numCanonBlocksInSidechain, blocksBetweenCommon Alloc: GenesisAlloc{addr: {Balance: big.NewInt(math.MaxInt64)}}, BaseFee: big.NewInt(params.InitialBaseFee), } - signer = types.LatestSigner(gspec.Config) - genesis, _ = gspec.Commit(db) + signer = types.LatestSigner(gspec.Config) + genesis = gspec.MustCommit(db) ) // Generate and import the canonical chain diskdb := rawdb.NewMemoryDatabase() diff --git a/core/genesis.go b/core/genesis.go index 45284f3de..71214e84f 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -80,10 +80,12 @@ func (ga *GenesisAlloc) UnmarshalJSON(data []byte) error { return nil } -// flush adds allocated genesis accounts into a fresh new statedb and -// commit the state changes into the given database handler. -func (ga *GenesisAlloc) flush(db ethdb.Database) (common.Hash, error) { - statedb, err := state.New(common.Hash{}, state.NewDatabase(db), nil) +// deriveHash computes the state root according to the genesis specification. +func (ga *GenesisAlloc) deriveHash() (common.Hash, error) { + // Create an ephemeral in-memory database for computing hash, + // all the derived states will be discarded to not pollute disk. + db := state.NewDatabase(rawdb.NewMemoryDatabase()) + statedb, err := state.New(common.Hash{}, db, nil) if err != nil { return common.Hash{}, err } @@ -95,25 +97,39 @@ func (ga *GenesisAlloc) flush(db ethdb.Database) (common.Hash, error) { statedb.SetState(addr, key, value) } } + return statedb.Commit(false) +} + +// flush is very similar with deriveHash, but the main difference is +// all the generated states will be persisted into the given database. +// Also, the genesis state specification will be flushed as well. +func (ga *GenesisAlloc) flush(db ethdb.Database) error { + statedb, err := state.New(common.Hash{}, state.NewDatabase(db), nil) + if err != nil { + return err + } + for addr, account := range *ga { + statedb.AddBalance(addr, account.Balance) + statedb.SetCode(addr, account.Code) + statedb.SetNonce(addr, account.Nonce) + for key, value := range account.Storage { + statedb.SetState(addr, key, value) + } + } root, err := statedb.Commit(false) if err != nil { - return common.Hash{}, err + return err } err = statedb.Database().TrieDB().Commit(root, true, nil) if err != nil { - return common.Hash{}, err + return err } - return root, nil -} - -// write writes the json marshaled genesis state into database -// with the given block hash as the unique identifier. -func (ga *GenesisAlloc) write(db ethdb.KeyValueWriter, hash common.Hash) error { + // Marshal the genesis state specification and persist. blob, err := json.Marshal(ga) if err != nil { return err } - rawdb.WriteGenesisState(db, hash, blob) + rawdb.WriteGenesisStateSpec(db, root, blob) return nil } @@ -121,7 +137,7 @@ func (ga *GenesisAlloc) write(db ethdb.KeyValueWriter, hash common.Hash) error { // hash and commits them into the given database handler. func CommitGenesisState(db ethdb.Database, hash common.Hash) error { var alloc GenesisAlloc - blob := rawdb.ReadGenesisState(db, hash) + blob := rawdb.ReadGenesisStateSpec(db, hash) if len(blob) != 0 { if err := alloc.UnmarshalJSON(blob); err != nil { return err @@ -151,8 +167,7 @@ func CommitGenesisState(db ethdb.Database, hash common.Hash) error { return errors.New("not found") } } - _, err := alloc.flush(db) - return err + return alloc.flush(db) } // GenesisAccount is an account in the state of the genesis block. @@ -273,7 +288,7 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, override genesis = DefaultGenesisBlock() } // Ensure the stored genesis matches with the given one. - hash := genesis.ToBlock(nil).Hash() + hash := genesis.ToBlock().Hash() if hash != stored { return genesis.Config, hash, &GenesisMismatchError{stored, hash} } @@ -286,7 +301,7 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, override } // Check whether the genesis block is already written. if genesis != nil { - hash := genesis.ToBlock(nil).Hash() + hash := genesis.ToBlock().Hash() if hash != stored { return genesis.Config, hash, &GenesisMismatchError{stored, hash} } @@ -347,13 +362,9 @@ func (g *Genesis) configOrDefault(ghash common.Hash) *params.ChainConfig { } } -// ToBlock creates the genesis block and writes state of a genesis specification -// to the given database (or discards it if nil). -func (g *Genesis) ToBlock(db ethdb.Database) *types.Block { - if db == nil { - db = rawdb.NewMemoryDatabase() - } - root, err := g.Alloc.flush(db) +// ToBlock returns the genesis block according to genesis specification. +func (g *Genesis) ToBlock() *types.Block { + root, err := g.Alloc.deriveHash() if err != nil { panic(err) } @@ -390,7 +401,7 @@ func (g *Genesis) ToBlock(db ethdb.Database) *types.Block { // Commit writes the block and state of a genesis specification to the database. // The block is committed as the canonical head block. func (g *Genesis) Commit(db ethdb.Database) (*types.Block, error) { - block := g.ToBlock(db) + block := g.ToBlock() if block.Number().Sign() != 0 { return nil, errors.New("can't commit genesis block with number > 0") } @@ -404,7 +415,10 @@ func (g *Genesis) Commit(db ethdb.Database) (*types.Block, error) { if config.Clique != nil && len(block.Extra()) < 32+crypto.SignatureLength { return nil, errors.New("can't start clique chain without signers") } - if err := g.Alloc.write(db, block.Hash()); err != nil { + // All the checks has passed, flush the states derived from the genesis + // specification as well as the specification itself into the provided + // database. + if err := g.Alloc.flush(db); err != nil { return nil, err } rawdb.WriteTd(db, block.Hash(), block.NumberU64(), block.Difficulty()) @@ -428,15 +442,6 @@ func (g *Genesis) MustCommit(db ethdb.Database) *types.Block { return block } -// GenesisBlockForTesting creates and writes a block in which addr has the given wei balance. -func GenesisBlockForTesting(db ethdb.Database, addr common.Address, balance *big.Int) *types.Block { - g := Genesis{ - Alloc: GenesisAlloc{addr: {Balance: balance}}, - BaseFee: big.NewInt(params.InitialBaseFee), - } - return g.MustCommit(db) -} - // DefaultGenesisBlock returns the Ethereum main net genesis block. func DefaultGenesisBlock() *Genesis { return &Genesis{ @@ -498,6 +503,7 @@ func DefaultSepoliaGenesisBlock() *Genesis { } } +// DefaultKilnGenesisBlock returns the kiln network genesis block. func DefaultKilnGenesisBlock() *Genesis { g := new(Genesis) reader := strings.NewReader(KilnAllocData) diff --git a/core/genesis_test.go b/core/genesis_test.go index e8010e3d4..ba3423e32 100644 --- a/core/genesis_test.go +++ b/core/genesis_test.go @@ -178,7 +178,7 @@ func TestGenesisHashes(t *testing.T) { t.Errorf("case: %d a), want: %s, got: %s", i, c.want.Hex(), have.Hex()) } // Test via ToBlock - if have := c.genesis.ToBlock(nil).Hash(); have != c.want { + if have := c.genesis.ToBlock().Hash(); have != c.want { t.Errorf("case: %d a), want: %s, got: %s", i, c.want.Hex(), have.Hex()) } } @@ -192,11 +192,7 @@ func TestGenesis_Commit(t *testing.T) { } db := rawdb.NewMemoryDatabase() - genesisBlock, err := genesis.Commit(db) - if err != nil { - t.Fatal(err) - } - + genesisBlock := genesis.MustCommit(db) if genesis.Difficulty != nil { t.Fatalf("assumption wrong") } @@ -221,12 +217,12 @@ func TestReadWriteGenesisAlloc(t *testing.T) { {1}: {Balance: big.NewInt(1), Storage: map[common.Hash]common.Hash{{1}: {1}}}, {2}: {Balance: big.NewInt(2), Storage: map[common.Hash]common.Hash{{2}: {2}}}, } - hash = common.HexToHash("0xdeadbeef") + hash, _ = alloc.deriveHash() ) - alloc.write(db, hash) + alloc.flush(db) var reload GenesisAlloc - err := reload.UnmarshalJSON(rawdb.ReadGenesisState(db, hash)) + err := reload.UnmarshalJSON(rawdb.ReadGenesisStateSpec(db, hash)) if err != nil { t.Fatalf("Failed to load genesis state %v", err) } diff --git a/core/rawdb/accessors_metadata.go b/core/rawdb/accessors_metadata.go index f5a161adb..7a9e6442f 100644 --- a/core/rawdb/accessors_metadata.go +++ b/core/rawdb/accessors_metadata.go @@ -81,15 +81,16 @@ func WriteChainConfig(db ethdb.KeyValueWriter, hash common.Hash, cfg *params.Cha } } -// ReadGenesisState retrieves the genesis state based on the given genesis hash. -func ReadGenesisState(db ethdb.KeyValueReader, hash common.Hash) []byte { - data, _ := db.Get(genesisKey(hash)) +// ReadGenesisStateSpec retrieves the genesis state specification based on the +// given genesis hash. +func ReadGenesisStateSpec(db ethdb.KeyValueReader, hash common.Hash) []byte { + data, _ := db.Get(genesisStateSpecKey(hash)) return data } -// WriteGenesisState writes the genesis state into the disk. -func WriteGenesisState(db ethdb.KeyValueWriter, hash common.Hash, data []byte) { - if err := db.Put(genesisKey(hash), data); err != nil { +// WriteGenesisStateSpec writes the genesis state specification into the disk. +func WriteGenesisStateSpec(db ethdb.KeyValueWriter, hash common.Hash, data []byte) { + if err := db.Put(genesisStateSpecKey(hash), data); err != nil { log.Crit("Failed to store genesis state", "err", err) } } diff --git a/core/rawdb/schema.go b/core/rawdb/schema.go index a55ebdff7..d5f751da3 100644 --- a/core/rawdb/schema.go +++ b/core/rawdb/schema.go @@ -220,7 +220,7 @@ func configKey(hash common.Hash) []byte { return append(configPrefix, hash.Bytes()...) } -// genesisKey = genesisPrefix + hash -func genesisKey(hash common.Hash) []byte { +// genesisStateSpecKey = genesisPrefix + hash +func genesisStateSpecKey(hash common.Hash) []byte { return append(genesisPrefix, hash.Bytes()...) } diff --git a/eth/catalyst/api_test.go b/eth/catalyst/api_test.go index df302f821..0d945993e 100644 --- a/eth/catalyst/api_test.go +++ b/eth/catalyst/api_test.go @@ -69,7 +69,7 @@ func generatePreMergeChain(n int) (*core.Genesis, []*types.Block) { g.AddTx(tx) testNonce++ } - gblock := genesis.ToBlock(db) + gblock := genesis.MustCommit(db) engine := ethash.NewFaker() blocks, _ := core.GenerateChain(config, gblock, engine, db, n, generate) totalDifficulty := big.NewInt(0) diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index 5e77d3272..1c126bdae 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -68,7 +68,11 @@ func newTesterWithNotification(t *testing.T, success func()) *downloadTester { t.Cleanup(func() { db.Close() }) - core.GenesisBlockForTesting(db, testAddress, big.NewInt(1000000000000000)) + gspec := core.Genesis{ + Alloc: core.GenesisAlloc{testAddress: {Balance: big.NewInt(1000000000000000)}}, + BaseFee: big.NewInt(params.InitialBaseFee), + } + gspec.MustCommit(db) chain, err := core.NewBlockChain(db, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil) if err != nil { diff --git a/eth/downloader/queue_test.go b/eth/downloader/queue_test.go index d2dfbceff..8631b27c9 100644 --- a/eth/downloader/queue_test.go +++ b/eth/downloader/queue_test.go @@ -27,24 +27,18 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/trie" ) -var ( - testdb = rawdb.NewMemoryDatabase() - genesis = core.GenesisBlockForTesting(testdb, testAddress, big.NewInt(1000000000000000)) -) - // makeChain creates a chain of n blocks starting at and including parent. // the returned hash chain is ordered head->parent. In addition, every 3rd block // contains a transaction and every 5th an uncle to allow testing correct block // reassembly. func makeChain(n int, seed byte, parent *types.Block, empty bool) ([]*types.Block, []types.Receipts) { - blocks, receipts := core.GenerateChain(params.TestChainConfig, parent, ethash.NewFaker(), testdb, n, func(i int, block *core.BlockGen) { + blocks, receipts := core.GenerateChain(params.TestChainConfig, parent, ethash.NewFaker(), testDB, n, func(i int, block *core.BlockGen) { block.SetCoinbase(common.Address{seed}) // Add one tx to every secondblock if !empty && i%2 == 0 { @@ -70,10 +64,10 @@ var emptyChain *chainData func init() { // Create a chain of blocks to import targetBlocks := 128 - blocks, _ := makeChain(targetBlocks, 0, genesis, false) + blocks, _ := makeChain(targetBlocks, 0, testGenesis, false) chain = &chainData{blocks, 0} - blocks, _ = makeChain(targetBlocks, 0, genesis, true) + blocks, _ = makeChain(targetBlocks, 0, testGenesis, true) emptyChain = &chainData{blocks, 0} } @@ -271,7 +265,7 @@ func TestEmptyBlocks(t *testing.T) { // some more advanced scenarios func XTestDelivery(t *testing.T) { // the outside network, holding blocks - blo, rec := makeChain(128, 0, genesis, false) + blo, rec := makeChain(128, 0, testGenesis, false) world := newNetwork() world.receipts = rec world.chain = blo diff --git a/eth/downloader/testchain_test.go b/eth/downloader/testchain_test.go index 8b873343c..785da40b5 100644 --- a/eth/downloader/testchain_test.go +++ b/eth/downloader/testchain_test.go @@ -37,7 +37,12 @@ var ( testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") testAddress = crypto.PubkeyToAddress(testKey.PublicKey) testDB = rawdb.NewMemoryDatabase() - testGenesis = core.GenesisBlockForTesting(testDB, testAddress, big.NewInt(1000000000000000)) + + testGspec = core.Genesis{ + Alloc: core.GenesisAlloc{testAddress: {Balance: big.NewInt(1000000000000000)}}, + BaseFee: big.NewInt(params.InitialBaseFee), + } + testGenesis = testGspec.MustCommit(testDB) ) // The common prefix of all test chains: @@ -212,7 +217,7 @@ func newTestBlockchain(blocks []*types.Block) *core.BlockChain { panic("Requested chain generation outside of init") } db := rawdb.NewMemoryDatabase() - core.GenesisBlockForTesting(db, testAddress, big.NewInt(1000000000000000)) + testGspec.MustCommit(db) chain, err := core.NewBlockChain(db, nil, params.TestChainConfig, ethash.NewFaker(), vm.Config{}, nil, nil) if err != nil { diff --git a/eth/fetcher/block_fetcher_test.go b/eth/fetcher/block_fetcher_test.go index 06c61ae55..bf7946952 100644 --- a/eth/fetcher/block_fetcher_test.go +++ b/eth/fetcher/block_fetcher_test.go @@ -36,10 +36,14 @@ import ( ) var ( - testdb = rawdb.NewMemoryDatabase() - testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - testAddress = crypto.PubkeyToAddress(testKey.PublicKey) - genesis = core.GenesisBlockForTesting(testdb, testAddress, big.NewInt(1000000000000000)) + testdb = rawdb.NewMemoryDatabase() + testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + testAddress = crypto.PubkeyToAddress(testKey.PublicKey) + gspec = core.Genesis{ + Alloc: core.GenesisAlloc{testAddress: {Balance: big.NewInt(1000000000000000)}}, + BaseFee: big.NewInt(params.InitialBaseFee), + } + genesis = gspec.MustCommit(testdb) unknownBlock = types.NewBlock(&types.Header{GasLimit: params.GenesisGasLimit, BaseFee: big.NewInt(params.InitialBaseFee)}, nil, nil, nil, trie.NewStackTrie(nil)) ) diff --git a/eth/filters/filter_test.go b/eth/filters/filter_test.go index ae4f06904..59fdde7e8 100644 --- a/eth/filters/filter_test.go +++ b/eth/filters/filter_test.go @@ -50,10 +50,17 @@ func BenchmarkFilters(b *testing.B) { addr2 = common.BytesToAddress([]byte("jeff")) addr3 = common.BytesToAddress([]byte("ethereum")) addr4 = common.BytesToAddress([]byte("random addresses please")) + + gspec = core.Genesis{ + Alloc: core.GenesisAlloc{addr1: {Balance: big.NewInt(1000000)}}, + BaseFee: big.NewInt(params.InitialBaseFee), + } + genesis = gspec.ToBlock() ) defer db.Close() - genesis := core.GenesisBlockForTesting(db, addr1, big.NewInt(1000000)) + gspec.MustCommit(db) + chain, receipts := core.GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), db, 100010, func(i int, gen *core.BlockGen) { switch i { case 2403: @@ -105,10 +112,17 @@ func TestFilters(t *testing.T) { hash2 = common.BytesToHash([]byte("topic2")) hash3 = common.BytesToHash([]byte("topic3")) hash4 = common.BytesToHash([]byte("topic4")) + + gspec = core.Genesis{ + Alloc: core.GenesisAlloc{addr: {Balance: big.NewInt(1000000)}}, + BaseFee: big.NewInt(params.InitialBaseFee), + } + genesis = gspec.ToBlock() ) defer db.Close() - genesis := core.GenesisBlockForTesting(db, addr, big.NewInt(1000000)) + gspec.MustCommit(db) + chain, receipts := core.GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), db, 1000, func(i int, gen *core.BlockGen) { switch i { case 1: diff --git a/eth/gasprice/gasprice_test.go b/eth/gasprice/gasprice_test.go index 3806e993c..d405188f8 100644 --- a/eth/gasprice/gasprice_test.go +++ b/eth/gasprice/gasprice_test.go @@ -130,10 +130,8 @@ func newTestBackend(t *testing.T, londonBlock *big.Int, pending bool) *testBacke config.TerminalTotalDifficulty = common.Big0 engine := ethash.NewFaker() db := rawdb.NewMemoryDatabase() - genesis, err := gspec.Commit(db) - if err != nil { - t.Fatal(err) - } + genesis := gspec.MustCommit(db) + // Generate testing blocks blocks, _ := core.GenerateChain(gspec.Config, genesis, engine, db, testHead+1, func(i int, b *core.BlockGen) { b.SetCoinbase(common.Address{1}) @@ -163,7 +161,7 @@ func newTestBackend(t *testing.T, londonBlock *big.Int, pending bool) *testBacke }) // Construct testing chain diskdb := rawdb.NewMemoryDatabase() - gspec.Commit(diskdb) + gspec.MustCommit(diskdb) chain, err := core.NewBlockChain(diskdb, &core.CacheConfig{TrieCleanNoPrefetch: true}, gspec.Config, engine, vm.Config{}, nil, nil) if err != nil { t.Fatalf("Failed to create local chain, %v", err) diff --git a/ethclient/ethclient_test.go b/ethclient/ethclient_test.go index 37add1bd3..d78a12fae 100644 --- a/ethclient/ethclient_test.go +++ b/ethclient/ethclient_test.go @@ -248,7 +248,7 @@ func generateTestChain() []*types.Block { g.AddTx(testTx2) } } - gblock := genesis.ToBlock(db) + gblock := genesis.MustCommit(db) engine := ethash.NewFaker() blocks, _ := core.GenerateChain(genesis.Config, gblock, engine, db, 2, generate) blocks = append([]*types.Block{gblock}, blocks...) diff --git a/ethclient/gethclient/gethclient_test.go b/ethclient/gethclient/gethclient_test.go index a0c17a034..b78d11c32 100644 --- a/ethclient/gethclient/gethclient_test.go +++ b/ethclient/gethclient/gethclient_test.go @@ -83,7 +83,7 @@ func generateTestChain() (*core.Genesis, []*types.Block) { g.OffsetTime(5) g.SetExtra([]byte("test")) } - gblock := genesis.ToBlock(db) + gblock := genesis.MustCommit(db) engine := ethash.NewFaker() blocks, _ := core.GenerateChain(config, gblock, engine, db, 1, generate) blocks = append([]*types.Block{gblock}, blocks...) diff --git a/les/catalyst/api_test.go b/les/catalyst/api_test.go index 70a6d2471..26c49d6ef 100644 --- a/les/catalyst/api_test.go +++ b/les/catalyst/api_test.go @@ -55,7 +55,7 @@ func generatePreMergeChain(n int) (*core.Genesis, []*types.Header, []*types.Bloc Timestamp: 9000, BaseFee: big.NewInt(params.InitialBaseFee), } - gblock := genesis.ToBlock(db) + gblock := genesis.MustCommit(db) engine := ethash.NewFaker() blocks, _ := core.GenerateChain(config, gblock, engine, db, n, nil) totalDifficulty := big.NewInt(0) diff --git a/les/downloader/queue_test.go b/les/downloader/queue_test.go index 848979479..44b220859 100644 --- a/les/downloader/queue_test.go +++ b/les/downloader/queue_test.go @@ -27,23 +27,17 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" ) -var ( - testdb = rawdb.NewMemoryDatabase() - genesis = core.GenesisBlockForTesting(testdb, testAddress, big.NewInt(1000000000000000)) -) - // makeChain creates a chain of n blocks starting at and including parent. // the returned hash chain is ordered head->parent. In addition, every 3rd block // contains a transaction and every 5th an uncle to allow testing correct block // reassembly. func makeChain(n int, seed byte, parent *types.Block, empty bool) ([]*types.Block, []types.Receipts) { - blocks, receipts := core.GenerateChain(params.TestChainConfig, parent, ethash.NewFaker(), testdb, n, func(i int, block *core.BlockGen) { + blocks, receipts := core.GenerateChain(params.TestChainConfig, parent, ethash.NewFaker(), testDB, n, func(i int, block *core.BlockGen) { block.SetCoinbase(common.Address{seed}) // Add one tx to every secondblock if !empty && i%2 == 0 { @@ -69,10 +63,10 @@ var emptyChain *chainData func init() { // Create a chain of blocks to import targetBlocks := 128 - blocks, _ := makeChain(targetBlocks, 0, genesis, false) + blocks, _ := makeChain(targetBlocks, 0, testGenesis, false) chain = &chainData{blocks, 0} - blocks, _ = makeChain(targetBlocks, 0, genesis, true) + blocks, _ = makeChain(targetBlocks, 0, testGenesis, true) emptyChain = &chainData{blocks, 0} } @@ -259,7 +253,7 @@ func TestEmptyBlocks(t *testing.T) { // some more advanced scenarios func XTestDelivery(t *testing.T) { // the outside network, holding blocks - blo, rec := makeChain(128, 0, genesis, false) + blo, rec := makeChain(128, 0, testGenesis, false) world := newNetwork() world.receipts = rec world.chain = blo diff --git a/les/downloader/testchain_test.go b/les/downloader/testchain_test.go index b9865f7e0..400eec94e 100644 --- a/les/downloader/testchain_test.go +++ b/les/downloader/testchain_test.go @@ -35,7 +35,12 @@ var ( testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") testAddress = crypto.PubkeyToAddress(testKey.PublicKey) testDB = rawdb.NewMemoryDatabase() - testGenesis = core.GenesisBlockForTesting(testDB, testAddress, big.NewInt(1000000000000000)) + + gspec = core.Genesis{ + Alloc: core.GenesisAlloc{testAddress: {Balance: big.NewInt(1000000000000000)}}, + BaseFee: big.NewInt(params.InitialBaseFee), + } + testGenesis = gspec.MustCommit(testDB) ) // The common prefix of all test chains: diff --git a/les/fetcher/block_fetcher_test.go b/les/fetcher/block_fetcher_test.go index de066ac26..caff7a3b3 100644 --- a/les/fetcher/block_fetcher_test.go +++ b/les/fetcher/block_fetcher_test.go @@ -35,10 +35,15 @@ import ( ) var ( - testdb = rawdb.NewMemoryDatabase() - testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - testAddress = crypto.PubkeyToAddress(testKey.PublicKey) - genesis = core.GenesisBlockForTesting(testdb, testAddress, big.NewInt(1000000000000000)) + testdb = rawdb.NewMemoryDatabase() + testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + testAddress = crypto.PubkeyToAddress(testKey.PublicKey) + + gspec = core.Genesis{ + Alloc: core.GenesisAlloc{testAddress: {Balance: big.NewInt(1000000000000000)}}, + BaseFee: big.NewInt(params.InitialBaseFee), + } + genesis = gspec.MustCommit(testdb) unknownBlock = types.NewBlock(&types.Header{GasLimit: params.GenesisGasLimit, BaseFee: big.NewInt(params.InitialBaseFee)}, nil, nil, nil, trie.NewStackTrie(nil)) ) diff --git a/les/peer_test.go b/les/peer_test.go index d6551ce6b..b8a1482a0 100644 --- a/les/peer_test.go +++ b/les/peer_test.go @@ -28,7 +28,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/forkid" - "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" @@ -100,7 +99,7 @@ type fakeChain struct{} func (f *fakeChain) Config() *params.ChainConfig { return params.MainnetChainConfig } func (f *fakeChain) Genesis() *types.Block { - return core.DefaultGenesisBlock().ToBlock(rawdb.NewMemoryDatabase()) + return core.DefaultGenesisBlock().ToBlock() } func (f *fakeChain) CurrentHeader() *types.Header { return &types.Header{Number: big.NewInt(10000000)} } diff --git a/miner/stress/beacon/main.go b/miner/stress/beacon/main.go index 439bcc5d1..88af84c7f 100644 --- a/miner/stress/beacon/main.go +++ b/miner/stress/beacon/main.go @@ -236,7 +236,7 @@ func newNodeManager(genesis *core.Genesis) *nodeManager { return &nodeManager{ close: make(chan struct{}), genesis: genesis, - genesisBlock: genesis.ToBlock(nil), + genesisBlock: genesis.ToBlock(), } } diff --git a/tests/state_test.go b/tests/state_test.go index 965ef71ba..d33ebc4b0 100644 --- a/tests/state_test.go +++ b/tests/state_test.go @@ -193,7 +193,7 @@ func runBenchmark(b *testing.B, t *StateTest) { return } vmconfig.ExtraEips = eips - block := t.genesis(config).ToBlock(nil) + block := t.genesis(config).ToBlock() _, statedb := MakePreState(rawdb.NewMemoryDatabase(), t.json.Pre, false) var baseFee *big.Int diff --git a/tests/state_test_util.go b/tests/state_test_util.go index d698b7c6f..38cdbc4d6 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -184,7 +184,7 @@ func (t *StateTest) RunNoVerify(subtest StateSubtest, vmconfig vm.Config, snapsh return nil, nil, common.Hash{}, UnsupportedForkError{subtest.Fork} } vmconfig.ExtraEips = eips - block := t.genesis(config).ToBlock(nil) + block := t.genesis(config).ToBlock() snaps, statedb := MakePreState(rawdb.NewMemoryDatabase(), t.json.Pre, snapshotter) var baseFee *big.Int From 877ef7f09e5296b15033dccb0e3a318f3583a6ca Mon Sep 17 00:00:00 2001 From: ycyraum Date: Wed, 10 Aug 2022 08:02:37 +0200 Subject: [PATCH 32/59] core: remove unused bc ChainContext in applyTransaction --- core/state_processor.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/state_processor.go b/core/state_processor.go index d4c77ae41..e511697c5 100644 --- a/core/state_processor.go +++ b/core/state_processor.go @@ -79,7 +79,7 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg return nil, nil, 0, fmt.Errorf("could not apply tx %d [%v]: %w", i, tx.Hash().Hex(), err) } statedb.Prepare(tx.Hash(), i) - receipt, err := applyTransaction(msg, p.config, p.bc, nil, gp, statedb, blockNumber, blockHash, tx, usedGas, vmenv) + receipt, err := applyTransaction(msg, p.config, nil, gp, statedb, blockNumber, blockHash, tx, usedGas, vmenv) if err != nil { return nil, nil, 0, fmt.Errorf("could not apply tx %d [%v]: %w", i, tx.Hash().Hex(), err) } @@ -92,7 +92,7 @@ func (p *StateProcessor) Process(block *types.Block, statedb *state.StateDB, cfg return receipts, allLogs, *usedGas, nil } -func applyTransaction(msg types.Message, config *params.ChainConfig, bc ChainContext, author *common.Address, gp *GasPool, statedb *state.StateDB, blockNumber *big.Int, blockHash common.Hash, tx *types.Transaction, usedGas *uint64, evm *vm.EVM) (*types.Receipt, error) { +func applyTransaction(msg types.Message, config *params.ChainConfig, author *common.Address, gp *GasPool, statedb *state.StateDB, blockNumber *big.Int, blockHash common.Hash, tx *types.Transaction, usedGas *uint64, evm *vm.EVM) (*types.Receipt, error) { // Create a new context to be used in the EVM environment. txContext := NewEVMTxContext(msg) evm.Reset(txContext, statedb) @@ -149,5 +149,5 @@ func ApplyTransaction(config *params.ChainConfig, bc ChainContext, author *commo // Create a new context to be used in the EVM environment blockContext := NewEVMBlockContext(header, bc, author) vmenv := vm.NewEVM(blockContext, vm.TxContext{}, statedb, config, cfg) - return applyTransaction(msg, config, bc, author, gp, statedb, header.Number, header.Hash(), tx, usedGas, vmenv) + return applyTransaction(msg, config, author, gp, statedb, header.Number, header.Hash(), tx, usedGas, vmenv) } From 1a18c14c43035fbcad98e986b8ac40719a8c15dc Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Wed, 10 Aug 2022 09:31:15 +0200 Subject: [PATCH 33/59] signer/rules: register clef api properly when rules are used (#25455) signer/rules: register clef api properly when rules are used, fixes #25298 --- signer/rules/rules.go | 1 + 1 file changed, 1 insertion(+) diff --git a/signer/rules/rules.go b/signer/rules/rules.go index 95b02e9ce..5ed4514e0 100644 --- a/signer/rules/rules.go +++ b/signer/rules/rules.go @@ -59,6 +59,7 @@ func NewRuleEvaluator(next core.UIClientAPI, jsbackend storage.Storage) (*rulese return c, nil } func (r *rulesetUI) RegisterUIServer(api *core.UIServerAPI) { + r.next.RegisterUIServer(api) // TODO, make it possible to query from js } From c0cc6f63629678ad7a8d9a33dc17bcf26122bde0 Mon Sep 17 00:00:00 2001 From: 0xe3b0c4 <110295932+0xe3b0c4@users.noreply.github.com> Date: Wed, 10 Aug 2022 16:30:59 +0800 Subject: [PATCH 34/59] build: add static linking support (#25492) This adds support for building statically-linked executables using ci.go. Static linking is enabled by default in Docker builds, making it possible to use the geth executable in any Docker image, regardless of the Linux distribution the Dockerfile is based on. Co-authored-by: Felix Lange --- Dockerfile | 2 +- Dockerfile.alltools | 2 +- build/ci.go | 33 ++++++++++++++++++++++----------- internal/build/util.go | 19 +++++++++++++++++-- 4 files changed, 41 insertions(+), 15 deletions(-) diff --git a/Dockerfile b/Dockerfile index 70299190f..143c92f27 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,7 @@ COPY go.sum /go-ethereum/ RUN cd /go-ethereum && go mod download ADD . /go-ethereum -RUN cd /go-ethereum && go run build/ci.go install ./cmd/geth +RUN cd /go-ethereum && go run build/ci.go install -static ./cmd/geth # Pull Geth into a second stage deploy alpine container FROM alpine:latest diff --git a/Dockerfile.alltools b/Dockerfile.alltools index b11492cab..176c45922 100644 --- a/Dockerfile.alltools +++ b/Dockerfile.alltools @@ -14,7 +14,7 @@ COPY go.sum /go-ethereum/ RUN cd /go-ethereum && go mod download ADD . /go-ethereum -RUN cd /go-ethereum && go run build/ci.go install +RUN cd /go-ethereum && go run build/ci.go install -static # Pull all binaries into a second stage deploy alpine container FROM alpine:latest diff --git a/build/ci.go b/build/ci.go index 526e76f38..f91de328b 100644 --- a/build/ci.go +++ b/build/ci.go @@ -200,9 +200,10 @@ func main() { func doInstall(cmdline []string) { var ( - dlgo = flag.Bool("dlgo", false, "Download Go and build with it") - arch = flag.String("arch", "", "Architecture to cross build for") - cc = flag.String("cc", "", "C compiler to cross build with") + dlgo = flag.Bool("dlgo", false, "Download Go and build with it") + arch = flag.String("arch", "", "Architecture to cross build for") + cc = flag.String("cc", "", "C compiler to cross build with") + staticlink = flag.Bool("static", false, "Create statically-linked executable") ) flag.CommandLine.Parse(cmdline) @@ -213,9 +214,12 @@ func doInstall(cmdline []string) { tc.Root = build.DownloadGo(csdb, dlgoVersion) } + // Disable CLI markdown doc generation in release builds. + buildTags := []string{"urfave_cli_no_docs"} + // Configure the build. env := build.Env() - gobuild := tc.Go("build", buildFlags(env)...) + gobuild := tc.Go("build", buildFlags(env, *staticlink, buildTags)...) // arm64 CI builders are memory-constrained and can't handle concurrent builds, // better disable it. This check isn't the best, it should probably @@ -224,9 +228,6 @@ func doInstall(cmdline []string) { gobuild.Args = append(gobuild.Args, "-p", "1") } - // Disable CLI markdown doc generation in release builds. - gobuild.Args = append(gobuild.Args, "-tags", "urfave_cli_no_docs") - // We use -trimpath to avoid leaking local paths into the built executables. gobuild.Args = append(gobuild.Args, "-trimpath") @@ -251,7 +252,7 @@ func doInstall(cmdline []string) { } // buildFlags returns the go tool flags for building. -func buildFlags(env build.Environment) (flags []string) { +func buildFlags(env build.Environment, staticLinking bool, buildTags []string) (flags []string) { var ld []string if env.Commit != "" { ld = append(ld, "-X", "main.gitCommit="+env.Commit) @@ -262,14 +263,24 @@ func buildFlags(env build.Environment) (flags []string) { if runtime.GOOS == "darwin" { ld = append(ld, "-s") } - // Enforce the stacksize to 8M, which is the case on most platforms apart from - // alpine Linux. if runtime.GOOS == "linux" { - ld = append(ld, "-extldflags", "-Wl,-z,stack-size=0x800000") + // Enforce the stacksize to 8M, which is the case on most platforms apart from + // alpine Linux. + extld := []string{"-Wl,-z,stack-size=0x800000"} + if staticLinking { + extld = append(extld, "-static") + // Under static linking, use of certain glibc features must be + // disabled to avoid shared library dependencies. + buildTags = append(buildTags, "osusergo", "netgo") + } + ld = append(ld, "-extldflags", "'"+strings.Join(extld, " ")+"'") } if len(ld) > 0 { flags = append(flags, "-ldflags", strings.Join(ld, " ")) } + if len(buildTags) > 0 { + flags = append(flags, "-tags", strings.Join(buildTags, ",")) + } return flags } diff --git a/internal/build/util.go b/internal/build/util.go index 654349fac..9a721e9b8 100644 --- a/internal/build/util.go +++ b/internal/build/util.go @@ -29,6 +29,7 @@ import ( "os/exec" "path" "path/filepath" + "strconv" "strings" "text/template" "time" @@ -39,7 +40,7 @@ var DryRunFlag = flag.Bool("n", false, "dry run, don't execute commands") // MustRun executes the given command and exits the host process for // any error. func MustRun(cmd *exec.Cmd) { - fmt.Println(">>>", strings.Join(cmd.Args, " ")) + fmt.Println(">>>", printArgs(cmd.Args)) if !*DryRunFlag { cmd.Stderr = os.Stderr cmd.Stdout = os.Stdout @@ -49,6 +50,20 @@ func MustRun(cmd *exec.Cmd) { } } +func printArgs(args []string) string { + var s strings.Builder + for i, arg := range args { + if i > 0 { + s.WriteByte(' ') + } + if strings.IndexByte(arg, ' ') >= 0 { + arg = strconv.QuoteToASCII(arg) + } + s.WriteString(arg) + } + return s.String() +} + func MustRunCommand(cmd string, args ...string) { MustRun(exec.Command(cmd, args...)) } @@ -121,7 +136,7 @@ func UploadSFTP(identityFile, host, dir string, files []string) error { sftp.Args = append(sftp.Args, "-i", identityFile) } sftp.Args = append(sftp.Args, host) - fmt.Println(">>>", strings.Join(sftp.Args, " ")) + fmt.Println(">>>", printArgs(sftp.Args)) if *DryRunFlag { return nil } From 366d2169fbc0e0f803b68c042b77b6b480836dbc Mon Sep 17 00:00:00 2001 From: Darioush Jalali Date: Thu, 11 Aug 2022 00:25:54 -0700 Subject: [PATCH 35/59] accounts/abi: display name in "method/event not found" error (#25512) --- accounts/abi/abi.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/accounts/abi/abi.go b/accounts/abi/abi.go index ed5b6e92e..81bbee2f2 100644 --- a/accounts/abi/abi.go +++ b/accounts/abi/abi.go @@ -95,7 +95,7 @@ func (abi ABI) getArguments(name string, data []byte) (Arguments, error) { args = event.Inputs } if args == nil { - return nil, errors.New("abi: could not locate named method or event") + return nil, fmt.Errorf("abi: could not locate named method or event: %s", name) } return args, nil } From 0be9d76e3702cfae92f042cf6a113e22479de3f5 Mon Sep 17 00:00:00 2001 From: lightclient <14004106+lightclient@users.noreply.github.com> Date: Thu, 11 Aug 2022 02:56:53 -0600 Subject: [PATCH 36/59] internal/ethapi: rework setDefaults for tx args so fee logic is separate (#25197) Co-authored-by: bobpkr --- internal/ethapi/transaction_args.go | 118 ++++---- internal/ethapi/transaction_args_test.go | 342 +++++++++++++++++++++++ 2 files changed, 410 insertions(+), 50 deletions(-) create mode 100644 internal/ethapi/transaction_args_test.go diff --git a/internal/ethapi/transaction_args.go b/internal/ethapi/transaction_args.go index cb2782ca0..787ac6577 100644 --- a/internal/ethapi/transaction_args.go +++ b/internal/ethapi/transaction_args.go @@ -75,56 +75,8 @@ func (args *TransactionArgs) data() []byte { // setDefaults fills in default values for unspecified tx fields. func (args *TransactionArgs) setDefaults(ctx context.Context, b Backend) error { - if args.GasPrice != nil && (args.MaxFeePerGas != nil || args.MaxPriorityFeePerGas != nil) { - return errors.New("both gasPrice and (maxFeePerGas or maxPriorityFeePerGas) specified") - } - // After london, default to 1559 unless gasPrice is set - head := b.CurrentHeader() - // If user specifies both maxPriorityfee and maxFee, then we do not - // need to consult the chain for defaults. It's definitely a London tx. - if args.MaxPriorityFeePerGas == nil || args.MaxFeePerGas == nil { - // In this clause, user left some fields unspecified. - if b.ChainConfig().IsLondon(head.Number) && args.GasPrice == nil { - if args.MaxPriorityFeePerGas == nil { - tip, err := b.SuggestGasTipCap(ctx) - if err != nil { - return err - } - args.MaxPriorityFeePerGas = (*hexutil.Big)(tip) - } - if args.MaxFeePerGas == nil { - gasFeeCap := new(big.Int).Add( - (*big.Int)(args.MaxPriorityFeePerGas), - new(big.Int).Mul(head.BaseFee, big.NewInt(2)), - ) - args.MaxFeePerGas = (*hexutil.Big)(gasFeeCap) - } - if args.MaxFeePerGas.ToInt().Cmp(args.MaxPriorityFeePerGas.ToInt()) < 0 { - return fmt.Errorf("maxFeePerGas (%v) < maxPriorityFeePerGas (%v)", args.MaxFeePerGas, args.MaxPriorityFeePerGas) - } - } else { - if args.MaxFeePerGas != nil || args.MaxPriorityFeePerGas != nil { - return errors.New("maxFeePerGas or maxPriorityFeePerGas specified but london is not active yet") - } - if args.GasPrice == nil { - price, err := b.SuggestGasTipCap(ctx) - if err != nil { - return err - } - if b.ChainConfig().IsLondon(head.Number) { - // The legacy tx gas price suggestion should not add 2x base fee - // because all fees are consumed, so it would result in a spiral - // upwards. - price.Add(price, head.BaseFee) - } - args.GasPrice = (*hexutil.Big)(price) - } - } - } else { - // Both maxPriorityfee and maxFee set by caller. Sanity-check their internal relation - if args.MaxFeePerGas.ToInt().Cmp(args.MaxPriorityFeePerGas.ToInt()) < 0 { - return fmt.Errorf("maxFeePerGas (%v) < maxPriorityFeePerGas (%v)", args.MaxFeePerGas, args.MaxPriorityFeePerGas) - } + if err := args.setFeeDefaults(ctx, b); err != nil { + return err } if args.Value == nil { args.Value = new(hexutil.Big) @@ -178,6 +130,72 @@ func (args *TransactionArgs) setDefaults(ctx context.Context, b Backend) error { return nil } +// setFeeDefaults fills in default fee values for unspecified tx fields. +func (args *TransactionArgs) setFeeDefaults(ctx context.Context, b Backend) error { + // If both gasPrice and at least one of the EIP-1559 fee parameters are specified, error. + if args.GasPrice != nil && (args.MaxFeePerGas != nil || args.MaxPriorityFeePerGas != nil) { + return errors.New("both gasPrice and (maxFeePerGas or maxPriorityFeePerGas) specified") + } + // If the tx has completely specified a fee mechanism, no default is needed. This allows users + // who are not yet synced past London to get defaults for other tx values. See + // https://github.com/ethereum/go-ethereum/pull/23274 for more information. + eip1559ParamsSet := args.MaxFeePerGas != nil && args.MaxPriorityFeePerGas != nil + if (args.GasPrice != nil && !eip1559ParamsSet) || (args.GasPrice == nil && eip1559ParamsSet) { + // Sanity check the EIP-1559 fee parameters if present. + if args.GasPrice == nil && args.MaxFeePerGas.ToInt().Cmp(args.MaxPriorityFeePerGas.ToInt()) < 0 { + return fmt.Errorf("maxFeePerGas (%v) < maxPriorityFeePerGas (%v)", args.MaxFeePerGas, args.MaxPriorityFeePerGas) + } + return nil + } + // Now attempt to fill in default value depending on whether London is active or not. + head := b.CurrentHeader() + if b.ChainConfig().IsLondon(head.Number) { + // London is active, set maxPriorityFeePerGas and maxFeePerGas. + if err := args.setLondonFeeDefaults(ctx, head, b); err != nil { + return err + } + } else { + if args.MaxFeePerGas != nil || args.MaxPriorityFeePerGas != nil { + return fmt.Errorf("maxFeePerGas and maxPriorityFeePerGas are not valid before London is active") + } + // London not active, set gas price. + price, err := b.SuggestGasTipCap(ctx) + if err != nil { + return err + } + args.GasPrice = (*hexutil.Big)(price) + } + return nil +} + +// setLondonFeeDefaults fills in reasonable default fee values for unspecified fields. +func (args *TransactionArgs) setLondonFeeDefaults(ctx context.Context, head *types.Header, b Backend) error { + // Set maxPriorityFeePerGas if it is missing. + if args.MaxPriorityFeePerGas == nil { + tip, err := b.SuggestGasTipCap(ctx) + if err != nil { + return err + } + args.MaxPriorityFeePerGas = (*hexutil.Big)(tip) + } + // Set maxFeePerGas if it is missing. + if args.MaxFeePerGas == nil { + // Set the max fee to be 2 times larger than the previous block's base fee. + // The additional slack allows the tx to not become invalidated if the base + // fee is rising. + val := new(big.Int).Add( + args.MaxPriorityFeePerGas.ToInt(), + new(big.Int).Mul(head.BaseFee, big.NewInt(2)), + ) + args.MaxFeePerGas = (*hexutil.Big)(val) + } + // Both EIP-1559 fee parameters are now set; sanity check them. + if args.MaxFeePerGas.ToInt().Cmp(args.MaxPriorityFeePerGas.ToInt()) < 0 { + return fmt.Errorf("maxFeePerGas (%v) < maxPriorityFeePerGas (%v)", args.MaxFeePerGas, args.MaxPriorityFeePerGas) + } + return nil +} + // ToMessage converts the transaction arguments to the Message type used by the // core evm. This method is used in calls and traces that do not require a real // live transaction. diff --git a/internal/ethapi/transaction_args_test.go b/internal/ethapi/transaction_args_test.go new file mode 100644 index 000000000..92f009aa8 --- /dev/null +++ b/internal/ethapi/transaction_args_test.go @@ -0,0 +1,342 @@ +// Copyright 2022 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package ethapi + +import ( + "context" + "fmt" + "math/big" + "reflect" + "testing" + "time" + + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/accounts" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/consensus" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/bloombits" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/rpc" +) + +// TestSetFeeDefaults tests the logic for filling in default fee values works as expected. +func TestSetFeeDefaults(t *testing.T) { + type test struct { + name string + isLondon bool + in *TransactionArgs + want *TransactionArgs + err error + } + + var ( + b = newBackendMock() + fortytwo = (*hexutil.Big)(big.NewInt(42)) + maxFee = (*hexutil.Big)(new(big.Int).Add(new(big.Int).Mul(b.current.BaseFee, big.NewInt(2)), fortytwo.ToInt())) + al = &types.AccessList{types.AccessTuple{Address: common.Address{0xaa}, StorageKeys: []common.Hash{{0x01}}}} + ) + + tests := []test{ + // Legacy txs + { + "legacy tx pre-London", + false, + &TransactionArgs{}, + &TransactionArgs{GasPrice: fortytwo}, + nil, + }, + { + "legacy tx post-London, explicit gas price", + true, + &TransactionArgs{GasPrice: fortytwo}, + &TransactionArgs{GasPrice: fortytwo}, + nil, + }, + + // Access list txs + { + "access list tx pre-London", + false, + &TransactionArgs{AccessList: al}, + &TransactionArgs{AccessList: al, GasPrice: fortytwo}, + nil, + }, + { + "access list tx post-London, explicit gas price", + false, + &TransactionArgs{AccessList: al, GasPrice: fortytwo}, + &TransactionArgs{AccessList: al, GasPrice: fortytwo}, + nil, + }, + { + "access list tx post-London", + true, + &TransactionArgs{AccessList: al}, + &TransactionArgs{AccessList: al, MaxFeePerGas: maxFee, MaxPriorityFeePerGas: fortytwo}, + nil, + }, + { + "access list tx post-London, only max fee", + true, + &TransactionArgs{AccessList: al, MaxFeePerGas: maxFee}, + &TransactionArgs{AccessList: al, MaxFeePerGas: maxFee, MaxPriorityFeePerGas: fortytwo}, + nil, + }, + { + "access list tx post-London, only priority fee", + true, + &TransactionArgs{AccessList: al, MaxFeePerGas: maxFee}, + &TransactionArgs{AccessList: al, MaxFeePerGas: maxFee, MaxPriorityFeePerGas: fortytwo}, + nil, + }, + + // Dynamic fee txs + { + "dynamic tx post-London", + true, + &TransactionArgs{}, + &TransactionArgs{MaxFeePerGas: maxFee, MaxPriorityFeePerGas: fortytwo}, + nil, + }, + { + "dynamic tx post-London, only max fee", + true, + &TransactionArgs{MaxFeePerGas: maxFee}, + &TransactionArgs{MaxFeePerGas: maxFee, MaxPriorityFeePerGas: fortytwo}, + nil, + }, + { + "dynamic tx post-London, only priority fee", + true, + &TransactionArgs{MaxFeePerGas: maxFee}, + &TransactionArgs{MaxFeePerGas: maxFee, MaxPriorityFeePerGas: fortytwo}, + nil, + }, + { + "dynamic fee tx pre-London, maxFee set", + false, + &TransactionArgs{MaxFeePerGas: maxFee}, + nil, + fmt.Errorf("maxFeePerGas and maxPriorityFeePerGas are not valid before London is active"), + }, + { + "dynamic fee tx pre-London, priorityFee set", + false, + &TransactionArgs{MaxPriorityFeePerGas: fortytwo}, + nil, + fmt.Errorf("maxFeePerGas and maxPriorityFeePerGas are not valid before London is active"), + }, + { + "dynamic fee tx, maxFee < priorityFee", + true, + &TransactionArgs{MaxFeePerGas: maxFee, MaxPriorityFeePerGas: (*hexutil.Big)(big.NewInt(1000))}, + nil, + fmt.Errorf("maxFeePerGas (0x3e) < maxPriorityFeePerGas (0x3e8)"), + }, + { + "dynamic fee tx, maxFee < priorityFee while setting default", + true, + &TransactionArgs{MaxFeePerGas: (*hexutil.Big)(big.NewInt(7))}, + nil, + fmt.Errorf("maxFeePerGas (0x7) < maxPriorityFeePerGas (0x2a)"), + }, + + // Misc + { + "set all fee parameters", + false, + &TransactionArgs{GasPrice: fortytwo, MaxFeePerGas: maxFee, MaxPriorityFeePerGas: fortytwo}, + nil, + fmt.Errorf("both gasPrice and (maxFeePerGas or maxPriorityFeePerGas) specified"), + }, + { + "set gas price and maxPriorityFee", + false, + &TransactionArgs{GasPrice: fortytwo, MaxPriorityFeePerGas: fortytwo}, + nil, + fmt.Errorf("both gasPrice and (maxFeePerGas or maxPriorityFeePerGas) specified"), + }, + { + "set gas price and maxFee", + true, + &TransactionArgs{GasPrice: fortytwo, MaxFeePerGas: maxFee}, + nil, + fmt.Errorf("both gasPrice and (maxFeePerGas or maxPriorityFeePerGas) specified"), + }, + } + + ctx := context.Background() + for i, test := range tests { + if test.isLondon { + b.activateLondon() + } else { + b.deactivateLondon() + } + got := test.in + err := got.setFeeDefaults(ctx, b) + if err != nil && err.Error() == test.err.Error() { + // Test threw expected error. + continue + } else if err != nil { + t.Fatalf("test %d (%s): unexpected error: %s", i, test.name, err) + } + if !reflect.DeepEqual(got, test.want) { + t.Fatalf("test %d (%s): did not fill defaults as expected: (got: %v, want: %v)", i, test.name, got, test.want) + } + } +} + +type backendMock struct { + current *types.Header + config *params.ChainConfig +} + +func newBackendMock() *backendMock { + config := ¶ms.ChainConfig{ + ChainID: big.NewInt(42), + HomesteadBlock: big.NewInt(0), + DAOForkBlock: nil, + DAOForkSupport: true, + EIP150Block: big.NewInt(0), + EIP155Block: big.NewInt(0), + EIP158Block: big.NewInt(0), + ByzantiumBlock: big.NewInt(0), + ConstantinopleBlock: big.NewInt(0), + PetersburgBlock: big.NewInt(0), + IstanbulBlock: big.NewInt(0), + MuirGlacierBlock: big.NewInt(0), + BerlinBlock: big.NewInt(0), + LondonBlock: big.NewInt(1000), + } + return &backendMock{ + current: &types.Header{ + Difficulty: big.NewInt(10000000000), + Number: big.NewInt(1100), + GasLimit: 8_000_000, + GasUsed: 8_000_000, + Time: 555, + Extra: make([]byte, 32), + BaseFee: big.NewInt(10), + }, + config: config, + } +} + +func (b *backendMock) activateLondon() { + b.current.Number = big.NewInt(1100) +} + +func (b *backendMock) deactivateLondon() { + b.current.Number = big.NewInt(900) +} +func (b *backendMock) SuggestGasTipCap(ctx context.Context) (*big.Int, error) { + return big.NewInt(42), nil +} +func (b *backendMock) CurrentHeader() *types.Header { return b.current } +func (b *backendMock) ChainConfig() *params.ChainConfig { return b.config } + +// Other methods needed to implement Backend interface. +func (b *backendMock) SyncProgress() ethereum.SyncProgress { return ethereum.SyncProgress{} } +func (b *backendMock) FeeHistory(ctx context.Context, blockCount int, lastBlock rpc.BlockNumber, rewardPercentiles []float64) (*big.Int, [][]*big.Int, []*big.Int, []float64, error) { + return nil, nil, nil, nil, nil +} +func (b *backendMock) ChainDb() ethdb.Database { return nil } +func (b *backendMock) AccountManager() *accounts.Manager { return nil } +func (b *backendMock) ExtRPCEnabled() bool { return false } +func (b *backendMock) RPCGasCap() uint64 { return 0 } +func (b *backendMock) RPCEVMTimeout() time.Duration { return time.Second } +func (b *backendMock) RPCTxFeeCap() float64 { return 0 } +func (b *backendMock) UnprotectedAllowed() bool { return false } +func (b *backendMock) SetHead(number uint64) {} +func (b *backendMock) HeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Header, error) { + return nil, nil +} +func (b *backendMock) HeaderByHash(ctx context.Context, hash common.Hash) (*types.Header, error) { + return nil, nil +} +func (b *backendMock) HeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Header, error) { + return nil, nil +} +func (b *backendMock) CurrentBlock() *types.Block { return nil } +func (b *backendMock) BlockByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Block, error) { + return nil, nil +} +func (b *backendMock) BlockByHash(ctx context.Context, hash common.Hash) (*types.Block, error) { + return nil, nil +} +func (b *backendMock) BlockByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*types.Block, error) { + return nil, nil +} +func (b *backendMock) StateAndHeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*state.StateDB, *types.Header, error) { + return nil, nil, nil +} +func (b *backendMock) StateAndHeaderByNumberOrHash(ctx context.Context, blockNrOrHash rpc.BlockNumberOrHash) (*state.StateDB, *types.Header, error) { + return nil, nil, nil +} +func (b *backendMock) PendingBlockAndReceipts() (*types.Block, types.Receipts) { return nil, nil } +func (b *backendMock) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) { + return nil, nil +} +func (b *backendMock) GetTd(ctx context.Context, hash common.Hash) *big.Int { return nil } +func (b *backendMock) GetEVM(ctx context.Context, msg core.Message, state *state.StateDB, header *types.Header, vmConfig *vm.Config) (*vm.EVM, func() error, error) { + return nil, nil, nil +} +func (b *backendMock) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription { return nil } +func (b *backendMock) SubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription { + return nil +} +func (b *backendMock) SubscribeChainSideEvent(ch chan<- core.ChainSideEvent) event.Subscription { + return nil +} +func (b *backendMock) SendTx(ctx context.Context, signedTx *types.Transaction) error { return nil } +func (b *backendMock) GetTransaction(ctx context.Context, txHash common.Hash) (*types.Transaction, common.Hash, uint64, uint64, error) { + return nil, [32]byte{}, 0, 0, nil +} +func (b *backendMock) GetPoolTransactions() (types.Transactions, error) { return nil, nil } +func (b *backendMock) GetPoolTransaction(txHash common.Hash) *types.Transaction { return nil } +func (b *backendMock) GetPoolNonce(ctx context.Context, addr common.Address) (uint64, error) { + return 0, nil +} +func (b *backendMock) Stats() (pending int, queued int) { return 0, 0 } +func (b *backendMock) TxPoolContent() (map[common.Address]types.Transactions, map[common.Address]types.Transactions) { + return nil, nil +} +func (b *backendMock) TxPoolContentFrom(addr common.Address) (types.Transactions, types.Transactions) { + return nil, nil +} +func (b *backendMock) SubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription { return nil } +func (b *backendMock) BloomStatus() (uint64, uint64) { return 0, 0 } +func (b *backendMock) GetLogs(ctx context.Context, blockHash common.Hash) ([][]*types.Log, error) { + return nil, nil +} +func (b *backendMock) ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) {} +func (b *backendMock) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription { return nil } +func (b *backendMock) SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription { + return nil +} +func (b *backendMock) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription { + return nil +} + +func (b *backendMock) Engine() consensus.Engine { return nil } From 141cd425310b503c5678e674a8c3872cf46b7086 Mon Sep 17 00:00:00 2001 From: ycyraum Date: Fri, 12 Aug 2022 13:58:06 +0200 Subject: [PATCH 37/59] core/genesis: remove calaverasAllocData (#25516) core/genesis: calaverasAllocData no longer used --- core/genesis_alloc.go | 1 - 1 file changed, 1 deletion(-) diff --git a/core/genesis_alloc.go b/core/genesis_alloc.go index 041c55424..16df39057 100644 --- a/core/genesis_alloc.go +++ b/core/genesis_alloc.go @@ -25,7 +25,6 @@ const mainnetAllocData = "\xfa\x04]X\u0793\r\x83b\x011\x8e\u0189\x9agT\x06\x908' const ropstenAllocData = "\xf9\x03\xa4\u0080\x01\xc2\x01\x01\xc2\x02\x01\xc2\x03\x01\xc2\x04\x01\xc2\x05\x01\xc2\x06\x01\xc2\a\x01\xc2\b\x01\xc2\t\x01\xc2\n\x80\xc2\v\x80\xc2\f\x80\xc2\r\x80\xc2\x0e\x80\xc2\x0f\x80\xc2\x10\x80\xc2\x11\x80\xc2\x12\x80\xc2\x13\x80\xc2\x14\x80\xc2\x15\x80\xc2\x16\x80\xc2\x17\x80\xc2\x18\x80\xc2\x19\x80\xc2\x1a\x80\xc2\x1b\x80\xc2\x1c\x80\xc2\x1d\x80\xc2\x1e\x80\xc2\x1f\x80\xc2 \x80\xc2!\x80\xc2\"\x80\xc2#\x80\xc2$\x80\xc2%\x80\xc2&\x80\xc2'\x80\xc2(\x80\xc2)\x80\xc2*\x80\xc2+\x80\xc2,\x80\xc2-\x80\xc2.\x80\xc2/\x80\xc20\x80\xc21\x80\xc22\x80\xc23\x80\xc24\x80\xc25\x80\xc26\x80\xc27\x80\xc28\x80\xc29\x80\xc2:\x80\xc2;\x80\xc2<\x80\xc2=\x80\xc2>\x80\xc2?\x80\xc2@\x80\xc2A\x80\xc2B\x80\xc2C\x80\xc2D\x80\xc2E\x80\xc2F\x80\xc2G\x80\xc2H\x80\xc2I\x80\xc2J\x80\xc2K\x80\xc2L\x80\xc2M\x80\xc2N\x80\xc2O\x80\xc2P\x80\xc2Q\x80\xc2R\x80\xc2S\x80\xc2T\x80\xc2U\x80\xc2V\x80\xc2W\x80\xc2X\x80\xc2Y\x80\xc2Z\x80\xc2[\x80\xc2\\\x80\xc2]\x80\xc2^\x80\xc2_\x80\xc2`\x80\xc2a\x80\xc2b\x80\xc2c\x80\xc2d\x80\xc2e\x80\xc2f\x80\xc2g\x80\xc2h\x80\xc2i\x80\xc2j\x80\xc2k\x80\xc2l\x80\xc2m\x80\xc2n\x80\xc2o\x80\xc2p\x80\xc2q\x80\xc2r\x80\xc2s\x80\xc2t\x80\xc2u\x80\xc2v\x80\xc2w\x80\xc2x\x80\xc2y\x80\xc2z\x80\xc2{\x80\xc2|\x80\xc2}\x80\xc2~\x80\xc2\u007f\x80\u00c1\x80\x80\u00c1\x81\x80\u00c1\x82\x80\u00c1\x83\x80\u00c1\x84\x80\u00c1\x85\x80\u00c1\x86\x80\u00c1\x87\x80\u00c1\x88\x80\u00c1\x89\x80\u00c1\x8a\x80\u00c1\x8b\x80\u00c1\x8c\x80\u00c1\x8d\x80\u00c1\x8e\x80\u00c1\x8f\x80\u00c1\x90\x80\u00c1\x91\x80\u00c1\x92\x80\u00c1\x93\x80\u00c1\x94\x80\u00c1\x95\x80\u00c1\x96\x80\u00c1\x97\x80\u00c1\x98\x80\u00c1\x99\x80\u00c1\x9a\x80\u00c1\x9b\x80\u00c1\x9c\x80\u00c1\x9d\x80\u00c1\x9e\x80\u00c1\x9f\x80\u00c1\xa0\x80\u00c1\xa1\x80\u00c1\xa2\x80\u00c1\xa3\x80\u00c1\xa4\x80\u00c1\xa5\x80\u00c1\xa6\x80\u00c1\xa7\x80\u00c1\xa8\x80\u00c1\xa9\x80\u00c1\xaa\x80\u00c1\xab\x80\u00c1\xac\x80\u00c1\xad\x80\u00c1\xae\x80\u00c1\xaf\x80\u00c1\xb0\x80\u00c1\xb1\x80\u00c1\xb2\x80\u00c1\xb3\x80\u00c1\xb4\x80\u00c1\xb5\x80\u00c1\xb6\x80\u00c1\xb7\x80\u00c1\xb8\x80\u00c1\xb9\x80\u00c1\xba\x80\u00c1\xbb\x80\u00c1\xbc\x80\u00c1\xbd\x80\u00c1\xbe\x80\u00c1\xbf\x80\u00c1\xc0\x80\u00c1\xc1\x80\u00c1\u0080\u00c1\u00c0\u00c1\u0100\u00c1\u0140\u00c1\u0180\u00c1\u01c0\u00c1\u0200\u00c1\u0240\u00c1\u0280\u00c1\u02c0\u00c1\u0300\u00c1\u0340\u00c1\u0380\u00c1\u03c0\u00c1\u0400\u00c1\u0440\u00c1\u0480\u00c1\u04c0\u00c1\u0500\u00c1\u0540\u00c1\u0580\u00c1\u05c0\u00c1\u0600\u00c1\u0640\u00c1\u0680\u00c1\u06c0\u00c1\u0700\u00c1\u0740\u00c1\u0780\u00c1\u07c0\u00c1\xe0\x80\u00c1\xe1\x80\u00c1\xe2\x80\u00c1\xe3\x80\u00c1\xe4\x80\u00c1\xe5\x80\u00c1\xe6\x80\u00c1\xe7\x80\u00c1\xe8\x80\u00c1\xe9\x80\u00c1\xea\x80\u00c1\xeb\x80\u00c1\xec\x80\u00c1\xed\x80\u00c1\xee\x80\u00c1\xef\x80\u00c1\xf0\x80\u00c1\xf1\x80\u00c1\xf2\x80\u00c1\xf3\x80\u00c1\xf4\x80\u00c1\xf5\x80\u00c1\xf6\x80\u00c1\xf7\x80\u00c1\xf8\x80\u00c1\xf9\x80\u00c1\xfa\x80\u00c1\xfb\x80\u00c1\xfc\x80\u00c1\xfd\x80\u00c1\xfe\x80\u00c1\xff\x80\u3507KT\xa8\xbd\x15)f\xd6?pk\xae\x1f\xfe\xb0A\x19!\xe5\x8d\f\x9f,\x9c\xd0Ft\xed\xea@\x00\x00\x00" const rinkebyAllocData = "\xf9\x03\xb7\u0080\x01\xc2\x01\x01\xc2\x02\x01\xc2\x03\x01\xc2\x04\x01\xc2\x05\x01\xc2\x06\x01\xc2\a\x01\xc2\b\x01\xc2\t\x01\xc2\n\x01\xc2\v\x01\xc2\f\x01\xc2\r\x01\xc2\x0e\x01\xc2\x0f\x01\xc2\x10\x01\xc2\x11\x01\xc2\x12\x01\xc2\x13\x01\xc2\x14\x01\xc2\x15\x01\xc2\x16\x01\xc2\x17\x01\xc2\x18\x01\xc2\x19\x01\xc2\x1a\x01\xc2\x1b\x01\xc2\x1c\x01\xc2\x1d\x01\xc2\x1e\x01\xc2\x1f\x01\xc2 \x01\xc2!\x01\xc2\"\x01\xc2#\x01\xc2$\x01\xc2%\x01\xc2&\x01\xc2'\x01\xc2(\x01\xc2)\x01\xc2*\x01\xc2+\x01\xc2,\x01\xc2-\x01\xc2.\x01\xc2/\x01\xc20\x01\xc21\x01\xc22\x01\xc23\x01\xc24\x01\xc25\x01\xc26\x01\xc27\x01\xc28\x01\xc29\x01\xc2:\x01\xc2;\x01\xc2<\x01\xc2=\x01\xc2>\x01\xc2?\x01\xc2@\x01\xc2A\x01\xc2B\x01\xc2C\x01\xc2D\x01\xc2E\x01\xc2F\x01\xc2G\x01\xc2H\x01\xc2I\x01\xc2J\x01\xc2K\x01\xc2L\x01\xc2M\x01\xc2N\x01\xc2O\x01\xc2P\x01\xc2Q\x01\xc2R\x01\xc2S\x01\xc2T\x01\xc2U\x01\xc2V\x01\xc2W\x01\xc2X\x01\xc2Y\x01\xc2Z\x01\xc2[\x01\xc2\\\x01\xc2]\x01\xc2^\x01\xc2_\x01\xc2`\x01\xc2a\x01\xc2b\x01\xc2c\x01\xc2d\x01\xc2e\x01\xc2f\x01\xc2g\x01\xc2h\x01\xc2i\x01\xc2j\x01\xc2k\x01\xc2l\x01\xc2m\x01\xc2n\x01\xc2o\x01\xc2p\x01\xc2q\x01\xc2r\x01\xc2s\x01\xc2t\x01\xc2u\x01\xc2v\x01\xc2w\x01\xc2x\x01\xc2y\x01\xc2z\x01\xc2{\x01\xc2|\x01\xc2}\x01\xc2~\x01\xc2\u007f\x01\u00c1\x80\x01\u00c1\x81\x01\u00c1\x82\x01\u00c1\x83\x01\u00c1\x84\x01\u00c1\x85\x01\u00c1\x86\x01\u00c1\x87\x01\u00c1\x88\x01\u00c1\x89\x01\u00c1\x8a\x01\u00c1\x8b\x01\u00c1\x8c\x01\u00c1\x8d\x01\u00c1\x8e\x01\u00c1\x8f\x01\u00c1\x90\x01\u00c1\x91\x01\u00c1\x92\x01\u00c1\x93\x01\u00c1\x94\x01\u00c1\x95\x01\u00c1\x96\x01\u00c1\x97\x01\u00c1\x98\x01\u00c1\x99\x01\u00c1\x9a\x01\u00c1\x9b\x01\u00c1\x9c\x01\u00c1\x9d\x01\u00c1\x9e\x01\u00c1\x9f\x01\u00c1\xa0\x01\u00c1\xa1\x01\u00c1\xa2\x01\u00c1\xa3\x01\u00c1\xa4\x01\u00c1\xa5\x01\u00c1\xa6\x01\u00c1\xa7\x01\u00c1\xa8\x01\u00c1\xa9\x01\u00c1\xaa\x01\u00c1\xab\x01\u00c1\xac\x01\u00c1\xad\x01\u00c1\xae\x01\u00c1\xaf\x01\u00c1\xb0\x01\u00c1\xb1\x01\u00c1\xb2\x01\u00c1\xb3\x01\u00c1\xb4\x01\u00c1\xb5\x01\u00c1\xb6\x01\u00c1\xb7\x01\u00c1\xb8\x01\u00c1\xb9\x01\u00c1\xba\x01\u00c1\xbb\x01\u00c1\xbc\x01\u00c1\xbd\x01\u00c1\xbe\x01\u00c1\xbf\x01\u00c1\xc0\x01\u00c1\xc1\x01\u00c1\xc2\x01\u00c1\xc3\x01\u00c1\xc4\x01\u00c1\xc5\x01\u00c1\xc6\x01\u00c1\xc7\x01\u00c1\xc8\x01\u00c1\xc9\x01\u00c1\xca\x01\u00c1\xcb\x01\u00c1\xcc\x01\u00c1\xcd\x01\u00c1\xce\x01\u00c1\xcf\x01\u00c1\xd0\x01\u00c1\xd1\x01\u00c1\xd2\x01\u00c1\xd3\x01\u00c1\xd4\x01\u00c1\xd5\x01\u00c1\xd6\x01\u00c1\xd7\x01\u00c1\xd8\x01\u00c1\xd9\x01\u00c1\xda\x01\u00c1\xdb\x01\u00c1\xdc\x01\u00c1\xdd\x01\u00c1\xde\x01\u00c1\xdf\x01\u00c1\xe0\x01\u00c1\xe1\x01\u00c1\xe2\x01\u00c1\xe3\x01\u00c1\xe4\x01\u00c1\xe5\x01\u00c1\xe6\x01\u00c1\xe7\x01\u00c1\xe8\x01\u00c1\xe9\x01\u00c1\xea\x01\u00c1\xeb\x01\u00c1\xec\x01\u00c1\xed\x01\u00c1\xee\x01\u00c1\xef\x01\u00c1\xf0\x01\u00c1\xf1\x01\u00c1\xf2\x01\u00c1\xf3\x01\u00c1\xf4\x01\u00c1\xf5\x01\u00c1\xf6\x01\u00c1\xf7\x01\u00c1\xf8\x01\u00c1\xf9\x01\u00c1\xfa\x01\u00c1\xfb\x01\u00c1\xfc\x01\u00c1\xfd\x01\u00c1\xfe\x01\u00c1\xff\x01\xf6\x941\xb9\x8d\x14\x00{\xde\xe67)\x80\x86\x98\x8a\v\xbd1\x18E#\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" const goerliAllocData = "\xf9\x04\x06\u0080\x01\xc2\x01\x01\xc2\x02\x01\xc2\x03\x01\xc2\x04\x01\xc2\x05\x01\xc2\x06\x01\xc2\a\x01\xc2\b\x01\xc2\t\x01\xc2\n\x01\xc2\v\x01\xc2\f\x01\xc2\r\x01\xc2\x0e\x01\xc2\x0f\x01\xc2\x10\x01\xc2\x11\x01\xc2\x12\x01\xc2\x13\x01\xc2\x14\x01\xc2\x15\x01\xc2\x16\x01\xc2\x17\x01\xc2\x18\x01\xc2\x19\x01\xc2\x1a\x01\xc2\x1b\x01\xc2\x1c\x01\xc2\x1d\x01\xc2\x1e\x01\xc2\x1f\x01\xc2 \x01\xc2!\x01\xc2\"\x01\xc2#\x01\xc2$\x01\xc2%\x01\xc2&\x01\xc2'\x01\xc2(\x01\xc2)\x01\xc2*\x01\xc2+\x01\xc2,\x01\xc2-\x01\xc2.\x01\xc2/\x01\xc20\x01\xc21\x01\xc22\x01\xc23\x01\xc24\x01\xc25\x01\xc26\x01\xc27\x01\xc28\x01\xc29\x01\xc2:\x01\xc2;\x01\xc2<\x01\xc2=\x01\xc2>\x01\xc2?\x01\xc2@\x01\xc2A\x01\xc2B\x01\xc2C\x01\xc2D\x01\xc2E\x01\xc2F\x01\xc2G\x01\xc2H\x01\xc2I\x01\xc2J\x01\xc2K\x01\xc2L\x01\xc2M\x01\xc2N\x01\xc2O\x01\xc2P\x01\xc2Q\x01\xc2R\x01\xc2S\x01\xc2T\x01\xc2U\x01\xc2V\x01\xc2W\x01\xc2X\x01\xc2Y\x01\xc2Z\x01\xc2[\x01\xc2\\\x01\xc2]\x01\xc2^\x01\xc2_\x01\xc2`\x01\xc2a\x01\xc2b\x01\xc2c\x01\xc2d\x01\xc2e\x01\xc2f\x01\xc2g\x01\xc2h\x01\xc2i\x01\xc2j\x01\xc2k\x01\xc2l\x01\xc2m\x01\xc2n\x01\xc2o\x01\xc2p\x01\xc2q\x01\xc2r\x01\xc2s\x01\xc2t\x01\xc2u\x01\xc2v\x01\xc2w\x01\xc2x\x01\xc2y\x01\xc2z\x01\xc2{\x01\xc2|\x01\xc2}\x01\xc2~\x01\xc2\u007f\x01\u00c1\x80\x01\u00c1\x81\x01\u00c1\x82\x01\u00c1\x83\x01\u00c1\x84\x01\u00c1\x85\x01\u00c1\x86\x01\u00c1\x87\x01\u00c1\x88\x01\u00c1\x89\x01\u00c1\x8a\x01\u00c1\x8b\x01\u00c1\x8c\x01\u00c1\x8d\x01\u00c1\x8e\x01\u00c1\x8f\x01\u00c1\x90\x01\u00c1\x91\x01\u00c1\x92\x01\u00c1\x93\x01\u00c1\x94\x01\u00c1\x95\x01\u00c1\x96\x01\u00c1\x97\x01\u00c1\x98\x01\u00c1\x99\x01\u00c1\x9a\x01\u00c1\x9b\x01\u00c1\x9c\x01\u00c1\x9d\x01\u00c1\x9e\x01\u00c1\x9f\x01\u00c1\xa0\x01\u00c1\xa1\x01\u00c1\xa2\x01\u00c1\xa3\x01\u00c1\xa4\x01\u00c1\xa5\x01\u00c1\xa6\x01\u00c1\xa7\x01\u00c1\xa8\x01\u00c1\xa9\x01\u00c1\xaa\x01\u00c1\xab\x01\u00c1\xac\x01\u00c1\xad\x01\u00c1\xae\x01\u00c1\xaf\x01\u00c1\xb0\x01\u00c1\xb1\x01\u00c1\xb2\x01\u00c1\xb3\x01\u00c1\xb4\x01\u00c1\xb5\x01\u00c1\xb6\x01\u00c1\xb7\x01\u00c1\xb8\x01\u00c1\xb9\x01\u00c1\xba\x01\u00c1\xbb\x01\u00c1\xbc\x01\u00c1\xbd\x01\u00c1\xbe\x01\u00c1\xbf\x01\u00c1\xc0\x01\u00c1\xc1\x01\u00c1\xc2\x01\u00c1\xc3\x01\u00c1\xc4\x01\u00c1\xc5\x01\u00c1\xc6\x01\u00c1\xc7\x01\u00c1\xc8\x01\u00c1\xc9\x01\u00c1\xca\x01\u00c1\xcb\x01\u00c1\xcc\x01\u00c1\xcd\x01\u00c1\xce\x01\u00c1\xcf\x01\u00c1\xd0\x01\u00c1\xd1\x01\u00c1\xd2\x01\u00c1\xd3\x01\u00c1\xd4\x01\u00c1\xd5\x01\u00c1\xd6\x01\u00c1\xd7\x01\u00c1\xd8\x01\u00c1\xd9\x01\u00c1\xda\x01\u00c1\xdb\x01\u00c1\xdc\x01\u00c1\xdd\x01\u00c1\xde\x01\u00c1\xdf\x01\u00c1\xe0\x01\u00c1\xe1\x01\u00c1\xe2\x01\u00c1\xe3\x01\u00c1\xe4\x01\u00c1\xe5\x01\u00c1\xe6\x01\u00c1\xe7\x01\u00c1\xe8\x01\u00c1\xe9\x01\u00c1\xea\x01\u00c1\xeb\x01\u00c1\xec\x01\u00c1\xed\x01\u00c1\xee\x01\u00c1\xef\x01\u00c1\xf0\x01\u00c1\xf1\x01\u00c1\xf2\x01\u00c1\xf3\x01\u00c1\xf4\x01\u00c1\xf5\x01\u00c1\xf6\x01\u00c1\xf7\x01\u00c1\xf8\x01\u00c1\xf9\x01\u00c1\xfa\x01\u00c1\xfb\x01\u00c1\xfc\x01\u00c1\xfd\x01\u00c1\xfe\x01\u00c1\xff\x01\xe0\x94L*\xe4\x82Y5\x05\xf0\x16<\xde\xfc\a>\x81\xc6<\xdaA\a\x8a\x15-\x02\xc7\xe1J\xf6\x80\x00\x00\xe0\x94\xa8\xe8\xf1G2e\x8eKQ\xe8q\x191\x05:\x8ai\xba\xf2\xb1\x8a\x15-\x02\xc7\xe1J\xf6\x80\x00\x00\xe1\x94\u0665\x17\x9f\t\x1d\x85\x05\x1d<\x98'\x85\xef\xd1E\\\uc199\x8b\bE\x95\x16\x14\x01HJ\x00\x00\x00\xe1\x94\u08bdBX\xd2v\x887\xba\xa2j(\xfeq\xdc\a\x9f\x84\u01cbJG\xe3\xc1$H\xf4\xad\x00\x00\x00" -const calaverasAllocData = "\xf9\x06\x14\u0080\x01\xc2\x01\x01\xc2\x02\x01\xc2\x03\x01\xc2\x04\x01\xc2\x05\x01\xc2\x06\x01\xc2\a\x01\xc2\b\x01\xc2\t\x01\xc2\n\x01\xc2\v\x01\xc2\f\x01\xc2\r\x01\xc2\x0e\x01\xc2\x0f\x01\xc2\x10\x01\xc2\x11\x01\xc2\x12\x01\xc2\x13\x01\xc2\x14\x01\xc2\x15\x01\xc2\x16\x01\xc2\x17\x01\xc2\x18\x01\xc2\x19\x01\xc2\x1a\x01\xc2\x1b\x01\xc2\x1c\x01\xc2\x1d\x01\xc2\x1e\x01\xc2\x1f\x01\xc2 \x01\xc2!\x01\xc2\"\x01\xc2#\x01\xc2$\x01\xc2%\x01\xc2&\x01\xc2'\x01\xc2(\x01\xc2)\x01\xc2*\x01\xc2+\x01\xc2,\x01\xc2-\x01\xc2.\x01\xc2/\x01\xc20\x01\xc21\x01\xc22\x01\xc23\x01\xc24\x01\xc25\x01\xc26\x01\xc27\x01\xc28\x01\xc29\x01\xc2:\x01\xc2;\x01\xc2<\x01\xc2=\x01\xc2>\x01\xc2?\x01\xc2@\x01\xc2A\x01\xc2B\x01\xc2C\x01\xc2D\x01\xc2E\x01\xc2F\x01\xc2G\x01\xc2H\x01\xc2I\x01\xc2J\x01\xc2K\x01\xc2L\x01\xc2M\x01\xc2N\x01\xc2O\x01\xc2P\x01\xc2Q\x01\xc2R\x01\xc2S\x01\xc2T\x01\xc2U\x01\xc2V\x01\xc2W\x01\xc2X\x01\xc2Y\x01\xc2Z\x01\xc2[\x01\xc2\\\x01\xc2]\x01\xc2^\x01\xc2_\x01\xc2`\x01\xc2a\x01\xc2b\x01\xc2c\x01\xc2d\x01\xc2e\x01\xc2f\x01\xc2g\x01\xc2h\x01\xc2i\x01\xc2j\x01\xc2k\x01\xc2l\x01\xc2m\x01\xc2n\x01\xc2o\x01\xc2p\x01\xc2q\x01\xc2r\x01\xc2s\x01\xc2t\x01\xc2u\x01\xc2v\x01\xc2w\x01\xc2x\x01\xc2y\x01\xc2z\x01\xc2{\x01\xc2|\x01\xc2}\x01\xc2~\x01\xc2\u007f\x01\u00c1\x80\x01\u00c1\x81\x01\u00c1\x82\x01\u00c1\x83\x01\u00c1\x84\x01\u00c1\x85\x01\u00c1\x86\x01\u00c1\x87\x01\u00c1\x88\x01\u00c1\x89\x01\u00c1\x8a\x01\u00c1\x8b\x01\u00c1\x8c\x01\u00c1\x8d\x01\u00c1\x8e\x01\u00c1\x8f\x01\u00c1\x90\x01\u00c1\x91\x01\u00c1\x92\x01\u00c1\x93\x01\u00c1\x94\x01\u00c1\x95\x01\u00c1\x96\x01\u00c1\x97\x01\u00c1\x98\x01\u00c1\x99\x01\u00c1\x9a\x01\u00c1\x9b\x01\u00c1\x9c\x01\u00c1\x9d\x01\u00c1\x9e\x01\u00c1\x9f\x01\u00c1\xa0\x01\u00c1\xa1\x01\u00c1\xa2\x01\u00c1\xa3\x01\u00c1\xa4\x01\u00c1\xa5\x01\u00c1\xa6\x01\u00c1\xa7\x01\u00c1\xa8\x01\u00c1\xa9\x01\u00c1\xaa\x01\u00c1\xab\x01\u00c1\xac\x01\u00c1\xad\x01\u00c1\xae\x01\u00c1\xaf\x01\u00c1\xb0\x01\u00c1\xb1\x01\u00c1\xb2\x01\u00c1\xb3\x01\u00c1\xb4\x01\u00c1\xb5\x01\u00c1\xb6\x01\u00c1\xb7\x01\u00c1\xb8\x01\u00c1\xb9\x01\u00c1\xba\x01\u00c1\xbb\x01\u00c1\xbc\x01\u00c1\xbd\x01\u00c1\xbe\x01\u00c1\xbf\x01\u00c1\xc0\x01\u00c1\xc1\x01\u00c1\xc2\x01\u00c1\xc3\x01\u00c1\xc4\x01\u00c1\xc5\x01\u00c1\xc6\x01\u00c1\xc7\x01\u00c1\xc8\x01\u00c1\xc9\x01\u00c1\xca\x01\u00c1\xcb\x01\u00c1\xcc\x01\u00c1\xcd\x01\u00c1\xce\x01\u00c1\xcf\x01\u00c1\xd0\x01\u00c1\xd1\x01\u00c1\xd2\x01\u00c1\xd3\x01\u00c1\xd4\x01\u00c1\xd5\x01\u00c1\xd6\x01\u00c1\xd7\x01\u00c1\xd8\x01\u00c1\xd9\x01\u00c1\xda\x01\u00c1\xdb\x01\u00c1\xdc\x01\u00c1\xdd\x01\u00c1\xde\x01\u00c1\xdf\x01\u00c1\xe0\x01\u00c1\xe1\x01\u00c1\xe2\x01\u00c1\xe3\x01\u00c1\xe4\x01\u00c1\xe5\x01\u00c1\xe6\x01\u00c1\xe7\x01\u00c1\xe8\x01\u00c1\xe9\x01\u00c1\xea\x01\u00c1\xeb\x01\u00c1\xec\x01\u00c1\xed\x01\u00c1\xee\x01\u00c1\xef\x01\u00c1\xf0\x01\u00c1\xf1\x01\u00c1\xf2\x01\u00c1\xf3\x01\u00c1\xf4\x01\u00c1\xf5\x01\u00c1\xf6\x01\u00c1\xf7\x01\u00c1\xf8\x01\u00c1\xf9\x01\u00c1\xfa\x01\u00c1\xfb\x01\u00c1\xfc\x01\u00c1\xfd\x01\u00c1\xfe\x01\u00c1\xff\x01\xf6\x94\x0e\x89\xe2\xae\xdb\x1c\xfc\u06d4$\xd4\x1a\x1f!\x8fA2s\x81r\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94\x10A\xaf\xbc\xb3Y\u0568\xdcX\xc1[/\xf5\x13T\xff\x8a!}\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94#o\xf1\xe9t\x19\xae\x93\xad\x80\xca\xfb\xaa!\"\f]x\xfb}\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94`\xad\xc0\xf8\x9aA\xaf#|\xe75T\xed\xe1p\xd73\xec\x14\xe0\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94y\x9d2\x9e_X4\x19\x16|\xd7\"\x96$\x85\x92n3\x8fJ\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94|\xf5\xb7\x9b\xfe)\x1ag\xab\x02\xb3\x93\xe4V\xcc\xc4\xc2f\xf7S\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94\x8a\x8e\xaf\xb1\xcfb\xbf\xbe\xb1t\x17i\xda\xe1\xa9\xddG\x99a\x92\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94\x8b\xa1\xf1\tU\x1b\xd42\x800\x12dZ\xc16\xdd\xd6M\xbar\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94\xb0*.\xda\x1b1\u007f\xbd\x16v\x01(\x83k\n\u015bV\x0e\x9d\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94\xba\xdc\r\xe9\xe0yK\x04\x9b^\xa6<>\x1ei\x8a4v\xc1r\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94\xf00\v\ue24a\xe2r\xeb4~\x83i\xac\fv\xdfB\xc9?\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\xf6\x94\xfe;U~\x8f\xb6+\x89\xf4\x91kr\x1b\xe5\\\ub08d\xbds\xa0\x02\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00" const sepoliaAllocData = "\xf9\x01\xee\u0791i\x16\xa8{\x823?BE\x04f#\xb27\x94\xc6\\\x8b\bE\x95\x16\x14\x01HJ\x00\x00\x00\xe1\x94\x10\xf5\xd4XT\xe08\a\x14\x85\xac\x9e@#\b\u03c0\xd2\xd2\xfe\x8bR\xb7\xd2\xdc\xc8\f\xd2\xe4\x00\x00\x00\u0794y\x9d2\x9e_X4\x19\x16|\xd7\"\x96$\x85\x92n3\x8fJ\x88\r\u0db3\xa7d\x00\x00\xe0\x94|\xf5\xb7\x9b\xfe)\x1ag\xab\x02\xb3\x93\xe4V\xcc\xc4\xc2f\xf7S\x8a\xd3\xc2\x1b\xce\xcc\xed\xa1\x00\x00\x00\xe0\x94\x8b\u007f\tw\xbbO\x0f\xbepv\xfa\"\xbc$\xac\xa0CX?^\x8a\xd3\xc2\x1b\xce\xcc\xed\xa1\x00\x00\x00\xe0\x94\xa2\xa6\xd949\x14O\xfeM'\xc9\xe0\x88\xdc\u0637\x83\x94bc\x8a\xd3\xc2\x1b\xce\xcc\xed\xa1\x00\x00\x00\xe0\x94\xaa\xec\x869DA\xf9\x15\xbc\xe3\xe6\xab9\x99w\xe9\x90o;i\x8a\xd3\xc2\x1b\xce\xcc\xed\xa1\x00\x00\x00\u1532\x1c3\xde\x1f\xab?\xa1T\x99\xc6+Y\xfe\f\xc3%\x00 \u044bR\xb7\xd2\xdc\xc8\f\xd2\xe4\x00\x00\x00\xe0\x94\xbc\x11)Y6\xaay\u0554\x13\x9d\xe1\xb2\xe1&)AO;\u06ca\xd3\xc2\x1b\xce\xcc\xed\xa1\x00\x00\x00\xe0\x94\xbe\xef2\xca[\x9a\x19\x8d'\xb4\xe0/LpC\x9f\xe6\x03V\u03ca\xd3\xc2\x1b\xce\xcc\xed\xa1\x00\x00\x00\xe1\x94\xd7\xd7lX\xb3\xa5\x19\xe9\xfal\xc4\xd2-\xc0\x17%\x9b\u011f\x1e\x8bR\xb7\xd2\xdc\xc8\f\xd2\xe4\x00\x00\x00\xe0\x94\xd7\xed\xdbx\xed)[<\x96)$\x0e\x89$\xfb\x8d\x88t\xdd\u060a\xd3\xc2\x1b\xce\xcc\xed\xa1\x00\x00\x00\xe0\x94\u0665\x17\x9f\t\x1d\x85\x05\x1d<\x98'\x85\xef\xd1E\\\uc199\x8a\xd3\xc2\x1b\xce\xcc\xed\xa1\x00\x00\x00\xe0\x94\xe2\xe2e\x90(\x147\x84\xd5W\xbc\xeco\xf3\xa0r\x10H\x88\n\x8a\xd3\xc2\x1b\xce\xcc\xed\xa1\x00\x00\x00\xe0\x94\xf4|\xae\x1c\xf7\x9c\xa6u\x8b\xfcx}\xbd!\u6f7eq\x12\xb8\x8a\xd3\xc2\x1b\xce\xcc\xed\xa1\x00\x00\x00" const KilnAllocData = `{ "config": { From 0016eb7eeeb42568c8c20d0cb560ddfc9a938fad Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Mon, 15 Aug 2022 10:04:58 +0200 Subject: [PATCH 38/59] params: set ttdpassed on goerli (#25519) --- params/config.go | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/params/config.go b/params/config.go index fda4c1aec..87658b372 100644 --- a/params/config.go +++ b/params/config.go @@ -217,22 +217,23 @@ var ( // GoerliChainConfig contains the chain parameters to run a node on the Görli test network. GoerliChainConfig = &ChainConfig{ - ChainID: big.NewInt(5), - HomesteadBlock: big.NewInt(0), - DAOForkBlock: nil, - DAOForkSupport: true, - EIP150Block: big.NewInt(0), - EIP155Block: big.NewInt(0), - EIP158Block: big.NewInt(0), - ByzantiumBlock: big.NewInt(0), - ConstantinopleBlock: big.NewInt(0), - PetersburgBlock: big.NewInt(0), - IstanbulBlock: big.NewInt(1_561_651), - MuirGlacierBlock: nil, - BerlinBlock: big.NewInt(4_460_644), - LondonBlock: big.NewInt(5_062_605), - ArrowGlacierBlock: nil, - TerminalTotalDifficulty: big.NewInt(10_790_000), + ChainID: big.NewInt(5), + HomesteadBlock: big.NewInt(0), + DAOForkBlock: nil, + DAOForkSupport: true, + EIP150Block: big.NewInt(0), + EIP155Block: big.NewInt(0), + EIP158Block: big.NewInt(0), + ByzantiumBlock: big.NewInt(0), + ConstantinopleBlock: big.NewInt(0), + PetersburgBlock: big.NewInt(0), + IstanbulBlock: big.NewInt(1_561_651), + MuirGlacierBlock: nil, + BerlinBlock: big.NewInt(4_460_644), + LondonBlock: big.NewInt(5_062_605), + ArrowGlacierBlock: nil, + TerminalTotalDifficulty: big.NewInt(10_790_000), + TerminalTotalDifficultyPassed: true, Clique: &CliqueConfig{ Period: 15, Epoch: 30000, From c4ab7d229117fdaa24292d84c1d450a727cb3e9f Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Tue, 16 Aug 2022 08:38:23 +0200 Subject: [PATCH 39/59] params: set mainnet terminal total difficulty for the merge (#25528) * params: set mainnet ttd to 58_750_000_000_000_000_000_000 * params: set mainnet ttd to 58_750_000_000_000_000_000_000 --- params/config.go | 43 +++++++++++++++++++++++-------------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/params/config.go b/params/config.go index 87658b372..4d6eec893 100644 --- a/params/config.go +++ b/params/config.go @@ -55,26 +55,29 @@ var CheckpointOracles = map[common.Hash]*CheckpointOracleConfig{ } var ( + MainnetTerminalTotalDifficulty, _ = new(big.Int).SetString("58_750_000_000_000_000_000_000", 0) + // MainnetChainConfig is the chain parameters to run a node on the main network. MainnetChainConfig = &ChainConfig{ - ChainID: big.NewInt(1), - HomesteadBlock: big.NewInt(1_150_000), - DAOForkBlock: big.NewInt(1_920_000), - DAOForkSupport: true, - EIP150Block: big.NewInt(2_463_000), - EIP150Hash: common.HexToHash("0x2086799aeebeae135c246c65021c82b4e15a2c451340993aacfd2751886514f0"), - EIP155Block: big.NewInt(2_675_000), - EIP158Block: big.NewInt(2_675_000), - ByzantiumBlock: big.NewInt(4_370_000), - ConstantinopleBlock: big.NewInt(7_280_000), - PetersburgBlock: big.NewInt(7_280_000), - IstanbulBlock: big.NewInt(9_069_000), - MuirGlacierBlock: big.NewInt(9_200_000), - BerlinBlock: big.NewInt(12_244_000), - LondonBlock: big.NewInt(12_965_000), - ArrowGlacierBlock: big.NewInt(13_773_000), - GrayGlacierBlock: big.NewInt(15_050_000), - Ethash: new(EthashConfig), + ChainID: big.NewInt(1), + HomesteadBlock: big.NewInt(1_150_000), + DAOForkBlock: big.NewInt(1_920_000), + DAOForkSupport: true, + EIP150Block: big.NewInt(2_463_000), + EIP150Hash: common.HexToHash("0x2086799aeebeae135c246c65021c82b4e15a2c451340993aacfd2751886514f0"), + EIP155Block: big.NewInt(2_675_000), + EIP158Block: big.NewInt(2_675_000), + ByzantiumBlock: big.NewInt(4_370_000), + ConstantinopleBlock: big.NewInt(7_280_000), + PetersburgBlock: big.NewInt(7_280_000), + IstanbulBlock: big.NewInt(9_069_000), + MuirGlacierBlock: big.NewInt(9_200_000), + BerlinBlock: big.NewInt(12_244_000), + LondonBlock: big.NewInt(12_965_000), + ArrowGlacierBlock: big.NewInt(13_773_000), + GrayGlacierBlock: big.NewInt(15_050_000), + TerminalTotalDifficulty: MainnetTerminalTotalDifficulty, // 58_750_000_000_000_000_000_000 + Ethash: new(EthashConfig), } // MainnetTrustedCheckpoint contains the light client trusted checkpoint for the main network. @@ -471,10 +474,10 @@ func (c *ChainConfig) String() string { // Add a special section for the merge as it's non-obvious if c.TerminalTotalDifficulty == nil { banner += "The Merge is not yet available for this network!\n" - banner += " - Hard-fork specification: https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/paris.md)" + banner += " - Hard-fork specification: https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/paris.md" } else { banner += "Merge configured:\n" - banner += " - Hard-fork specification: https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/paris.md)\n" + banner += " - Hard-fork specification: https://github.com/ethereum/execution-specs/blob/master/network-upgrades/mainnet-upgrades/paris.md\n" banner += fmt.Sprintf(" - Network known to be merged: %v\n", c.TerminalTotalDifficultyPassed) banner += fmt.Sprintf(" - Total terminal difficulty: %v\n", c.TerminalTotalDifficulty) banner += fmt.Sprintf(" - Merge netsplit block: %-8v", c.MergeNetsplitBlock) From 12185e40e06ec9d68a3780f56ec70a0a831b25bc Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Wed, 17 Aug 2022 13:12:10 +0200 Subject: [PATCH 40/59] core, trie: flush preimages to db on blockchain close (#25533) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * core, trie: flush preimages to db on database close Co-authored-by: rjl493456442 * rename Close to CommitPreimages for clarity * core, trie: nitpick fixes Co-authored-by: rjl493456442 Co-authored-by: Péter Szilágyi --- core/blockchain.go | 4 ++++ trie/database.go | 13 +++++++++++++ 2 files changed, 17 insertions(+) diff --git a/core/blockchain.go b/core/blockchain.go index f35de4046..ee95cfb6c 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -893,6 +893,10 @@ func (bc *BlockChain) Stop() { log.Error("Dangling trie nodes after full cleanup") } } + // Flush the collected preimages to disk + if err := bc.stateCache.TrieDB().CommitPreimages(); err != nil { + log.Error("Failed to commit trie preimages", "err", err) + } // Ensure all live cached entries be saved into disk, so that we can skip // cache warmup when node restarts. if bc.cacheConfig.TrieCleanJournal != "" { diff --git a/trie/database.go b/trie/database.go index 81f0477ae..8c9f47176 100644 --- a/trie/database.go +++ b/trie/database.go @@ -852,3 +852,16 @@ func (db *Database) SaveCachePeriodically(dir string, interval time.Duration, st } } } + +// CommitPreimages flushes the dangling preimages to disk. It is meant to be +// called when closing the blockchain object, so that preimages are persisted +// to the database. +func (db *Database) CommitPreimages() error { + db.lock.Lock() + defer db.lock.Unlock() + + if db.preimages == nil { + return nil + } + return db.preimages.commit(true) +} From 6da5c1644db87655d07db4b2a8a23b0c5fad619d Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Wed, 17 Aug 2022 13:14:49 +0200 Subject: [PATCH 41/59] core/state, trie, light: add a TryDeleteAccount method (#25531) * core/state, trie, light: Add a DeleteAccount method * review feedback * Update database.go * pr triage feedback Co-authored-by: rjl493456442 --- core/state/database.go | 3 +++ core/state/statedb.go | 2 +- light/trie.go | 8 ++++++++ trie/secure_trie.go | 7 +++++++ 4 files changed, 19 insertions(+), 1 deletion(-) diff --git a/core/state/database.go b/core/state/database.go index edbf78ae3..96b6bcfe6 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -87,6 +87,9 @@ type Trie interface { // found in the database, a trie.MissingNodeError is returned. TryDelete(key []byte) error + // TryDeleteAccount abstracts an account deletion from the trie. + TryDeleteAccount(key []byte) error + // Hash returns the root hash of the trie. It does not write to the database and // can be used even if the trie doesn't have one. Hash() common.Hash diff --git a/core/state/statedb.go b/core/state/statedb.go index cd388d6a3..5c97dd94a 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -484,7 +484,7 @@ func (s *StateDB) deleteStateObject(obj *stateObject) { } // Delete the account from the trie addr := obj.Address() - if err := s.trie.TryDelete(addr[:]); err != nil { + if err := s.trie.TryDeleteAccount(addr[:]); err != nil { s.setError(fmt.Errorf("deleteStateObject (%x) error: %v", addr[:], err)) } } diff --git a/light/trie.go b/light/trie.go index 5755e2cc1..f60edaa3b 100644 --- a/light/trie.go +++ b/light/trie.go @@ -153,6 +153,14 @@ func (t *odrTrie) TryDelete(key []byte) error { }) } +// TryDeleteACcount abstracts an account deletion from the trie. +func (t *odrTrie) TryDeleteAccount(key []byte) error { + key = crypto.Keccak256(key) + return t.do(key, func() error { + return t.trie.TryDelete(key) + }) +} + func (t *odrTrie) Commit(collectLeaf bool) (common.Hash, *trie.NodeSet, error) { if t.trie == nil { return t.id.Root, nil, nil diff --git a/trie/secure_trie.go b/trie/secure_trie.go index 28b3473c0..3d468f56e 100644 --- a/trie/secure_trie.go +++ b/trie/secure_trie.go @@ -189,6 +189,13 @@ func (t *StateTrie) TryDelete(key []byte) error { return t.trie.TryDelete(hk) } +// TryDeleteACcount abstracts an account deletion from the trie. +func (t *StateTrie) TryDeleteAccount(key []byte) error { + hk := t.hashKey(key) + delete(t.getSecKeyCache(), string(hk)) + return t.trie.TryDelete(hk) +} + // GetKey returns the sha3 preimage of a hashed key that was // previously used to store a value. func (t *StateTrie) GetKey(shaKey []byte) []byte { From a50c006b49ef0411e3389b801f5b4594c72fe3b3 Mon Sep 17 00:00:00 2001 From: Seungbae Yu Date: Wed, 17 Aug 2022 20:16:18 +0900 Subject: [PATCH 42/59] core: make tx journal check and open atomic (#25530) * core: reduce system call about `os` * avoid deprecated method --- core/tx_journal.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/core/tx_journal.go b/core/tx_journal.go index 5453ee191..62344f564 100644 --- a/core/tx_journal.go +++ b/core/tx_journal.go @@ -19,6 +19,7 @@ package core import ( "errors" "io" + "io/fs" "os" "github.com/ethereum/go-ethereum/common" @@ -57,12 +58,12 @@ func newTxJournal(path string) *txJournal { // load parses a transaction journal dump from disk, loading its contents into // the specified pool. func (journal *txJournal) load(add func([]*types.Transaction) []error) error { - // Skip the parsing if the journal file doesn't exist at all - if !common.FileExist(journal.path) { - return nil - } // Open the journal for loading any past transactions input, err := os.Open(journal.path) + if errors.Is(err, fs.ErrNotExist) { + // Skip the parsing if the journal file doesn't exist at all + return nil + } if err != nil { return err } From 23ac8df15302bbde098cab6d711abdd24843d66a Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Thu, 18 Aug 2022 13:28:06 +0200 Subject: [PATCH 43/59] cmd. core: save preimages on genesis creation (#25538) force preimage dump for genesis --- core/genesis.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/genesis.go b/core/genesis.go index 71214e84f..c3b5d0b57 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -104,7 +104,7 @@ func (ga *GenesisAlloc) deriveHash() (common.Hash, error) { // all the generated states will be persisted into the given database. // Also, the genesis state specification will be flushed as well. func (ga *GenesisAlloc) flush(db ethdb.Database) error { - statedb, err := state.New(common.Hash{}, state.NewDatabase(db), nil) + statedb, err := state.New(common.Hash{}, state.NewDatabaseWithConfig(db, &trie.Config{Preimages: true}), nil) if err != nil { return err } From cce7f084388adf9097f730ae48631a55de2e3be1 Mon Sep 17 00:00:00 2001 From: Justin Traglia <95511699+jtraglia@users.noreply.github.com> Date: Thu, 18 Aug 2022 17:34:57 -0500 Subject: [PATCH 44/59] rlp/rlpgen: fix error handling when target type not found (#25547) typ will be nil when lookupStructType returns an error. cfg.Type should be used instead. --- rlp/rlpgen/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rlp/rlpgen/main.go b/rlp/rlpgen/main.go index 17d7e64e0..25d4393cc 100644 --- a/rlp/rlpgen/main.go +++ b/rlp/rlpgen/main.go @@ -106,7 +106,7 @@ func (cfg *Config) process() (code []byte, err error) { // Find the type and generate. typ, err := lookupStructType(pkg.Scope(), cfg.Type) if err != nil { - return nil, fmt.Errorf("can't find %s in %s: %v", typ, pkg, err) + return nil, fmt.Errorf("can't find %s in %s: %v", cfg.Type, pkg, err) } code, err = bctx.generate(typ, cfg.GenerateEncoder, cfg.GenerateDecoder) if err != nil { From a1b8892384eed99beb3f3364cfd13b049a1c0167 Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Fri, 19 Aug 2022 06:39:47 +0800 Subject: [PATCH 45/59] trie: improve node rlp decoding performance (#25357) This avoids copying the input []byte while decoding trie nodes. In most cases, particularly when the input slice is provided by the underlying database, this optimization is safe to use. For cases where the origin of the input slice is unclear, the copying version is retained. The new code performs better even when the input must be copied, because it is now only copied once in decodeNode. --- trie/database.go | 14 ++++-- trie/node.go | 30 ++++++++++-- trie/node_test.go | 121 ++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 158 insertions(+), 7 deletions(-) diff --git a/trie/database.go b/trie/database.go index 8c9f47176..8c154ba96 100644 --- a/trie/database.go +++ b/trie/database.go @@ -163,7 +163,10 @@ func (n *cachedNode) rlp() []byte { // or by regenerating it from the rlp encoded blob. func (n *cachedNode) obj(hash common.Hash) node { if node, ok := n.node.(rawNode); ok { - return mustDecodeNode(hash[:], node) + // The raw-blob format nodes are loaded from either from + // clean cache or the database, they are all in their own + // copy and safe to use unsafe decoder. + return mustDecodeNodeUnsafe(hash[:], node) } return expandNode(hash[:], n.node) } @@ -346,7 +349,10 @@ func (db *Database) node(hash common.Hash) node { if enc := db.cleans.Get(nil, hash[:]); enc != nil { memcacheCleanHitMeter.Mark(1) memcacheCleanReadMeter.Mark(int64(len(enc))) - return mustDecodeNode(hash[:], enc) + + // The returned value from cache is in its own copy, + // safe to use mustDecodeNodeUnsafe for decoding. + return mustDecodeNodeUnsafe(hash[:], enc) } } // Retrieve the node from the dirty cache if available @@ -371,7 +377,9 @@ func (db *Database) node(hash common.Hash) node { memcacheCleanMissMeter.Mark(1) memcacheCleanWriteMeter.Mark(int64(len(enc))) } - return mustDecodeNode(hash[:], enc) + // The returned value from database is in its own copy, + // safe to use mustDecodeNodeUnsafe for decoding. + return mustDecodeNodeUnsafe(hash[:], enc) } // Node retrieves an encoded cached trie node from memory. If it cannot be found diff --git a/trie/node.go b/trie/node.go index bf3f024bb..6ce6551de 100644 --- a/trie/node.go +++ b/trie/node.go @@ -99,6 +99,7 @@ func (n valueNode) fstring(ind string) string { return fmt.Sprintf("%x ", []byte(n)) } +// mustDecodeNode is a wrapper of decodeNode and panic if any error is encountered. func mustDecodeNode(hash, buf []byte) node { n, err := decodeNode(hash, buf) if err != nil { @@ -107,8 +108,29 @@ func mustDecodeNode(hash, buf []byte) node { return n } -// decodeNode parses the RLP encoding of a trie node. +// mustDecodeNodeUnsafe is a wrapper of decodeNodeUnsafe and panic if any error is +// encountered. +func mustDecodeNodeUnsafe(hash, buf []byte) node { + n, err := decodeNodeUnsafe(hash, buf) + if err != nil { + panic(fmt.Sprintf("node %x: %v", hash, err)) + } + return n +} + +// decodeNode parses the RLP encoding of a trie node. It will deep-copy the passed +// byte slice for decoding, so it's safe to modify the byte slice afterwards. The- +// decode performance of this function is not optimal, but it is suitable for most +// scenarios with low performance requirements and hard to determine whether the +// byte slice be modified or not. func decodeNode(hash, buf []byte) (node, error) { + return decodeNodeUnsafe(hash, common.CopyBytes(buf)) +} + +// decodeNodeUnsafe parses the RLP encoding of a trie node. The passed byte slice +// will be directly referenced by node without bytes deep copy, so the input MUST +// not be changed after. +func decodeNodeUnsafe(hash, buf []byte) (node, error) { if len(buf) == 0 { return nil, io.ErrUnexpectedEOF } @@ -141,7 +163,7 @@ func decodeShort(hash, elems []byte) (node, error) { if err != nil { return nil, fmt.Errorf("invalid value node: %v", err) } - return &shortNode{key, append(valueNode{}, val...), flag}, nil + return &shortNode{key, valueNode(val), flag}, nil } r, _, err := decodeRef(rest) if err != nil { @@ -164,7 +186,7 @@ func decodeFull(hash, elems []byte) (*fullNode, error) { return n, err } if len(val) > 0 { - n.Children[16] = append(valueNode{}, val...) + n.Children[16] = valueNode(val) } return n, nil } @@ -190,7 +212,7 @@ func decodeRef(buf []byte) (node, []byte, error) { // empty node return nil, rest, nil case kind == rlp.String && len(val) == 32: - return append(hashNode{}, val...), rest, nil + return hashNode(val), rest, nil default: return nil, nil, fmt.Errorf("invalid RLP string size %d (want 0 or 32)", len(val)) } diff --git a/trie/node_test.go b/trie/node_test.go index ac1d8fbef..9b8b33748 100644 --- a/trie/node_test.go +++ b/trie/node_test.go @@ -20,6 +20,7 @@ import ( "bytes" "testing" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/rlp" ) @@ -92,3 +93,123 @@ func TestDecodeFullNode(t *testing.T) { t.Fatalf("decode full node err: %v", err) } } + +// goos: darwin +// goarch: arm64 +// pkg: github.com/ethereum/go-ethereum/trie +// BenchmarkEncodeShortNode +// BenchmarkEncodeShortNode-8 16878850 70.81 ns/op 48 B/op 1 allocs/op +func BenchmarkEncodeShortNode(b *testing.B) { + node := &shortNode{ + Key: []byte{0x1, 0x2}, + Val: hashNode(randBytes(32)), + } + b.ResetTimer() + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + nodeToBytes(node) + } +} + +// goos: darwin +// goarch: arm64 +// pkg: github.com/ethereum/go-ethereum/trie +// BenchmarkEncodeFullNode +// BenchmarkEncodeFullNode-8 4323273 284.4 ns/op 576 B/op 1 allocs/op +func BenchmarkEncodeFullNode(b *testing.B) { + node := &fullNode{} + for i := 0; i < 16; i++ { + node.Children[i] = hashNode(randBytes(32)) + } + b.ResetTimer() + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + nodeToBytes(node) + } +} + +// goos: darwin +// goarch: arm64 +// pkg: github.com/ethereum/go-ethereum/trie +// BenchmarkDecodeShortNode +// BenchmarkDecodeShortNode-8 7925638 151.0 ns/op 157 B/op 4 allocs/op +func BenchmarkDecodeShortNode(b *testing.B) { + node := &shortNode{ + Key: []byte{0x1, 0x2}, + Val: hashNode(randBytes(32)), + } + blob := nodeToBytes(node) + hash := crypto.Keccak256(blob) + + b.ResetTimer() + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + mustDecodeNode(hash, blob) + } +} + +// goos: darwin +// goarch: arm64 +// pkg: github.com/ethereum/go-ethereum/trie +// BenchmarkDecodeShortNodeUnsafe +// BenchmarkDecodeShortNodeUnsafe-8 9027476 128.6 ns/op 109 B/op 3 allocs/op +func BenchmarkDecodeShortNodeUnsafe(b *testing.B) { + node := &shortNode{ + Key: []byte{0x1, 0x2}, + Val: hashNode(randBytes(32)), + } + blob := nodeToBytes(node) + hash := crypto.Keccak256(blob) + + b.ResetTimer() + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + mustDecodeNodeUnsafe(hash, blob) + } +} + +// goos: darwin +// goarch: arm64 +// pkg: github.com/ethereum/go-ethereum/trie +// BenchmarkDecodeFullNode +// BenchmarkDecodeFullNode-8 1597462 761.9 ns/op 1280 B/op 18 allocs/op +func BenchmarkDecodeFullNode(b *testing.B) { + node := &fullNode{} + for i := 0; i < 16; i++ { + node.Children[i] = hashNode(randBytes(32)) + } + blob := nodeToBytes(node) + hash := crypto.Keccak256(blob) + + b.ResetTimer() + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + mustDecodeNode(hash, blob) + } +} + +// goos: darwin +// goarch: arm64 +// pkg: github.com/ethereum/go-ethereum/trie +// BenchmarkDecodeFullNodeUnsafe +// BenchmarkDecodeFullNodeUnsafe-8 1789070 687.1 ns/op 704 B/op 17 allocs/op +func BenchmarkDecodeFullNodeUnsafe(b *testing.B) { + node := &fullNode{} + for i := 0; i < 16; i++ { + node.Children[i] = hashNode(randBytes(32)) + } + blob := nodeToBytes(node) + hash := crypto.Keccak256(blob) + + b.ResetTimer() + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + mustDecodeNodeUnsafe(hash, blob) + } +} From 2c5648d8915dc6f4cd822a379819d774beba0dc5 Mon Sep 17 00:00:00 2001 From: Justin Traglia <95511699+jtraglia@users.noreply.github.com> Date: Fri, 19 Aug 2022 01:00:21 -0500 Subject: [PATCH 46/59] all: fix some typos (#25551) * Fix some typos * Fix some mistakes * Revert 4byte.json * Fix an incorrect fix * Change files to fails --- accounts/abi/reflect_test.go | 2 +- accounts/keystore/account_cache_test.go | 2 +- accounts/keystore/file_cache.go | 4 +-- accounts/keystore/keystore_test.go | 2 +- accounts/usbwallet/trezor.go | 4 +-- accounts/usbwallet/wallet.go | 2 +- build/ci.go | 2 +- cmd/faucet/faucet.go | 2 +- cmd/geth/consolecmd_test.go | 2 +- cmd/puppeth/ssh.go | 2 +- common/prque/prque.go | 4 +-- consensus/clique/snapshot_test.go | 4 +-- console/console.go | 6 ++-- core/blockchain.go | 4 +-- core/blockchain_repair_test.go | 32 +++++++++---------- core/rawdb/accessors_chain_test.go | 2 +- core/rawdb/database.go | 2 +- core/rawdb/freezer_table.go | 2 +- core/state/snapshot/iterator_fast.go | 4 +-- core/state/snapshot/snapshot_test.go | 2 +- core/state/statedb.go | 6 ++-- core/state/statedb_test.go | 2 +- core/state/sync_test.go | 12 +++---- core/state/trie_prefetcher.go | 2 +- core/types/block.go | 2 +- core/types/block_test.go | 2 +- core/vm/runtime/runtime_test.go | 4 +-- crypto/bls12381/isogeny.go | 4 +-- eth/catalyst/api.go | 4 +-- eth/downloader/api.go | 2 +- eth/downloader/beaconsync.go | 2 +- eth/downloader/downloader.go | 4 +-- eth/downloader/downloader_test.go | 8 ++--- eth/downloader/fetchers_concurrent.go | 6 ++-- eth/downloader/fetchers_concurrent_bodies.go | 4 +-- eth/downloader/fetchers_concurrent_headers.go | 4 +-- .../fetchers_concurrent_receipts.go | 6 ++-- eth/downloader/queue.go | 2 +- eth/downloader/skeleton.go | 12 +++---- eth/downloader/skeleton_test.go | 14 ++++---- eth/ethconfig/config.go | 2 +- eth/fetcher/tx_fetcher.go | 10 +++--- eth/fetcher/tx_fetcher_test.go | 6 ++-- eth/filters/api.go | 2 +- eth/peerset.go | 2 +- eth/protocols/eth/broadcast.go | 2 +- eth/protocols/eth/dispatcher.go | 2 +- eth/protocols/eth/handler_test.go | 2 +- eth/protocols/eth/peer.go | 2 +- eth/protocols/snap/handler.go | 2 +- eth/protocols/snap/peer.go | 8 ++--- eth/protocols/snap/sync.go | 26 +++++++-------- eth/state_accessor.go | 2 +- eth/tracers/api.go | 10 +++--- .../internal/tracetest/calltrace_test.go | 2 +- ethclient/ethclient_test.go | 2 +- ethdb/memorydb/memorydb.go | 2 +- interfaces.go | 2 +- internal/ethapi/api.go | 2 +- les/downloader/api.go | 2 +- les/downloader/downloader.go | 4 +-- les/downloader/downloader_test.go | 4 +-- les/downloader/queue.go | 2 +- les/fetcher.go | 2 +- les/flowcontrol/manager.go | 2 +- les/odr.go | 2 +- les/vflux/client/fillset_test.go | 2 +- les/vflux/client/serverpool_test.go | 2 +- les/vflux/server/balance.go | 2 +- les/vflux/server/balance_test.go | 4 +-- les/vflux/server/status.go | 2 +- light/lightchain.go | 2 +- light/odr_util.go | 4 +-- light/postprocess.go | 4 +-- light/trie.go | 2 +- light/txpool.go | 2 +- metrics/gauge_float64_test.go | 2 +- metrics/gauge_test.go | 2 +- metrics/prometheus/prometheus.go | 2 +- miner/unconfirmed_test.go | 2 +- miner/worker_test.go | 2 +- mobile/accounts.go | 6 ++-- mobile/init.go | 2 +- node/rpcstack_test.go | 2 +- p2p/discover/v4_udp.go | 2 +- p2p/discover/v5_udp.go | 2 +- p2p/msgrate/msgrate.go | 6 ++-- p2p/tracker/tracker.go | 2 +- rpc/server.go | 2 +- signer/rules/rules_test.go | 8 ++--- signer/storage/aes_gcm_storage.go | 2 +- trie/hasher.go | 2 +- trie/proof_test.go | 32 +++++++++---------- trie/secure_trie_test.go | 2 +- 94 files changed, 200 insertions(+), 200 deletions(-) diff --git a/accounts/abi/reflect_test.go b/accounts/abi/reflect_test.go index cf13a79da..76ef1ad2a 100644 --- a/accounts/abi/reflect_test.go +++ b/accounts/abi/reflect_test.go @@ -32,7 +32,7 @@ type reflectTest struct { var reflectTests = []reflectTest{ { - name: "OneToOneCorrespondance", + name: "OneToOneCorrespondence", args: []string{"fieldA"}, struc: struct { FieldA int `abi:"fieldA"` diff --git a/accounts/keystore/account_cache_test.go b/accounts/keystore/account_cache_test.go index bdcd81182..daea497d1 100644 --- a/accounts/keystore/account_cache_test.go +++ b/accounts/keystore/account_cache_test.go @@ -318,7 +318,7 @@ func waitForAccounts(wantAccounts []accounts.Account, ks *KeyStore) error { func TestUpdatedKeyfileContents(t *testing.T) { t.Parallel() - // Create a temporary kesytore to test with + // Create a temporary keystore to test with rand.Seed(time.Now().UnixNano()) dir := filepath.Join(os.TempDir(), fmt.Sprintf("eth-keystore-updatedkeyfilecontents-test-%d-%d", os.Getpid(), rand.Int())) ks := NewKeyStore(dir, LightScryptN, LightScryptP) diff --git a/accounts/keystore/file_cache.go b/accounts/keystore/file_cache.go index b3ecf8946..79f9a2963 100644 --- a/accounts/keystore/file_cache.go +++ b/accounts/keystore/file_cache.go @@ -39,7 +39,7 @@ type fileCache struct { func (fc *fileCache) scan(keyDir string) (mapset.Set, mapset.Set, mapset.Set, error) { t0 := time.Now() - // List all the failes from the keystore folder + // List all the files from the keystore folder files, err := os.ReadDir(keyDir) if err != nil { return nil, nil, nil, err @@ -61,7 +61,7 @@ func (fc *fileCache) scan(keyDir string) (mapset.Set, mapset.Set, mapset.Set, er log.Trace("Ignoring file on account scan", "path", path) continue } - // Gather the set of all and fresly modified files + // Gather the set of all and freshly modified files all.Add(path) info, err := fi.Info() diff --git a/accounts/keystore/keystore_test.go b/accounts/keystore/keystore_test.go index 6a2e32f6d..4cdf0b1ed 100644 --- a/accounts/keystore/keystore_test.go +++ b/accounts/keystore/keystore_test.go @@ -214,7 +214,7 @@ func TestSignRace(t *testing.T) { // Tests that the wallet notifier loop starts and stops correctly based on the // addition and removal of wallet event subscriptions. func TestWalletNotifierLifecycle(t *testing.T) { - // Create a temporary kesytore to test with + // Create a temporary keystore to test with _, ks := tmpKeyStore(t, false) // Ensure that the notification updater is not running yet diff --git a/accounts/usbwallet/trezor.go b/accounts/usbwallet/trezor.go index c2182b88d..e385682a5 100644 --- a/accounts/usbwallet/trezor.go +++ b/accounts/usbwallet/trezor.go @@ -196,10 +196,10 @@ func (w *trezorDriver) trezorDerive(derivationPath []uint32) (common.Address, er if _, err := w.trezorExchange(&trezor.EthereumGetAddress{AddressN: derivationPath}, address); err != nil { return common.Address{}, err } - if addr := address.GetAddressBin(); len(addr) > 0 { // Older firmwares use binary fomats + if addr := address.GetAddressBin(); len(addr) > 0 { // Older firmwares use binary formats return common.BytesToAddress(addr), nil } - if addr := address.GetAddressHex(); len(addr) > 0 { // Newer firmwares use hexadecimal fomats + if addr := address.GetAddressHex(); len(addr) > 0 { // Newer firmwares use hexadecimal formats return common.HexToAddress(addr), nil } return common.Address{}, errors.New("missing derived address") diff --git a/accounts/usbwallet/wallet.go b/accounts/usbwallet/wallet.go index 06ff0636a..0e399a6d0 100644 --- a/accounts/usbwallet/wallet.go +++ b/accounts/usbwallet/wallet.go @@ -380,7 +380,7 @@ func (w *wallet) selfDerive() { // of legacy-ledger, the first account on the legacy-path will // be shown to the user, even if we don't actively track it if i < len(nextAddrs)-1 { - w.log.Info("Skipping trakcking first account on legacy path, use personal.deriveAccount(,, false) to track", + w.log.Info("Skipping tracking first account on legacy path, use personal.deriveAccount(,, false) to track", "path", path, "address", nextAddrs[i]) break } diff --git a/build/ci.go b/build/ci.go index f91de328b..80b040045 100644 --- a/build/ci.go +++ b/build/ci.go @@ -608,7 +608,7 @@ func doDocker(cmdline []string) { } if mismatch { // Build numbers mismatching, retry in a short time to - // avoid concurrent failes in both publisher images. If + // avoid concurrent fails in both publisher images. If // however the retry failed too, it means the concurrent // builder is still crunching, let that do the publish. if i == 0 { diff --git a/cmd/faucet/faucet.go b/cmd/faucet/faucet.go index d49b9ed29..dfb7d326d 100644 --- a/cmd/faucet/faucet.go +++ b/cmd/faucet/faucet.go @@ -709,7 +709,7 @@ func authTwitter(url string, tokenV1, tokenV2 string) (string, string, string, c case tokenV2 != "": return authTwitterWithTokenV2(tweetID, tokenV2) } - // Twiter API token isn't provided so we just load the public posts + // Twitter API token isn't provided so we just load the public posts // and scrape it for the Ethereum address and profile URL. We need to load // the mobile page though since the main page loads tweet contents via JS. url = strings.Replace(url, "https://twitter.com/", "https://mobile.twitter.com/", 1) diff --git a/cmd/geth/consolecmd_test.go b/cmd/geth/consolecmd_test.go index d5ebd74ae..442b82df0 100644 --- a/cmd/geth/consolecmd_test.go +++ b/cmd/geth/consolecmd_test.go @@ -155,7 +155,7 @@ To exit, press ctrl-d or type exit } // trulyRandInt generates a crypto random integer used by the console tests to -// not clash network ports with other tests running cocurrently. +// not clash network ports with other tests running concurrently. func trulyRandInt(lo, hi int) int { num, _ := rand.Int(rand.Reader, big.NewInt(int64(hi-lo))) return int(num.Int64()) + lo diff --git a/cmd/puppeth/ssh.go b/cmd/puppeth/ssh.go index 0c23ab556..a20b3bfda 100644 --- a/cmd/puppeth/ssh.go +++ b/cmd/puppeth/ssh.go @@ -163,7 +163,7 @@ func dial(server string, pubkey []byte) (*sshClient, error) { return nil } // We have a mismatch, forbid connecting - return errors.New("ssh key mismatch, readd the machine to update") + return errors.New("ssh key mismatch, re-add the machine to update") } client, err := ssh.Dial("tcp", hostport, &ssh.ClientConfig{User: username, Auth: auths, HostKeyCallback: keycheck}) if err != nil { diff --git a/common/prque/prque.go b/common/prque/prque.go index 54c78b5fc..fb02e3418 100755 --- a/common/prque/prque.go +++ b/common/prque/prque.go @@ -41,13 +41,13 @@ func (p *Prque) Push(data interface{}, priority int64) { heap.Push(p.cont, &item{data, priority}) } -// Peek returns the value with the greates priority but does not pop it off. +// Peek returns the value with the greatest priority but does not pop it off. func (p *Prque) Peek() (interface{}, int64) { item := p.cont.blocks[0][0] return item.value, item.priority } -// Pops the value with the greates priority off the stack and returns it. +// Pops the value with the greatest priority off the stack and returns it. // Currently no shrinking is done. func (p *Prque) Pop() (interface{}, int64) { item := heap.Pop(p.cont).(*item) diff --git a/consensus/clique/snapshot_test.go b/consensus/clique/snapshot_test.go index b87ad8c23..4a067c625 100644 --- a/consensus/clique/snapshot_test.go +++ b/consensus/clique/snapshot_test.go @@ -305,7 +305,7 @@ func TestClique(t *testing.T) { }, { // Ensure that pending votes don't survive authorization status changes. This // corner case can only appear if a signer is quickly added, removed and then - // readded (or the inverse), while one of the original voters dropped. If a + // re-added (or the inverse), while one of the original voters dropped. If a // past vote is left cached in the system somewhere, this will interfere with // the final signer outcome. signers: []string{"A", "B", "C", "D", "E"}, @@ -344,7 +344,7 @@ func TestClique(t *testing.T) { }, failure: errUnauthorizedSigner, }, { - // An authorized signer that signed recenty should not be able to sign again + // An authorized signer that signed recently should not be able to sign again signers: []string{"A", "B"}, votes: []testerVote{ {signer: "A"}, diff --git a/console/console.go b/console/console.go index c8f6c9cfe..7b9ed27e1 100644 --- a/console/console.go +++ b/console/console.go @@ -290,7 +290,7 @@ func (c *Console) AutoCompleteInput(line string, pos int) (string, []string, str if len(line) == 0 || pos == 0 { return "", nil, "" } - // Chunck data to relevant part for autocompletion + // Chunk data to relevant part for autocompletion // E.g. in case of nested lines eth.getBalance(eth.coinb start := pos - 1 for ; start > 0; start-- { @@ -407,7 +407,7 @@ func (c *Console) StopInteractive() { } } -// Interactive starts an interactive user session, where in.put is propted from +// Interactive starts an interactive user session, where input is prompted from // the configured user prompter. func (c *Console) Interactive() { var ( @@ -497,7 +497,7 @@ func (c *Console) readLines(input chan<- string, errc chan<- error, prompt <-cha } } -// countIndents returns the number of identations for the given input. +// countIndents returns the number of indentations for the given input. // In case of invalid input such as var a = } the result can be negative. func countIndents(input string) int { var ( diff --git a/core/blockchain.go b/core/blockchain.go index ee95cfb6c..a98c3b4db 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1375,7 +1375,7 @@ func (bc *BlockChain) writeBlockAndSetHead(block *types.Block, receipts []*types } // In theory we should fire a ChainHeadEvent when we inject // a canonical block, but sometimes we can insert a batch of - // canonicial blocks. Avoid firing too many ChainHeadEvents, + // canonical blocks. Avoid firing too many ChainHeadEvents, // we will fire an accumulated ChainHeadEvent and disable fire // event here. if emitHeadEvent { @@ -1612,7 +1612,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals, setHead bool) // block in the middle. It can only happen in the clique chain. Whenever // we insert blocks via `insertSideChain`, we only commit `td`, `header` // and `body` if it's non-existent. Since we don't have receipts without - // reexecution, so nothing to commit. But if the sidechain will be adpoted + // reexecution, so nothing to commit. But if the sidechain will be adopted // as the canonical chain eventually, it needs to be reexecuted for missing // state, but if it's this special case here(skip reexecution) we will lose // the empty receipt entry. diff --git a/core/blockchain_repair_test.go b/core/blockchain_repair_test.go index 24309405d..feed8a177 100644 --- a/core/blockchain_repair_test.go +++ b/core/blockchain_repair_test.go @@ -564,7 +564,7 @@ func testShortReorgedSnapSyncingRepair(t *testing.T, snapshots bool) { // Tests a recovery for a long canonical chain with frozen blocks where a recent // block - newer than the ancient limit - was already committed to disk and then // the process crashed. In this case we expect the chain to be rolled back to the -// committed block, with everything afterwads kept as fast sync data. +// committed block, with everything afterwards kept as fast sync data. func TestLongShallowRepair(t *testing.T) { testLongShallowRepair(t, false) } func TestLongShallowRepairWithSnapshots(t *testing.T) { testLongShallowRepair(t, true) } @@ -609,7 +609,7 @@ func testLongShallowRepair(t *testing.T, snapshots bool) { // Tests a recovery for a long canonical chain with frozen blocks where a recent // block - older than the ancient limit - was already committed to disk and then // the process crashed. In this case we expect the chain to be rolled back to the -// committed block, with everything afterwads deleted. +// committed block, with everything afterwards deleted. func TestLongDeepRepair(t *testing.T) { testLongDeepRepair(t, false) } func TestLongDeepRepairWithSnapshots(t *testing.T) { testLongDeepRepair(t, true) } @@ -653,7 +653,7 @@ func testLongDeepRepair(t *testing.T, snapshots bool) { // Tests a recovery for a long canonical chain with frozen blocks where the fast // sync pivot point - newer than the ancient limit - was already committed, after // which the process crashed. In this case we expect the chain to be rolled back -// to the committed block, with everything afterwads kept as fast sync data. +// to the committed block, with everything afterwards kept as fast sync data. func TestLongSnapSyncedShallowRepair(t *testing.T) { testLongSnapSyncedShallowRepair(t, false) } @@ -702,7 +702,7 @@ func testLongSnapSyncedShallowRepair(t *testing.T, snapshots bool) { // Tests a recovery for a long canonical chain with frozen blocks where the fast // sync pivot point - older than the ancient limit - was already committed, after // which the process crashed. In this case we expect the chain to be rolled back -// to the committed block, with everything afterwads deleted. +// to the committed block, with everything afterwards deleted. func TestLongSnapSyncedDeepRepair(t *testing.T) { testLongSnapSyncedDeepRepair(t, false) } func TestLongSnapSyncedDeepRepairWithSnapshots(t *testing.T) { testLongSnapSyncedDeepRepair(t, true) } @@ -843,7 +843,7 @@ func testLongSnapSyncingDeepRepair(t *testing.T, snapshots bool) { // side chain, where a recent block - newer than the ancient limit - was already // committed to disk and then the process crashed. In this test scenario the side // chain is below the committed block. In this case we expect the chain to be -// rolled back to the committed block, with everything afterwads kept as fast +// rolled back to the committed block, with everything afterwards kept as fast // sync data; the side chain completely nuked by the freezer. func TestLongOldForkedShallowRepair(t *testing.T) { testLongOldForkedShallowRepair(t, false) @@ -895,7 +895,7 @@ func testLongOldForkedShallowRepair(t *testing.T, snapshots bool) { // side chain, where a recent block - older than the ancient limit - was already // committed to disk and then the process crashed. In this test scenario the side // chain is below the committed block. In this case we expect the canonical chain -// to be rolled back to the committed block, with everything afterwads deleted; +// to be rolled back to the committed block, with everything afterwards deleted; // the side chain completely nuked by the freezer. func TestLongOldForkedDeepRepair(t *testing.T) { testLongOldForkedDeepRepair(t, false) } func TestLongOldForkedDeepRepairWithSnapshots(t *testing.T) { testLongOldForkedDeepRepair(t, true) } @@ -942,7 +942,7 @@ func testLongOldForkedDeepRepair(t *testing.T, snapshots bool) { // side chain, where the fast sync pivot point - newer than the ancient limit - // was already committed to disk and then the process crashed. In this test scenario // the side chain is below the committed block. In this case we expect the chain -// to be rolled back to the committed block, with everything afterwads kept as +// to be rolled back to the committed block, with everything afterwards kept as // fast sync data; the side chain completely nuked by the freezer. func TestLongOldForkedSnapSyncedShallowRepair(t *testing.T) { testLongOldForkedSnapSyncedShallowRepair(t, false) @@ -994,7 +994,7 @@ func testLongOldForkedSnapSyncedShallowRepair(t *testing.T, snapshots bool) { // side chain, where the fast sync pivot point - older than the ancient limit - // was already committed to disk and then the process crashed. In this test scenario // the side chain is below the committed block. In this case we expect the canonical -// chain to be rolled back to the committed block, with everything afterwads deleted; +// chain to be rolled back to the committed block, with everything afterwards deleted; // the side chain completely nuked by the freezer. func TestLongOldForkedSnapSyncedDeepRepair(t *testing.T) { testLongOldForkedSnapSyncedDeepRepair(t, false) @@ -1149,7 +1149,7 @@ func testLongOldForkedSnapSyncingDeepRepair(t *testing.T, snapshots bool) { // side chain, where a recent block - newer than the ancient limit - was already // committed to disk and then the process crashed. In this test scenario the side // chain is above the committed block. In this case we expect the chain to be -// rolled back to the committed block, with everything afterwads kept as fast +// rolled back to the committed block, with everything afterwards kept as fast // sync data; the side chain completely nuked by the freezer. func TestLongNewerForkedShallowRepair(t *testing.T) { testLongNewerForkedShallowRepair(t, false) @@ -1201,7 +1201,7 @@ func testLongNewerForkedShallowRepair(t *testing.T, snapshots bool) { // side chain, where a recent block - older than the ancient limit - was already // committed to disk and then the process crashed. In this test scenario the side // chain is above the committed block. In this case we expect the canonical chain -// to be rolled back to the committed block, with everything afterwads deleted; +// to be rolled back to the committed block, with everything afterwards deleted; // the side chain completely nuked by the freezer. func TestLongNewerForkedDeepRepair(t *testing.T) { testLongNewerForkedDeepRepair(t, false) } func TestLongNewerForkedDeepRepairWithSnapshots(t *testing.T) { testLongNewerForkedDeepRepair(t, true) } @@ -1248,7 +1248,7 @@ func testLongNewerForkedDeepRepair(t *testing.T, snapshots bool) { // side chain, where the fast sync pivot point - newer than the ancient limit - // was already committed to disk and then the process crashed. In this test scenario // the side chain is above the committed block. In this case we expect the chain -// to be rolled back to the committed block, with everything afterwads kept as fast +// to be rolled back to the committed block, with everything afterwards kept as fast // sync data; the side chain completely nuked by the freezer. func TestLongNewerForkedSnapSyncedShallowRepair(t *testing.T) { testLongNewerForkedSnapSyncedShallowRepair(t, false) @@ -1300,7 +1300,7 @@ func testLongNewerForkedSnapSyncedShallowRepair(t *testing.T, snapshots bool) { // side chain, where the fast sync pivot point - older than the ancient limit - // was already committed to disk and then the process crashed. In this test scenario // the side chain is above the committed block. In this case we expect the canonical -// chain to be rolled back to the committed block, with everything afterwads deleted; +// chain to be rolled back to the committed block, with everything afterwards deleted; // the side chain completely nuked by the freezer. func TestLongNewerForkedSnapSyncedDeepRepair(t *testing.T) { testLongNewerForkedSnapSyncedDeepRepair(t, false) @@ -1454,7 +1454,7 @@ func testLongNewerForkedSnapSyncingDeepRepair(t *testing.T, snapshots bool) { // Tests a recovery for a long canonical chain with frozen blocks and a longer side // chain, where a recent block - newer than the ancient limit - was already committed // to disk and then the process crashed. In this case we expect the chain to be -// rolled back to the committed block, with everything afterwads kept as fast sync +// rolled back to the committed block, with everything afterwards kept as fast sync // data. The side chain completely nuked by the freezer. func TestLongReorgedShallowRepair(t *testing.T) { testLongReorgedShallowRepair(t, false) } func TestLongReorgedShallowRepairWithSnapshots(t *testing.T) { testLongReorgedShallowRepair(t, true) } @@ -1501,7 +1501,7 @@ func testLongReorgedShallowRepair(t *testing.T, snapshots bool) { // Tests a recovery for a long canonical chain with frozen blocks and a longer side // chain, where a recent block - older than the ancient limit - was already committed // to disk and then the process crashed. In this case we expect the canonical chains -// to be rolled back to the committed block, with everything afterwads deleted. The +// to be rolled back to the committed block, with everything afterwards deleted. The // side chain completely nuked by the freezer. func TestLongReorgedDeepRepair(t *testing.T) { testLongReorgedDeepRepair(t, false) } func TestLongReorgedDeepRepairWithSnapshots(t *testing.T) { testLongReorgedDeepRepair(t, true) } @@ -1548,7 +1548,7 @@ func testLongReorgedDeepRepair(t *testing.T, snapshots bool) { // side chain, where the fast sync pivot point - newer than the ancient limit - // was already committed to disk and then the process crashed. In this case we // expect the chain to be rolled back to the committed block, with everything -// afterwads kept as fast sync data. The side chain completely nuked by the +// afterwards kept as fast sync data. The side chain completely nuked by the // freezer. func TestLongReorgedSnapSyncedShallowRepair(t *testing.T) { testLongReorgedSnapSyncedShallowRepair(t, false) @@ -1600,7 +1600,7 @@ func testLongReorgedSnapSyncedShallowRepair(t *testing.T, snapshots bool) { // side chain, where the fast sync pivot point - older than the ancient limit - // was already committed to disk and then the process crashed. In this case we // expect the canonical chains to be rolled back to the committed block, with -// everything afterwads deleted. The side chain completely nuked by the freezer. +// everything afterwards deleted. The side chain completely nuked by the freezer. func TestLongReorgedSnapSyncedDeepRepair(t *testing.T) { testLongReorgedSnapSyncedDeepRepair(t, false) } diff --git a/core/rawdb/accessors_chain_test.go b/core/rawdb/accessors_chain_test.go index dbb13caa4..21d23e1f0 100644 --- a/core/rawdb/accessors_chain_test.go +++ b/core/rawdb/accessors_chain_test.go @@ -285,7 +285,7 @@ func TestTdStorage(t *testing.T) { func TestCanonicalMappingStorage(t *testing.T) { db := NewMemoryDatabase() - // Create a test canonical number and assinged hash to move around + // Create a test canonical number and assigned hash to move around hash, number := common.Hash{0: 0xff}, uint64(314) if entry := ReadCanonicalHash(db, number); entry != (common.Hash{}) { t.Fatalf("Non existent canonical mapping returned: %v", entry) diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 831ca69c4..1eaf033bb 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -260,7 +260,7 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, ancient string, namespace st if kvblob, _ := db.Get(headerHashKey(1)); len(kvblob) == 0 { return nil, errors.New("ancient chain segments already extracted, please set --datadir.ancient to the correct path") } - // Block #1 is still in the database, we're allowed to init a new feezer + // Block #1 is still in the database, we're allowed to init a new freezer } // Otherwise, the head header is still the genesis, we're allowed to init a new // freezer. diff --git a/core/rawdb/freezer_table.go b/core/rawdb/freezer_table.go index 51d7d1930..3fe691cf6 100644 --- a/core/rawdb/freezer_table.go +++ b/core/rawdb/freezer_table.go @@ -46,7 +46,7 @@ var ( errNotSupported = errors.New("this operation is not supported") ) -// indexEntry contains the number/id of the file that the data resides in, aswell as the +// indexEntry contains the number/id of the file that the data resides in, as well as the // offset within the file to the end of the data. // In serialized form, the filenum is stored as uint16. type indexEntry struct { diff --git a/core/state/snapshot/iterator_fast.go b/core/state/snapshot/iterator_fast.go index 48069b8fc..435c28e96 100644 --- a/core/state/snapshot/iterator_fast.go +++ b/core/state/snapshot/iterator_fast.go @@ -319,7 +319,7 @@ func (fi *fastIterator) Slot() []byte { } // Release iterates over all the remaining live layer iterators and releases each -// of thme individually. +// of them individually. func (fi *fastIterator) Release() { for _, it := range fi.iterators { it.it.Release() @@ -327,7 +327,7 @@ func (fi *fastIterator) Release() { fi.iterators = nil } -// Debug is a convencience helper during testing +// Debug is a convenience helper during testing func (fi *fastIterator) Debug() { for _, it := range fi.iterators { fmt.Printf("[p=%v v=%v] ", it.priority, it.it.Hash()[0]) diff --git a/core/state/snapshot/snapshot_test.go b/core/state/snapshot/snapshot_test.go index bc4e5cbd0..7c8077b65 100644 --- a/core/state/snapshot/snapshot_test.go +++ b/core/state/snapshot/snapshot_test.go @@ -265,7 +265,7 @@ func TestPostCapBasicDataAccess(t *testing.T) { snaps.Update(common.HexToHash("0xa3"), common.HexToHash("0xa2"), nil, setAccount("0xa3"), nil) snaps.Update(common.HexToHash("0xb3"), common.HexToHash("0xb2"), nil, setAccount("0xb3"), nil) - // checkExist verifies if an account exiss in a snapshot + // checkExist verifies if an account exists in a snapshot checkExist := func(layer *diffLayer, key string) error { if data, _ := layer.Account(common.HexToHash(key)); data == nil { return fmt.Errorf("expected %x to exist, got nil", common.HexToHash(key)) diff --git a/core/state/statedb.go b/core/state/statedb.go index 5c97dd94a..50eee8183 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -792,7 +792,7 @@ func (s *StateDB) Finalise(deleteEmptyObjects bool) { // If state snapshotting is active, also mark the destruction there. // Note, we can't do this only at the end of a block because multiple // transactions within the same block might self destruct and then - // ressurrect an account; but the snapshotter needs both events. + // resurrect an account; but the snapshotter needs both events. if s.snap != nil { s.snapDestructs[obj.addrHash] = struct{}{} // We need to maintain account deletions explicitly (will remain set indefinitely) delete(s.snapAccounts, obj.addrHash) // Clear out any previously updated account data (may be recreated via a ressurrect) @@ -891,7 +891,7 @@ func (s *StateDB) clearJournalAndRefund() { s.journal = newJournal() s.refund = 0 } - s.validRevisions = s.validRevisions[:0] // Snapshots can be created without journal entires + s.validRevisions = s.validRevisions[:0] // Snapshots can be created without journal entries } // Commit writes the state to the underlying in-memory trie database. @@ -938,7 +938,7 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { log.Crit("Failed to commit dirty codes", "error", err) } } - // Write the account trie changes, measuing the amount of wasted time + // Write the account trie changes, measuring the amount of wasted time var start time.Time if metrics.EnabledExpensive { start = time.Now() diff --git a/core/state/statedb_test.go b/core/state/statedb_test.go index 6af2d4523..092a4fb87 100644 --- a/core/state/statedb_test.go +++ b/core/state/statedb_test.go @@ -771,7 +771,7 @@ func TestStateDBAccessList(t *testing.T) { t.Fatalf("expected %x to be in access list", address) } } - // Check that only the expected addresses are present in the acesslist + // Check that only the expected addresses are present in the access list for address := range state.accessList.addresses { if _, exist := addressMap[address]; !exist { t.Fatalf("extra address %x in access list", address) diff --git a/core/state/sync_test.go b/core/state/sync_test.go index 95c79eaf3..3d9fe556d 100644 --- a/core/state/sync_test.go +++ b/core/state/sync_test.go @@ -305,8 +305,8 @@ func TestIterativeDelayedStateSync(t *testing.T) { } for len(nodeElements)+len(codeElements) > 0 { // Sync only half of the scheduled nodes - var nodeProcessd int - var codeProcessd int + var nodeProcessed int + var codeProcessed int if len(codeElements) > 0 { codeResults := make([]trie.CodeSyncResult, len(codeElements)/2+1) for i, element := range codeElements[:len(codeResults)] { @@ -321,7 +321,7 @@ func TestIterativeDelayedStateSync(t *testing.T) { t.Fatalf("failed to process result %v", err) } } - codeProcessd = len(codeResults) + codeProcessed = len(codeResults) } if len(nodeElements) > 0 { nodeResults := make([]trie.NodeSyncResult, len(nodeElements)/2+1) @@ -337,7 +337,7 @@ func TestIterativeDelayedStateSync(t *testing.T) { t.Fatalf("failed to process result %v", err) } } - nodeProcessd = len(nodeResults) + nodeProcessed = len(nodeResults) } batch := dstDb.NewBatch() if err := sched.Commit(batch); err != nil { @@ -346,7 +346,7 @@ func TestIterativeDelayedStateSync(t *testing.T) { batch.Write() paths, nodes, codes = sched.Missing(0) - nodeElements = nodeElements[nodeProcessd:] + nodeElements = nodeElements[nodeProcessed:] for i := 0; i < len(paths); i++ { nodeElements = append(nodeElements, stateElement{ path: paths[i], @@ -354,7 +354,7 @@ func TestIterativeDelayedStateSync(t *testing.T) { syncPath: trie.NewSyncPath([]byte(paths[i])), }) } - codeElements = codeElements[codeProcessd:] + codeElements = codeElements[codeProcessed:] for i := 0; i < len(codes); i++ { codeElements = append(codeElements, stateElement{ code: codes[i], diff --git a/core/state/trie_prefetcher.go b/core/state/trie_prefetcher.go index 0f6bce3b8..83e8966d4 100644 --- a/core/state/trie_prefetcher.go +++ b/core/state/trie_prefetcher.go @@ -212,7 +212,7 @@ type subfetcher struct { wake chan struct{} // Wake channel if a new task is scheduled stop chan struct{} // Channel to interrupt processing - term chan struct{} // Channel to signal iterruption + term chan struct{} // Channel to signal interruption copy chan chan Trie // Channel to request a copy of the current trie seen map[string]struct{} // Tracks the entries already loaded diff --git a/core/types/block.go b/core/types/block.go index 589a34cef..7525a88f5 100644 --- a/core/types/block.go +++ b/core/types/block.go @@ -317,7 +317,7 @@ func (b *Block) Header() *Header { return CopyHeader(b.header) } func (b *Block) Body() *Body { return &Body{b.transactions, b.uncles} } // Size returns the true RLP encoded storage size of the block, either by encoding -// and returning it, or returning a previsouly cached value. +// and returning it, or returning a previously cached value. func (b *Block) Size() common.StorageSize { if size := b.size.Load(); size != nil { return size.(common.StorageSize) diff --git a/core/types/block_test.go b/core/types/block_test.go index aa1db2f4f..9e7f581b1 100644 --- a/core/types/block_test.go +++ b/core/types/block_test.go @@ -314,7 +314,7 @@ func TestRlpDecodeParentHash(t *testing.T) { } // Also test a very very large header. { - // The rlp-encoding of the heder belowCauses _total_ length of 65540, + // The rlp-encoding of the header belowCauses _total_ length of 65540, // which is the first to blow the fast-path. h := &Header{ ParentHash: want, diff --git a/core/vm/runtime/runtime_test.go b/core/vm/runtime/runtime_test.go index 0fb287292..ab77e284d 100644 --- a/core/vm/runtime/runtime_test.go +++ b/core/vm/runtime/runtime_test.go @@ -457,7 +457,7 @@ func BenchmarkSimpleLoop(b *testing.B) { byte(vm.JUMP), } - calllRevertingContractWithInput := []byte{ + callRevertingContractWithInput := []byte{ byte(vm.JUMPDEST), // // push args for the call byte(vm.PUSH1), 0, // out size @@ -485,7 +485,7 @@ func BenchmarkSimpleLoop(b *testing.B) { benchmarkNonModifyingCode(100000000, loopingCode, "loop-100M", "", b) benchmarkNonModifyingCode(100000000, callInexistant, "call-nonexist-100M", "", b) benchmarkNonModifyingCode(100000000, callEOA, "call-EOA-100M", "", b) - benchmarkNonModifyingCode(100000000, calllRevertingContractWithInput, "call-reverting-100M", "", b) + benchmarkNonModifyingCode(100000000, callRevertingContractWithInput, "call-reverting-100M", "", b) //benchmarkNonModifyingCode(10000000, staticCallIdentity, "staticcall-identity-10M", b) //benchmarkNonModifyingCode(10000000, loopingCode, "loop-10M", b) diff --git a/crypto/bls12381/isogeny.go b/crypto/bls12381/isogeny.go index c3cb0a6f7..a63f585dd 100644 --- a/crypto/bls12381/isogeny.go +++ b/crypto/bls12381/isogeny.go @@ -19,7 +19,7 @@ package bls12381 // isogenyMapG1 applies 11-isogeny map for BLS12-381 G1 defined at draft-irtf-cfrg-hash-to-curve-06. func isogenyMapG1(x, y *fe) { // https://tools.ietf.org/html/draft-irtf-cfrg-hash-to-curve-06#appendix-C.2 - params := isogenyConstansG1 + params := isogenyConstantsG1 degree := 15 xNum, xDen, yNum, yDen := new(fe), new(fe), new(fe), new(fe) xNum.set(params[0][degree]) @@ -76,7 +76,7 @@ func isogenyMapG2(e *fp2, x, y *fe2) { y.set(yNum) } -var isogenyConstansG1 = [4][16]*fe{ +var isogenyConstantsG1 = [4][16]*fe{ { {0x4d18b6f3af00131c, 0x19fa219793fee28c, 0x3f2885f1467f19ae, 0x23dcea34f2ffb304, 0xd15b58d2ffc00054, 0x0913be200a20bef4}, {0x898985385cdbbd8b, 0x3c79e43cc7d966aa, 0x1597e193f4cd233a, 0x8637ef1e4d6623ad, 0x11b22deed20d827b, 0x07097bc5998784ad}, diff --git a/eth/catalyst/api.go b/eth/catalyst/api.go index 6b5f94646..358529459 100644 --- a/eth/catalyst/api.go +++ b/eth/catalyst/api.go @@ -539,7 +539,7 @@ func (api *ConsensusAPI) invalid(err error, latestValid *types.Header) beacon.Pa return beacon.PayloadStatusV1{Status: beacon.INVALID, LatestValidHash: ¤tHash, ValidationError: &errorMsg} } -// heatbeat loops indefinitely, and checks if there have been beacon client updates +// heartbeat loops indefinitely, and checks if there have been beacon client updates // received in the last while. If not - or if they but strange ones - it warns the // user that something might be off with their consensus node. // @@ -649,7 +649,7 @@ func (api *ConsensusAPI) heartbeat() { if eta == 0 { log.Warn(message) } else { - log.Warn(message, "eta", common.PrettyAge(time.Now().Add(-eta))) // weird hack, but duration formatted doens't handle days + log.Warn(message, "eta", common.PrettyAge(time.Now().Add(-eta))) // weird hack, but duration formatted doesn't handle days } offlineLogged = time.Now() } diff --git a/eth/downloader/api.go b/eth/downloader/api.go index b36dd6386..b3f7113bc 100644 --- a/eth/downloader/api.go +++ b/eth/downloader/api.go @@ -125,7 +125,7 @@ type SyncingResult struct { Status ethereum.SyncProgress `json:"status"` } -// uninstallSyncSubscriptionRequest uninstalles a syncing subscription in the API event loop. +// uninstallSyncSubscriptionRequest uninstalls a syncing subscription in the API event loop. type uninstallSyncSubscriptionRequest struct { c chan interface{} uninstalled chan interface{} diff --git a/eth/downloader/beaconsync.go b/eth/downloader/beaconsync.go index 773539258..484a4e20d 100644 --- a/eth/downloader/beaconsync.go +++ b/eth/downloader/beaconsync.go @@ -236,7 +236,7 @@ func (d *Downloader) findBeaconAncestor() (uint64, error) { // Binary search to find the ancestor start, end := beaconTail.Number.Uint64()-1, number if number := beaconHead.Number.Uint64(); end > number { - // This shouldn't really happen in a healty network, but if the consensus + // This shouldn't really happen in a healthy network, but if the consensus // clients feeds us a shorter chain as the canonical, we should not attempt // to access non-existent skeleton items. log.Warn("Beacon head lower than local chain", "beacon", number, "local", end) diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index f9ac8e487..c04352f0a 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -364,7 +364,7 @@ func (d *Downloader) synchronise(id string, hash common.Hash, td, ttd *big.Int, // The beacon header syncer is async. It will start this synchronization and // will continue doing other tasks. However, if synchronization needs to be // cancelled, the syncer needs to know if we reached the startup point (and - // inited the cancel cannel) or not yet. Make sure that we'll signal even in + // inited the cancel channel) or not yet. Make sure that we'll signal even in // case of a failure. if beaconPing != nil { defer func() { @@ -1461,7 +1461,7 @@ func (d *Downloader) processHeaders(origin uint64, td, ttd *big.Int, beaconMode } d.syncStatsLock.Unlock() - // Signal the content downloaders of the availablility of new tasks + // Signal the content downloaders of the availability of new tasks for _, ch := range []chan bool{d.queue.blockWakeCh, d.queue.receiptWakeCh} { select { case ch <- true: diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index 1c126bdae..450ed61ef 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -360,7 +360,7 @@ func (dlp *downloadTesterPeer) RequestAccountRange(id uint64, root, origin, limi } // RequestStorageRanges fetches a batch of storage slots belonging to one or -// more accounts. If slots from only one accout is requested, an origin marker +// more accounts. If slots from only one account is requested, an origin marker // may also be used to retrieve from there. func (dlp *downloadTesterPeer) RequestStorageRanges(id uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, bytes uint64) error { // Create the request and service it @@ -399,7 +399,7 @@ func (dlp *downloadTesterPeer) RequestByteCodes(id uint64, hashes []common.Hash, } // RequestTrieNodes fetches a batch of account or storage trie nodes rooted in -// a specificstate trie. +// a specific state trie. func (dlp *downloadTesterPeer) RequestTrieNodes(id uint64, root common.Hash, paths []snap.TrieNodePathSet, bytes uint64) error { req := &snap.GetTrieNodesPacket{ ID: id, @@ -571,8 +571,8 @@ func testForkedSync(t *testing.T, protocol uint, mode SyncMode) { assertOwnChain(t, tester, len(chainB.blocks)) } -// Tests that synchronising against a much shorter but much heavyer fork works -// corrently and is not dropped. +// Tests that synchronising against a much shorter but much heavier fork works +// currently and is not dropped. func TestHeavyForkedSync66Full(t *testing.T) { testHeavyForkedSync(t, eth.ETH66, FullSync) } func TestHeavyForkedSync66Snap(t *testing.T) { testHeavyForkedSync(t, eth.ETH66, SnapSync) } func TestHeavyForkedSync66Light(t *testing.T) { testHeavyForkedSync(t, eth.ETH66, LightSync) } diff --git a/eth/downloader/fetchers_concurrent.go b/eth/downloader/fetchers_concurrent.go index a0aa19717..44e6aa8f8 100644 --- a/eth/downloader/fetchers_concurrent.go +++ b/eth/downloader/fetchers_concurrent.go @@ -47,7 +47,7 @@ type typedQueue interface { // capacity is responsible for calculating how many items of the abstracted // type a particular peer is estimated to be able to retrieve within the - // alloted round trip time. + // allotted round trip time. capacity(peer *peerConnection, rtt time.Duration) int // updateCapacity is responsible for updating how many items of the abstracted @@ -58,7 +58,7 @@ type typedQueue interface { // from the download queue to the specified peer. reserve(peer *peerConnection, items int) (*fetchRequest, bool, bool) - // unreserve is resposible for removing the current retrieval allocation + // unreserve is responsible for removing the current retrieval allocation // assigned to a specific peer and placing it back into the pool to allow // reassigning to some other peer. unreserve(peer string) int @@ -190,7 +190,7 @@ func (d *Downloader) concurrentFetch(queue typedQueue, beaconMode bool) error { req, err := queue.request(peer, request, responses) if err != nil { // Sending the request failed, which generally means the peer - // was diconnected in between assignment and network send. + // was disconnected in between assignment and network send. // Although all peer removal operations return allocated tasks // to the queue, that is async, and we can do better here by // immediately pushing the unfulfilled requests. diff --git a/eth/downloader/fetchers_concurrent_bodies.go b/eth/downloader/fetchers_concurrent_bodies.go index a8de41032..e84206fe9 100644 --- a/eth/downloader/fetchers_concurrent_bodies.go +++ b/eth/downloader/fetchers_concurrent_bodies.go @@ -41,7 +41,7 @@ func (q *bodyQueue) pending() int { } // capacity is responsible for calculating how many bodies a particular peer is -// estimated to be able to retrieve within the alloted round trip time. +// estimated to be able to retrieve within the allotted round trip time. func (q *bodyQueue) capacity(peer *peerConnection, rtt time.Duration) int { return peer.BodyCapacity(rtt) } @@ -58,7 +58,7 @@ func (q *bodyQueue) reserve(peer *peerConnection, items int) (*fetchRequest, boo return q.queue.ReserveBodies(peer, items) } -// unreserve is resposible for removing the current body retrieval allocation +// unreserve is responsible for removing the current body retrieval allocation // assigned to a specific peer and placing it back into the pool to allow // reassigning to some other peer. func (q *bodyQueue) unreserve(peer string) int { diff --git a/eth/downloader/fetchers_concurrent_headers.go b/eth/downloader/fetchers_concurrent_headers.go index bd3bb3e00..84c7f2098 100644 --- a/eth/downloader/fetchers_concurrent_headers.go +++ b/eth/downloader/fetchers_concurrent_headers.go @@ -41,7 +41,7 @@ func (q *headerQueue) pending() int { } // capacity is responsible for calculating how many headers a particular peer is -// estimated to be able to retrieve within the alloted round trip time. +// estimated to be able to retrieve within the allotted round trip time. func (q *headerQueue) capacity(peer *peerConnection, rtt time.Duration) int { return peer.HeaderCapacity(rtt) } @@ -58,7 +58,7 @@ func (q *headerQueue) reserve(peer *peerConnection, items int) (*fetchRequest, b return q.queue.ReserveHeaders(peer, items), false, false } -// unreserve is resposible for removing the current header retrieval allocation +// unreserve is responsible for removing the current header retrieval allocation // assigned to a specific peer and placing it back into the pool to allow // reassigning to some other peer. func (q *headerQueue) unreserve(peer string) int { diff --git a/eth/downloader/fetchers_concurrent_receipts.go b/eth/downloader/fetchers_concurrent_receipts.go index fee2c3410..1c853c218 100644 --- a/eth/downloader/fetchers_concurrent_receipts.go +++ b/eth/downloader/fetchers_concurrent_receipts.go @@ -28,7 +28,7 @@ import ( // concurrent fetcher and the downloader. type receiptQueue Downloader -// waker returns a notification channel that gets pinged in case more reecipt +// waker returns a notification channel that gets pinged in case more receipt // fetches have been queued up, so the fetcher might assign it to idle peers. func (q *receiptQueue) waker() chan bool { return q.queue.receiptWakeCh @@ -41,7 +41,7 @@ func (q *receiptQueue) pending() int { } // capacity is responsible for calculating how many receipts a particular peer is -// estimated to be able to retrieve within the alloted round trip time. +// estimated to be able to retrieve within the allotted round trip time. func (q *receiptQueue) capacity(peer *peerConnection, rtt time.Duration) int { return peer.ReceiptCapacity(rtt) } @@ -58,7 +58,7 @@ func (q *receiptQueue) reserve(peer *peerConnection, items int) (*fetchRequest, return q.queue.ReserveReceipts(peer, items) } -// unreserve is resposible for removing the current receipt retrieval allocation +// unreserve is responsible for removing the current receipt retrieval allocation // assigned to a specific peer and placing it back into the pool to allow // reassigning to some other peer. func (q *receiptQueue) unreserve(peer string) int { diff --git a/eth/downloader/queue.go b/eth/downloader/queue.go index de5708be3..a8d2ea83a 100644 --- a/eth/downloader/queue.go +++ b/eth/downloader/queue.go @@ -859,7 +859,7 @@ func (q *queue) deliver(id string, taskPool map[common.Hash]*types.Header, if res, stale, err := q.resultCache.GetDeliverySlot(header.Number.Uint64()); err == nil { reconstruct(accepted, res) } else { - // else: betweeen here and above, some other peer filled this result, + // else: between here and above, some other peer filled this result, // or it was indeed a no-op. This should not happen, but if it does it's // not something to panic about log.Error("Delivery stale", "stale", stale, "number", header.Number.Uint64(), "err", err) diff --git a/eth/downloader/skeleton.go b/eth/downloader/skeleton.go index be4e8fbfc..e627c6ae5 100644 --- a/eth/downloader/skeleton.go +++ b/eth/downloader/skeleton.go @@ -51,7 +51,7 @@ const requestHeaders = 512 // errSyncLinked is an internal helper error to signal that the current sync // cycle linked up to the genesis block, this the skeleton syncer should ping // the backfiller to resume. Since we already have that logic on sync start, -// piggie-back on that instead of 2 entrypoints. +// piggy-back on that instead of 2 entrypoints. var errSyncLinked = errors.New("sync linked") // errSyncMerged is an internal helper error to signal that the current sync @@ -148,7 +148,7 @@ type backfiller interface { // suspend requests the backfiller to abort any running full or snap sync // based on the skeleton chain as it might be invalid. The backfiller should // gracefully handle multiple consecutive suspends without a resume, even - // on initial sartup. + // on initial startup. // // The method should return the last block header that has been successfully // backfilled, or nil if the backfiller was not resumed. @@ -209,7 +209,7 @@ type skeleton struct { headEvents chan *headUpdate // Notification channel for new heads terminate chan chan error // Termination channel to abort sync - terminated chan struct{} // Channel to signal that the syner is dead + terminated chan struct{} // Channel to signal that the syncer is dead // Callback hooks used during testing syncStarting func() // callback triggered after a sync cycle is inited but before started @@ -553,7 +553,7 @@ func (s *skeleton) initSync(head *types.Header) { return } } - // Either we've failed to decode the previus state, or there was none. Start + // Either we've failed to decode the previous state, or there was none. Start // a fresh sync with a single subchain represented by the currently sent // chain head. s.progress = &skeletonProgress{ @@ -823,7 +823,7 @@ func (s *skeleton) executeTask(peer *peerConnection, req *headerRequest) { } } -// revertRequests locates all the currently pending reuqests from a particular +// revertRequests locates all the currently pending requests from a particular // peer and reverts them, rescheduling for others to fulfill. func (s *skeleton) revertRequests(peer string) { // Gather the requests first, revertals need the lock too @@ -871,7 +871,7 @@ func (s *skeleton) revertRequest(req *headerRequest) { delete(s.requests, req.id) // Remove the request from the tracked set and mark the task as not-pending, - // ready for resheduling + // ready for rescheduling s.scratchOwners[(s.scratchHead-req.head)/requestHeaders] = "" } diff --git a/eth/downloader/skeleton_test.go b/eth/downloader/skeleton_test.go index 421925718..6bcbac3a8 100644 --- a/eth/downloader/skeleton_test.go +++ b/eth/downloader/skeleton_test.go @@ -53,7 +53,7 @@ func newHookedBackfiller() backfiller { // suspend requests the backfiller to abort any running full or snap sync // based on the skeleton chain as it might be invalid. The backfiller should // gracefully handle multiple consecutive suspends without a resume, even -// on initial sartup. +// on initial startup. func (hf *hookedBackfiller) suspend() *types.Header { if hf.suspendHook != nil { hf.suspendHook() @@ -111,7 +111,7 @@ func newSkeletonTestPeerWithHook(id string, headers []*types.Header, serve func( // function can be used to retrieve batches of headers from the particular peer. func (p *skeletonTestPeer) RequestHeadersByNumber(origin uint64, amount int, skip int, reverse bool, sink chan *eth.Response) (*eth.Request, error) { // Since skeleton test peer are in-memory mocks, dropping the does not make - // them inaccepssible. As such, check a local `dropped` field to see if the + // them inaccessible. As such, check a local `dropped` field to see if the // peer has been dropped and should not respond any more. if atomic.LoadUint64(&p.dropped) != 0 { return nil, errors.New("peer already dropped") @@ -204,7 +204,7 @@ func (p *skeletonTestPeer) RequestReceipts([]common.Hash, chan *eth.Response) (* panic("skeleton sync must not request receipts") } -// Tests various sync initialzations based on previous leftovers in the database +// Tests various sync initializations based on previous leftovers in the database // and announced heads. func TestSkeletonSyncInit(t *testing.T) { // Create a few key headers @@ -227,7 +227,7 @@ func TestSkeletonSyncInit(t *testing.T) { newstate: []*subchain{{Head: 50, Tail: 50}}, }, // Empty database with only the genesis set with a leftover empty sync - // progess. This is a synthetic case, just for the sake of covering things. + // progress. This is a synthetic case, just for the sake of covering things. { oldstate: []*subchain{}, head: block50, @@ -533,13 +533,13 @@ func TestSkeletonSyncRetrievals(t *testing.T) { peers []*skeletonTestPeer // Initial peer set to start the sync with midstate []*subchain // Expected sync state after initial cycle midserve uint64 // Expected number of header retrievals after initial cycle - middrop uint64 // Expectd number of peers dropped after initial cycle + middrop uint64 // Expected number of peers dropped after initial cycle - newHead *types.Header // New header to annount on top of the old one + newHead *types.Header // New header to anoint on top of the old one newPeer *skeletonTestPeer // New peer to join the skeleton syncer endstate []*subchain // Expected sync state after the post-init event endserve uint64 // Expected number of header retrievals after the post-init event - enddrop uint64 // Expectd number of peers dropped after the post-init event + enddrop uint64 // Expected number of peers dropped after the post-init event }{ // Completely empty database with only the genesis set. The sync is expected // to create a single subchain with the requested head. No peers however, so diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 7ba2faf79..d1323de7b 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -196,7 +196,7 @@ type Config struct { RPCEVMTimeout time.Duration // RPCTxFeeCap is the global transaction fee(price * gaslimit) cap for - // send-transction variants. The unit is ether. + // send-transaction variants. The unit is ether. RPCTxFeeCap float64 // Checkpoint is a hardcoded checkpoint which can be nil. diff --git a/eth/fetcher/tx_fetcher.go b/eth/fetcher/tx_fetcher.go index a23cd24bf..035e0c2ec 100644 --- a/eth/fetcher/tx_fetcher.go +++ b/eth/fetcher/tx_fetcher.go @@ -120,7 +120,7 @@ type txDelivery struct { direct bool // Whether this is a direct reply or a broadcast } -// txDrop is the notiication that a peer has disconnected. +// txDrop is the notification that a peer has disconnected. type txDrop struct { peer string } @@ -260,7 +260,7 @@ func (f *TxFetcher) Notify(peer string, hashes []common.Hash) error { // Enqueue imports a batch of received transaction into the transaction pool // and the fetcher. This method may be called by both transaction broadcasts and // direct request replies. The differentiation is important so the fetcher can -// re-shedule missing transactions as soon as possible. +// re-schedule missing transactions as soon as possible. func (f *TxFetcher) Enqueue(peer string, txs []*types.Transaction, direct bool) error { // Keep track of all the propagated transactions if direct { @@ -558,7 +558,7 @@ func (f *TxFetcher) loop() { // In case of a direct delivery, also reschedule anything missing // from the original query if delivery.direct { - // Mark the reqesting successful (independent of individual status) + // Mark the requesting successful (independent of individual status) txRequestDoneMeter.Mark(int64(len(delivery.hashes))) // Make sure something was pending, nuke it @@ -607,7 +607,7 @@ func (f *TxFetcher) loop() { delete(f.alternates, hash) delete(f.fetching, hash) } - // Something was delivered, try to rechedule requests + // Something was delivered, try to reschedule requests f.scheduleFetches(timeoutTimer, timeoutTrigger, nil) // Partial delivery may enable others to deliver too } @@ -719,7 +719,7 @@ func (f *TxFetcher) rescheduleWait(timer *mclock.Timer, trigger chan struct{}) { // should be rescheduled if some request is pending. In practice, a timeout will // cause the timer to be rescheduled every 5 secs (until the peer comes through or // disconnects). This is a limitation of the fetcher code because we don't trac -// pending requests and timed out requests separatey. Without double tracking, if +// pending requests and timed out requests separately. Without double tracking, if // we simply didn't reschedule the timer on all-timeout then the timer would never // be set again since len(request) > 0 => something's running. func (f *TxFetcher) rescheduleTimeout(timer *mclock.Timer, trigger chan struct{}) { diff --git a/eth/fetcher/tx_fetcher_test.go b/eth/fetcher/tx_fetcher_test.go index ce8d02af7..4c06712f7 100644 --- a/eth/fetcher/tx_fetcher_test.go +++ b/eth/fetcher/tx_fetcher_test.go @@ -1011,7 +1011,7 @@ func TestTransactionFetcherOutOfBoundDeliveries(t *testing.T) { } // Tests that dropping a peer cleans out all internal data structures in all the -// live or danglng stages. +// live or dangling stages. func TestTransactionFetcherDrop(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ init: func() *TxFetcher { @@ -1121,7 +1121,7 @@ func TestTransactionFetcherDropRescheduling(t *testing.T) { } // This test reproduces a crash caught by the fuzzer. The root cause was a -// dangling transaction timing out and clashing on readd with a concurrently +// dangling transaction timing out and clashing on re-add with a concurrently // announced one. func TestTransactionFetcherFuzzCrash01(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ @@ -1148,7 +1148,7 @@ func TestTransactionFetcherFuzzCrash01(t *testing.T) { } // This test reproduces a crash caught by the fuzzer. The root cause was a -// dangling transaction getting peer-dropped and clashing on readd with a +// dangling transaction getting peer-dropped and clashing on re-add with a // concurrently announced one. func TestTransactionFetcherFuzzCrash02(t *testing.T) { testTransactionFetcherParallel(t, txFetcherTest{ diff --git a/eth/filters/api.go b/eth/filters/api.go index 07714791d..3b8933d0a 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -36,7 +36,7 @@ import ( // and associated subscription in the event system. type filter struct { typ Type - deadline *time.Timer // filter is inactiv when deadline triggers + deadline *time.Timer // filter is inactive when deadline triggers hashes []common.Hash crit FilterCriteria logs []*types.Log diff --git a/eth/peerset.go b/eth/peerset.go index 3e54a481e..b9cc1e03a 100644 --- a/eth/peerset.go +++ b/eth/peerset.go @@ -41,7 +41,7 @@ var ( errPeerNotRegistered = errors.New("peer not registered") // errSnapWithoutEth is returned if a peer attempts to connect only on the - // snap protocol without advertizing the eth main protocol. + // snap protocol without advertising the eth main protocol. errSnapWithoutEth = errors.New("peer connected on snap without compatible eth support") ) diff --git a/eth/protocols/eth/broadcast.go b/eth/protocols/eth/broadcast.go index 09330cfdf..6fc15f136 100644 --- a/eth/protocols/eth/broadcast.go +++ b/eth/protocols/eth/broadcast.go @@ -36,7 +36,7 @@ type blockPropagation struct { td *big.Int } -// broadcastBlocks is a write loop that multiplexes blocks and block accouncements +// broadcastBlocks is a write loop that multiplexes blocks and block announcements // to the remote peer. The goal is to have an async writer that does not lock up // node internals and at the same time rate limits queued data. func (p *Peer) broadcastBlocks() { diff --git a/eth/protocols/eth/dispatcher.go b/eth/protocols/eth/dispatcher.go index bf88d400d..65a935d55 100644 --- a/eth/protocols/eth/dispatcher.go +++ b/eth/protocols/eth/dispatcher.go @@ -224,7 +224,7 @@ func (p *Peer) dispatcher() { switch { case res.Req == nil: // Response arrived with an untracked ID. Since even cancelled - // requests are tracked until fulfilment, a dangling repsponse + // requests are tracked until fulfilment, a dangling response // means the remote peer implements the protocol badly. resOp.fail <- errDanglingResponse diff --git a/eth/protocols/eth/handler_test.go b/eth/protocols/eth/handler_test.go index bf836e8f5..2707a420b 100644 --- a/eth/protocols/eth/handler_test.go +++ b/eth/protocols/eth/handler_test.go @@ -94,7 +94,7 @@ func (b *testBackend) Chain() *core.BlockChain { return b.chain } func (b *testBackend) TxPool() TxPool { return b.txpool } func (b *testBackend) RunPeer(peer *Peer, handler Handler) error { - // Normally the backend would do peer mainentance and handshakes. All that + // Normally the backend would do peer maintenance and handshakes. All that // is omitted and we will just give control back to the handler. return handler(peer) } diff --git a/eth/protocols/eth/peer.go b/eth/protocols/eth/peer.go index 22674d65b..a23726384 100644 --- a/eth/protocols/eth/peer.go +++ b/eth/protocols/eth/peer.go @@ -133,7 +133,7 @@ func (p *Peer) ID() string { return p.id } -// Version retrieves the peer's negoatiated `eth` protocol version. +// Version retrieves the peer's negotiated `eth` protocol version. func (p *Peer) Version() uint { return p.version } diff --git a/eth/protocols/snap/handler.go b/eth/protocols/snap/handler.go index 77bd96f46..41380d96f 100644 --- a/eth/protocols/snap/handler.go +++ b/eth/protocols/snap/handler.go @@ -504,7 +504,7 @@ func ServiceGetTrieNodesQuery(chain *core.BlockChain, req *GetTrieNodesPacket, s var ( nodes [][]byte bytes uint64 - loads int // Trie hash expansions to cound database reads + loads int // Trie hash expansions to count database reads ) for _, pathset := range req.Paths { switch len(pathset) { diff --git a/eth/protocols/snap/peer.go b/eth/protocols/snap/peer.go index 87a62d2f8..235d499ff 100644 --- a/eth/protocols/snap/peer.go +++ b/eth/protocols/snap/peer.go @@ -61,12 +61,12 @@ func (p *Peer) ID() string { return p.id } -// Version retrieves the peer's negoatiated `snap` protocol version. +// Version retrieves the peer's negotiated `snap` protocol version. func (p *Peer) Version() uint { return p.version } -// Log overrides the P2P logget with the higher level one containing only the id. +// Log overrides the P2P logger with the higher level one containing only the id. func (p *Peer) Log() log.Logger { return p.logger } @@ -87,7 +87,7 @@ func (p *Peer) RequestAccountRange(id uint64, root common.Hash, origin, limit co } // RequestStorageRange fetches a batch of storage slots belonging to one or more -// accounts. If slots from only one accout is requested, an origin marker may also +// accounts. If slots from only one account is requested, an origin marker may also // be used to retrieve from there. func (p *Peer) RequestStorageRanges(id uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, bytes uint64) error { if len(accounts) == 1 && origin != nil { @@ -119,7 +119,7 @@ func (p *Peer) RequestByteCodes(id uint64, hashes []common.Hash, bytes uint64) e } // RequestTrieNodes fetches a batch of account or storage trie nodes rooted in -// a specificstate trie. +// a specific state trie. func (p *Peer) RequestTrieNodes(id uint64, root common.Hash, paths []TrieNodePathSet, bytes uint64) error { p.logger.Trace("Fetching set of trie nodes", "reqid", id, "root", root, "pathsets", len(paths), "bytes", common.StorageSize(bytes)) diff --git a/eth/protocols/snap/sync.go b/eth/protocols/snap/sync.go index b2462f5f8..deaa4456e 100644 --- a/eth/protocols/snap/sync.go +++ b/eth/protocols/snap/sync.go @@ -365,7 +365,7 @@ type SyncPeer interface { RequestAccountRange(id uint64, root, origin, limit common.Hash, bytes uint64) error // RequestStorageRanges fetches a batch of storage slots belonging to one or - // more accounts. If slots from only one accout is requested, an origin marker + // more accounts. If slots from only one account is requested, an origin marker // may also be used to retrieve from there. RequestStorageRanges(id uint64, root common.Hash, accounts []common.Hash, origin, limit []byte, bytes uint64) error @@ -373,7 +373,7 @@ type SyncPeer interface { RequestByteCodes(id uint64, hashes []common.Hash, bytes uint64) error // RequestTrieNodes fetches a batch of account or storage trie nodes rooted in - // a specificstate trie. + // a specific state trie. RequestTrieNodes(id uint64, root common.Hash, paths []TrieNodePathSet, bytes uint64) error // Log retrieves the peer's own contextual logger. @@ -1183,10 +1183,10 @@ func (s *Syncer) assignStorageTasks(success chan *storageResponse, fail chan *st } if subtask == nil { // No large contract required retrieval, but small ones available - for acccount, root := range task.stateTasks { - delete(task.stateTasks, acccount) + for account, root := range task.stateTasks { + delete(task.stateTasks, account) - accounts = append(accounts, acccount) + accounts = append(accounts, account) roots = append(roots, root) if len(accounts) >= storageSets { @@ -1486,7 +1486,7 @@ func (s *Syncer) assignBytecodeHealTasks(success chan *bytecodeHealResponse, fai } } -// revertRequests locates all the currently pending reuqests from a particular +// revertRequests locates all the currently pending requests from a particular // peer and reverts them, rescheduling for others to fulfill. func (s *Syncer) revertRequests(peer string) { // Gather the requests first, revertals need the lock too @@ -1575,7 +1575,7 @@ func (s *Syncer) revertAccountRequest(req *accountRequest) { s.lock.Unlock() // If there's a timeout timer still running, abort it and mark the account - // task as not-pending, ready for resheduling + // task as not-pending, ready for rescheduling req.timeout.Stop() if req.task.req == req { req.task.req = nil @@ -1616,7 +1616,7 @@ func (s *Syncer) revertBytecodeRequest(req *bytecodeRequest) { s.lock.Unlock() // If there's a timeout timer still running, abort it and mark the code - // retrievals as not-pending, ready for resheduling + // retrievals as not-pending, ready for rescheduling req.timeout.Stop() for _, hash := range req.hashes { req.task.codeTasks[hash] = struct{}{} @@ -1657,7 +1657,7 @@ func (s *Syncer) revertStorageRequest(req *storageRequest) { s.lock.Unlock() // If there's a timeout timer still running, abort it and mark the storage - // task as not-pending, ready for resheduling + // task as not-pending, ready for rescheduling req.timeout.Stop() if req.subTask != nil { req.subTask.req = nil @@ -1743,7 +1743,7 @@ func (s *Syncer) revertBytecodeHealRequest(req *bytecodeHealRequest) { s.lock.Unlock() // If there's a timeout timer still running, abort it and mark the code - // retrievals as not-pending, ready for resheduling + // retrievals as not-pending, ready for rescheduling req.timeout.Stop() for _, hash := range req.hashes { req.task.codeTasks[hash] = struct{}{} @@ -2035,7 +2035,7 @@ func (s *Syncer) processStorageResponse(res *storageResponse) { } tr.Commit() } - // Persist the received storage segements. These flat state maybe + // Persist the received storage segments. These flat state maybe // outdated during the sync, but it can be fixed later during the // snapshot generation. for j := 0; j < len(res.hashes[i]); j++ { @@ -2170,7 +2170,7 @@ func (s *Syncer) forwardAccountTask(task *accountTask) { } task.res = nil - // Persist the received account segements. These flat state maybe + // Persist the received account segments. These flat state maybe // outdated during the sync, but it can be fixed later during the // snapshot generation. oldAccountBytes := s.accountBytes @@ -2773,7 +2773,7 @@ func (s *Syncer) onHealByteCodes(peer SyncPeer, id uint64, bytecodes [][]byte) e } // onHealState is a callback method to invoke when a flat state(account -// or storage slot) is downloded during the healing stage. The flat states +// or storage slot) is downloaded during the healing stage. The flat states // can be persisted blindly and can be fixed later in the generation stage. // Note it's not concurrent safe, please handle the concurrent issue outside. func (s *Syncer) onHealState(paths [][]byte, value []byte) error { diff --git a/eth/state_accessor.go b/eth/state_accessor.go index f01db93a6..12dba8a0a 100644 --- a/eth/state_accessor.go +++ b/eth/state_accessor.go @@ -44,7 +44,7 @@ import ( // perform Commit or other 'save-to-disk' changes, this should be set to false to avoid // storing trash persistently // - preferDisk: this arg can be used by the caller to signal that even though the 'base' is provided, -// it would be preferrable to start from a fresh state, if we have it on disk. +// it would be preferable to start from a fresh state, if we have it on disk. func (eth *Ethereum) StateAtBlock(block *types.Block, reexec uint64, base *state.StateDB, checkLive bool, preferDisk bool) (statedb *state.StateDB, err error) { var ( current *types.Block diff --git a/eth/tracers/api.go b/eth/tracers/api.go index 014e2f6ad..092950e78 100644 --- a/eth/tracers/api.go +++ b/eth/tracers/api.go @@ -116,7 +116,7 @@ func (context *chainContext) GetHeader(hash common.Hash, number uint64) *types.H return header } -// chainContext construts the context reader which is used by the evm for reading +// chainContext constructs the context reader which is used by the evm for reading // the necessary chain context. func (api *API) chainContext(ctx context.Context) core.ChainContext { return &chainContext{api: api, ctx: ctx} @@ -202,10 +202,10 @@ type blockTraceTask struct { statedb *state.StateDB // Intermediate state prepped for tracing block *types.Block // Block to trace the transactions from rootref common.Hash // Trie root reference held for this task - results []*txTraceResult // Trace results procudes by the task + results []*txTraceResult // Trace results produced by the task } -// blockTraceResult represets the results of tracing a single block when an entire +// blockTraceResult represents the results of tracing a single block when an entire // chain is being traced. type blockTraceResult struct { Block hexutil.Uint64 `json:"block"` // Block number corresponding to this trace @@ -563,7 +563,7 @@ func (api *API) StandardTraceBadBlockToFile(ctx context.Context, hash common.Has // traceBlock configures a new tracer according to the provided configuration, and // executes all the transactions contained within. The return value will be one item -// per transaction, dependent on the requestd tracer. +// per transaction, dependent on the requested tracer. func (api *API) traceBlock(ctx context.Context, block *types.Block, config *TraceConfig) ([]*txTraceResult, error) { if block.NumberU64() == 0 { return nil, errors.New("genesis is not traceable") @@ -707,7 +707,7 @@ func (api *API) standardTraceBlockToFile(ctx context.Context, block *types.Block } } for i, tx := range block.Transactions() { - // Prepare the trasaction for un-traced execution + // Prepare the transaction for un-traced execution var ( msg, _ = tx.AsMessage(signer, block.BaseFee()) txContext = core.NewEVMTxContext(msg) diff --git a/eth/tracers/internal/tracetest/calltrace_test.go b/eth/tracers/internal/tracetest/calltrace_test.go index cabddac49..d2c50656d 100644 --- a/eth/tracers/internal/tracetest/calltrace_test.go +++ b/eth/tracers/internal/tracetest/calltrace_test.go @@ -39,7 +39,7 @@ import ( "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/tests" - // Force-load native and js pacakges, to trigger registration + // Force-load native and js packages, to trigger registration _ "github.com/ethereum/go-ethereum/eth/tracers/js" _ "github.com/ethereum/go-ethereum/eth/tracers/native" ) diff --git a/ethclient/ethclient_test.go b/ethclient/ethclient_test.go index d78a12fae..f2f4a5765 100644 --- a/ethclient/ethclient_test.go +++ b/ethclient/ethclient_test.go @@ -581,7 +581,7 @@ func testCallContract(t *testing.T, client *rpc.Client) { if _, err := ec.CallContract(context.Background(), msg, big.NewInt(1)); err != nil { t.Fatalf("unexpected error: %v", err) } - // PendingCallCOntract + // PendingCallContract if _, err := ec.PendingCallContract(context.Background(), msg); err != nil { t.Fatalf("unexpected error: %v", err) } diff --git a/ethdb/memorydb/memorydb.go b/ethdb/memorydb/memorydb.go index e94570cb3..7e4fd7e5e 100644 --- a/ethdb/memorydb/memorydb.go +++ b/ethdb/memorydb/memorydb.go @@ -66,7 +66,7 @@ func NewWithCap(size int) *Database { } // Close deallocates the internal map and ensures any consecutive data access op -// failes with an error. +// fails with an error. func (db *Database) Close() error { db.lock.Lock() defer db.lock.Unlock() diff --git a/interfaces.go b/interfaces.go index e8d24a57c..eb9af6007 100644 --- a/interfaces.go +++ b/interfaces.go @@ -204,7 +204,7 @@ type GasPricer interface { // FeeHistory provides recent fee market data that consumers can use to determine // a reasonable maxPriorityFeePerGas value. type FeeHistory struct { - OldestBlock *big.Int // block coresponding to first response value + OldestBlock *big.Int // block corresponding to first response value Reward [][]*big.Int // list every txs priority fee per block BaseFee []*big.Int // list of each block's base fee GasUsedRatio []float64 // ratio of gas used out of the total available limit diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 939dd6939..e6740942d 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -988,7 +988,7 @@ func newRevertError(result *core.ExecutionResult) *revertError { } } -// revertError is an API error that encompassas an EVM revertal with JSON error +// revertError is an API error that encompasses an EVM revertal with JSON error // code and a binary data blob. type revertError struct { error diff --git a/les/downloader/api.go b/les/downloader/api.go index b1a81b6b7..21200b676 100644 --- a/les/downloader/api.go +++ b/les/downloader/api.go @@ -125,7 +125,7 @@ type SyncingResult struct { Status ethereum.SyncProgress `json:"status"` } -// uninstallSyncSubscriptionRequest uninstalles a syncing subscription in the API event loop. +// uninstallSyncSubscriptionRequest uninstalls a syncing subscription in the API event loop. type uninstallSyncSubscriptionRequest struct { c chan interface{} uninstalled chan interface{} diff --git a/les/downloader/downloader.go b/les/downloader/downloader.go index b9da76d34..6352b1c21 100644 --- a/les/downloader/downloader.go +++ b/les/downloader/downloader.go @@ -1625,7 +1625,7 @@ func (d *Downloader) processHeaders(origin uint64, td *big.Int) error { log.Warn("Invalid header encountered", "number", chunk[n].Number, "hash", chunk[n].Hash(), "parent", chunk[n].ParentHash, "err", err) return fmt.Errorf("%w: %v", errInvalidChain, err) } - // All verifications passed, track all headers within the alloted limits + // All verifications passed, track all headers within the allotted limits if mode == FastSync { head := chunk[len(chunk)-1].Number.Uint64() if head-rollback > uint64(fsHeaderSafetyNet) { @@ -1663,7 +1663,7 @@ func (d *Downloader) processHeaders(origin uint64, td *big.Int) error { } d.syncStatsLock.Unlock() - // Signal the content downloaders of the availablility of new tasks + // Signal the content downloaders of the availability of new tasks for _, ch := range []chan bool{d.bodyWakeCh, d.receiptWakeCh} { select { case ch <- true: diff --git a/les/downloader/downloader_test.go b/les/downloader/downloader_test.go index 5eea49877..c56870ff1 100644 --- a/les/downloader/downloader_test.go +++ b/les/downloader/downloader_test.go @@ -653,8 +653,8 @@ func testForkedSync(t *testing.T, protocol uint, mode SyncMode) { assertOwnForkedChain(t, tester, testChainBase.len(), []int{chainA.len(), chainB.len()}) } -// Tests that synchronising against a much shorter but much heavyer fork works -// corrently and is not dropped. +// Tests that synchronising against a much shorter but much heavier fork works +// correctly and is not dropped. func TestHeavyForkedSync66Full(t *testing.T) { testHeavyForkedSync(t, eth.ETH66, FullSync) } func TestHeavyForkedSync66Fast(t *testing.T) { testHeavyForkedSync(t, eth.ETH66, FastSync) } func TestHeavyForkedSync66Light(t *testing.T) { testHeavyForkedSync(t, eth.ETH66, LightSync) } diff --git a/les/downloader/queue.go b/les/downloader/queue.go index 5744b3ee2..b165b6b5c 100644 --- a/les/downloader/queue.go +++ b/les/downloader/queue.go @@ -872,7 +872,7 @@ func (q *queue) deliver(id string, taskPool map[common.Hash]*types.Header, if res, stale, err := q.resultCache.GetDeliverySlot(header.Number.Uint64()); err == nil { reconstruct(accepted, res) } else { - // else: betweeen here and above, some other peer filled this result, + // else: between here and above, some other peer filled this result, // or it was indeed a no-op. This should not happen, but if it does it's // not something to panic about log.Error("Delivery stale", "stale", stale, "number", header.Number.Uint64(), "err", err) diff --git a/les/fetcher.go b/les/fetcher.go index cf62c8f70..6861eebcf 100644 --- a/les/fetcher.go +++ b/les/fetcher.go @@ -252,7 +252,7 @@ func (f *lightFetcher) forEachPeer(check func(id enode.ID, p *fetcherPeer) bool) // request will be made for header retrieval. // // - re-sync trigger -// If the local chain lags too much, then the fetcher will enter "synnchronise" +// If the local chain lags too much, then the fetcher will enter "synchronise" // mode to retrieve missing headers in batch. func (f *lightFetcher) mainloop() { defer f.wg.Done() diff --git a/les/flowcontrol/manager.go b/les/flowcontrol/manager.go index 4ffbee58f..4367974d6 100644 --- a/les/flowcontrol/manager.go +++ b/les/flowcontrol/manager.go @@ -55,7 +55,7 @@ var ( // ClientManager controls the capacity assigned to the clients of a server. // Since ServerParams guarantee a safe lower estimate for processable requests // even in case of all clients being active, ClientManager calculates a -// corrigated buffer value and usually allows a higher remaining buffer value +// corrugated buffer value and usually allows a higher remaining buffer value // to be returned with each reply. type ClientManager struct { clock mclock.Clock diff --git a/les/odr.go b/les/odr.go index 10ff0854d..2643a5347 100644 --- a/les/odr.go +++ b/les/odr.go @@ -126,7 +126,7 @@ const ( // RetrieveTxStatus retrieves the transaction status from the LES network. // There is no guarantee in the LES protocol that the mined transaction will // be retrieved back for sure because of different reasons(the transaction -// is unindexed, the malicous server doesn't reply it deliberately, etc). +// is unindexed, the malicious server doesn't reply it deliberately, etc). // Therefore, unretrieved transactions(UNKNOWN) will receive a certain number // of retries, thus giving a weak guarantee. func (odr *LesOdr) RetrieveTxStatus(ctx context.Context, req *light.TxStatusRequest) error { diff --git a/les/vflux/client/fillset_test.go b/les/vflux/client/fillset_test.go index ca5af8f07..ddb12a82f 100644 --- a/les/vflux/client/fillset_test.go +++ b/les/vflux/client/fillset_test.go @@ -104,7 +104,7 @@ func TestFillSet(t *testing.T) { fs.SetTarget(10) expWaiting(4, true) expNotWaiting() - // remove all previosly set flags + // remove all previously set flags ns.ForEach(sfTest1, nodestate.Flags{}, func(node *enode.Node, state nodestate.Flags) { ns.SetState(node, nodestate.Flags{}, sfTest1, 0) }) diff --git a/les/vflux/client/serverpool_test.go b/les/vflux/client/serverpool_test.go index 9f83c5f7f..f1fd987d7 100644 --- a/les/vflux/client/serverpool_test.go +++ b/les/vflux/client/serverpool_test.go @@ -66,7 +66,7 @@ type ServerPoolTest struct { // (accessed from both the main thread and the preNeg callback) preNegLock sync.Mutex queryWg *sync.WaitGroup // a new wait group is created each time the simulation is started - stopping bool // stopping avoid callind queryWg.Add after queryWg.Wait + stopping bool // stopping avoid calling queryWg.Add after queryWg.Wait cycle, conn, servedConn int serviceCycles, dialCount int diff --git a/les/vflux/server/balance.go b/les/vflux/server/balance.go index 550c6d70c..b09f7bb50 100644 --- a/les/vflux/server/balance.go +++ b/les/vflux/server/balance.go @@ -623,7 +623,7 @@ func (n *nodeBalance) priorityToBalance(priority int64, capacity uint64) (uint64 return 0, uint64(-priority) } -// reducedBalance estimates the reduced balance at a given time in the fututre based +// reducedBalance estimates the reduced balance at a given time in the future based // on the given balance, the time factor and an estimated average request cost per time ratio func (n *nodeBalance) reducedBalance(b balance, start mclock.AbsTime, dt time.Duration, capacity uint64, avgReqCost float64) balance { // since the costs are applied continuously during the dt time period we calculate diff --git a/les/vflux/server/balance_test.go b/les/vflux/server/balance_test.go index 9f253cabf..7c100aab5 100644 --- a/les/vflux/server/balance_test.go +++ b/les/vflux/server/balance_test.go @@ -54,7 +54,7 @@ func newBalanceTestSetup(db ethdb.KeyValueStore, posExp, negExp utils.ValueExpir // Initialize and customize the setup for the balance testing clock := &mclock.Simulated{} setup := newServerSetup() - setup.clientField = setup.setup.NewField("balancTestClient", reflect.TypeOf(balanceTestClient{})) + setup.clientField = setup.setup.NewField("balanceTestClient", reflect.TypeOf(balanceTestClient{})) ns := nodestate.NewNodeStateMachine(nil, nil, clock, setup.setup) if posExp == nil { @@ -298,7 +298,7 @@ func TestEstimatedPriority(t *testing.T) { } } -func TestPostiveBalanceCounting(t *testing.T) { +func TestPositiveBalanceCounting(t *testing.T) { b := newBalanceTestSetup(nil, nil, nil) defer b.stop() diff --git a/les/vflux/server/status.go b/les/vflux/server/status.go index 469190777..2d7e25b68 100644 --- a/les/vflux/server/status.go +++ b/les/vflux/server/status.go @@ -41,7 +41,7 @@ type serverSetup struct { activeFlag nodestate.Flags // Flag is set if the node is active inactiveFlag nodestate.Flags // Flag is set if the node is inactive capacityField nodestate.Field // Field contains the capacity of the node - queueField nodestate.Field // Field contains the infomration in the priority queue + queueField nodestate.Field // Field contains the information in the priority queue } // newServerSetup initializes the setup for state machine and returns the flags/fields group. diff --git a/light/lightchain.go b/light/lightchain.go index 2a8e36721..dca97ce45 100644 --- a/light/lightchain.go +++ b/light/lightchain.go @@ -397,7 +397,7 @@ func (lc *LightChain) SetCanonical(header *types.Header) error { // // The verify parameter can be used to fine tune whether nonce verification // should be done or not. The reason behind the optional check is because some -// of the header retrieval mechanisms already need to verfy nonces, as well as +// of the header retrieval mechanisms already need to verify nonces, as well as // because nonces can be verified sparsely, not needing to check each. // // In the case of a light chain, InsertHeaderChain also creates and posts light diff --git a/light/odr_util.go b/light/odr_util.go index bbbcdbce2..48631139b 100644 --- a/light/odr_util.go +++ b/light/odr_util.go @@ -272,9 +272,9 @@ func GetBloomBits(ctx context.Context, odr OdrBackend, bit uint, sections []uint // GetTransaction retrieves a canonical transaction by hash and also returns // its position in the chain. There is no guarantee in the LES protocol that // the mined transaction will be retrieved back for sure because of different -// reasons(the transaction is unindexed, the malicous server doesn't reply it +// reasons(the transaction is unindexed, the malicious server doesn't reply it // deliberately, etc). Therefore, unretrieved transactions will receive a certain -// number of retrys, thus giving a weak guarantee. +// number of retries, thus giving a weak guarantee. func GetTransaction(ctx context.Context, odr OdrBackend, txHash common.Hash) (*types.Transaction, common.Hash, uint64, uint64, error) { r := &TxStatusRequest{Hashes: []common.Hash{txHash}} if err := odr.RetrieveTxStatus(ctx, r); err != nil || r.Status[0].Status != core.TxStatusIncluded { diff --git a/light/postprocess.go b/light/postprocess.go index 0e50dab96..3f9da6593 100644 --- a/light/postprocess.go +++ b/light/postprocess.go @@ -313,7 +313,7 @@ var ( BloomTrieTablePrefix = "blt-" ) -// GetBloomTrieRoot reads the BloomTrie root assoctiated to the given section from the database +// GetBloomTrieRoot reads the BloomTrie root associated to the given section from the database func GetBloomTrieRoot(db ethdb.Database, sectionIdx uint64, sectionHead common.Hash) common.Hash { var encNumber [8]byte binary.BigEndian.PutUint64(encNumber[:], sectionIdx) @@ -321,7 +321,7 @@ func GetBloomTrieRoot(db ethdb.Database, sectionIdx uint64, sectionHead common.H return common.BytesToHash(data) } -// StoreBloomTrieRoot writes the BloomTrie root assoctiated to the given section into the database +// StoreBloomTrieRoot writes the BloomTrie root associated to the given section into the database func StoreBloomTrieRoot(db ethdb.Database, sectionIdx uint64, sectionHead, root common.Hash) { var encNumber [8]byte binary.BigEndian.PutUint64(encNumber[:], sectionIdx) diff --git a/light/trie.go b/light/trie.go index f60edaa3b..b88265e87 100644 --- a/light/trie.go +++ b/light/trie.go @@ -153,7 +153,7 @@ func (t *odrTrie) TryDelete(key []byte) error { }) } -// TryDeleteACcount abstracts an account deletion from the trie. +// TryDeleteAccount abstracts an account deletion from the trie. func (t *odrTrie) TryDeleteAccount(key []byte) error { key = crypto.Keccak256(key) return t.do(key, func() error { diff --git a/light/txpool.go b/light/txpool.go index 413337208..b3e1a62e1 100644 --- a/light/txpool.go +++ b/light/txpool.go @@ -71,7 +71,7 @@ type TxPool struct { eip2718 bool // Fork indicator whether we are in the eip2718 stage. } -// TxRelayBackend provides an interface to the mechanism that forwards transacions +// TxRelayBackend provides an interface to the mechanism that forwards transactions // to the ETH network. The implementations of the functions should be non-blocking. // // Send instructs backend to forward new transactions diff --git a/metrics/gauge_float64_test.go b/metrics/gauge_float64_test.go index 02b75580c..7b854d232 100644 --- a/metrics/gauge_float64_test.go +++ b/metrics/gauge_float64_test.go @@ -2,7 +2,7 @@ package metrics import "testing" -func BenchmarkGuageFloat64(b *testing.B) { +func BenchmarkGaugeFloat64(b *testing.B) { g := NewGaugeFloat64() b.ResetTimer() for i := 0; i < b.N; i++ { diff --git a/metrics/gauge_test.go b/metrics/gauge_test.go index 3aee14345..a98fe985d 100644 --- a/metrics/gauge_test.go +++ b/metrics/gauge_test.go @@ -5,7 +5,7 @@ import ( "testing" ) -func BenchmarkGuage(b *testing.B) { +func BenchmarkGauge(b *testing.B) { g := NewGauge() b.ResetTimer() for i := 0; i < b.N; i++ { diff --git a/metrics/prometheus/prometheus.go b/metrics/prometheus/prometheus.go index 9ad5ec7e9..c8408d8ca 100644 --- a/metrics/prometheus/prometheus.go +++ b/metrics/prometheus/prometheus.go @@ -36,7 +36,7 @@ func Handler(reg metrics.Registry) http.Handler { }) sort.Strings(names) - // Aggregate all the metris into a Prometheus collector + // Aggregate all the metrics into a Prometheus collector c := newCollector() for _, name := range names { diff --git a/miner/unconfirmed_test.go b/miner/unconfirmed_test.go index dc83cb926..60958f658 100644 --- a/miner/unconfirmed_test.go +++ b/miner/unconfirmed_test.go @@ -74,7 +74,7 @@ func TestUnconfirmedShifts(t *testing.T) { if n := pool.blocks.Len(); n != int(limit)/2 { t.Errorf("unconfirmed count mismatch: have %d, want %d", n, limit/2) } - // Try to shift all the remaining blocks out and verify emptyness + // Try to shift all the remaining blocks out and verify emptiness pool.Shift(start + 2*uint64(limit)) if n := pool.blocks.Len(); n != 0 { t.Errorf("unconfirmed count mismatch: have %d, want %d", n, 0) diff --git a/miner/worker_test.go b/miner/worker_test.go index bda0fd489..ec5ba67e1 100644 --- a/miner/worker_test.go +++ b/miner/worker_test.go @@ -494,7 +494,7 @@ func testAdjustInterval(t *testing.T, chainConfig *params.ChainConfig, engine co } w.start() - time.Sleep(time.Second) // Ensure two tasks have been summitted due to start opt + time.Sleep(time.Second) // Ensure two tasks have been submitted due to start opt atomic.StoreUint32(&start, 1) w.setRecommitInterval(3 * time.Second) diff --git a/mobile/accounts.go b/mobile/accounts.go index 4d979bfff..d9eab93a7 100644 --- a/mobile/accounts.go +++ b/mobile/accounts.go @@ -212,10 +212,10 @@ func (ks *KeyStore) ImportECDSAKey(key []byte, passphrase string) (account *Acco // ImportPreSaleKey decrypts the given Ethereum presale wallet and stores // a key file in the key directory. The key file is encrypted with the same passphrase. -func (ks *KeyStore) ImportPreSaleKey(keyJSON []byte, passphrase string) (ccount *Account, _ error) { - account, err := ks.keystore.ImportPreSaleKey(common.CopyBytes(keyJSON), passphrase) +func (ks *KeyStore) ImportPreSaleKey(keyJSON []byte, passphrase string) (account *Account, _ error) { + acc, err := ks.keystore.ImportPreSaleKey(common.CopyBytes(keyJSON), passphrase) if err != nil { return nil, err } - return &Account{account}, nil + return &Account{acc}, nil } diff --git a/mobile/init.go b/mobile/init.go index 2025d85ed..94f5baf28 100644 --- a/mobile/init.go +++ b/mobile/init.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -// Contains initialization code for the mbile library. +// Contains initialization code for the mobile library. package geth diff --git a/node/rpcstack_test.go b/node/rpcstack_test.go index 6fb16c504..09acf7ea0 100644 --- a/node/rpcstack_test.go +++ b/node/rpcstack_test.go @@ -100,7 +100,7 @@ func TestWebsocketOrigins(t *testing.T) { expFail: []string{ "test", // no scheme, required by spec "http://test", // wrong scheme - "http://test.foo", "https://a.test.x", // subdomain variatoins + "http://test.foo", "https://a.test.x", // subdomain variations "http://testx:8540", "https://xtest:8540"}, }, // ip tests diff --git a/p2p/discover/v4_udp.go b/p2p/discover/v4_udp.go index 95a6df8e1..67cd2c004 100644 --- a/p2p/discover/v4_udp.go +++ b/p2p/discover/v4_udp.go @@ -525,7 +525,7 @@ func (t *UDPv4) readLoop(unhandled chan<- ReadPacket) { t.log.Debug("Temporary UDP read error", "err", err) continue } else if err != nil { - // Shut down the loop for permament errors. + // Shut down the loop for permanent errors. if !errors.Is(err, io.EOF) { t.log.Debug("UDP read error", "err", err) } diff --git a/p2p/discover/v5_udp.go b/p2p/discover/v5_udp.go index 6ffa7bef7..071ed65ad 100644 --- a/p2p/discover/v5_udp.go +++ b/p2p/discover/v5_udp.go @@ -625,7 +625,7 @@ func (t *UDPv5) readLoop() { t.log.Debug("Temporary UDP read error", "err", err) continue } else if err != nil { - // Shut down the loop for permament errors. + // Shut down the loop for permanent errors. if !errors.Is(err, io.EOF) { t.log.Debug("UDP read error", "err", err) } diff --git a/p2p/msgrate/msgrate.go b/p2p/msgrate/msgrate.go index 5bfa27b43..d4e0eb8b5 100644 --- a/p2p/msgrate/msgrate.go +++ b/p2p/msgrate/msgrate.go @@ -111,7 +111,7 @@ const tuningImpact = 0.25 // local link is saturated. In that case, the live measurements will force us // to reduce request sizes until the throughput gets stable. // -// Lastly, message rate measurements allows us to detect if a peer is unsuaully +// Lastly, message rate measurements allows us to detect if a peer is unusually // slow compared to other peers, in which case we can decide to keep it around // or free up the slot so someone closer. // @@ -127,7 +127,7 @@ type Tracker struct { // in their sizes. // // Callers of course are free to use the item counter as a byte counter if - // or when their protocol of choise if capped by bytes instead of items. + // or when their protocol of choice if capped by bytes instead of items. // (eg. eth.getHeaders vs snap.getAccountRange). capacity map[uint64]float64 @@ -157,7 +157,7 @@ func NewTracker(caps map[uint64]float64, rtt time.Duration) *Tracker { } // Capacity calculates the number of items the peer is estimated to be able to -// retrieve within the alloted time slot. The method will round up any division +// retrieve within the allotted time slot. The method will round up any division // errors and will add an additional overestimation ratio on top. The reason for // overshooting the capacity is because certain message types might not increase // the load proportionally to the requested items, so fetching a bit more might diff --git a/p2p/tracker/tracker.go b/p2p/tracker/tracker.go index 69a49087e..6a733b9ba 100644 --- a/p2p/tracker/tracker.go +++ b/p2p/tracker/tracker.go @@ -121,7 +121,7 @@ func (t *Tracker) Track(peer string, version uint, reqCode uint64, resCode uint6 } // clean is called automatically when a preset time passes without a response -// being dleivered for the first network request. +// being delivered for the first network request. func (t *Tracker) clean() { t.lock.Lock() defer t.lock.Unlock() diff --git a/rpc/server.go b/rpc/server.go index babc5688e..bf1f71a28 100644 --- a/rpc/server.go +++ b/rpc/server.go @@ -160,7 +160,7 @@ type PeerInfo struct { // Address of client. This will usually contain the IP address and port. RemoteAddr string - // Addditional information for HTTP and WebSocket connections. + // Additional information for HTTP and WebSocket connections. HTTP struct { // Protocol version, i.e. "HTTP/1.1". This is not set for WebSocket. Version string diff --git a/signer/rules/rules_test.go b/signer/rules/rules_test.go index 32901e2ff..c35da8ecc 100644 --- a/signer/rules/rules_test.go +++ b/signer/rules/rules_test.go @@ -44,7 +44,7 @@ Three things can happen: 3. Anything else; other return values [*], method not implemented or exception occurred during processing. This means that the operation will continue to manual processing, via the regular UI method chosen by the user. -[*] Note: Future version of the ruleset may use more complex json-based returnvalues, making it possible to not +[*] Note: Future version of the ruleset may use more complex json-based return values, making it possible to not only respond Approve/Reject/Manual, but also modify responses. For example, choose to list only one, but not all accounts in a list-request. The points above will continue to hold for non-json based responses ("Approve"/"Reject"). @@ -242,7 +242,7 @@ func (d *dummyUI) OnApprovedTx(tx ethapi.SignTransactionResult) { func (d *dummyUI) OnSignerStartup(info core.StartupInfo) { } -//TestForwarding tests that the rule-engine correctly dispatches requests to the next caller +// TestForwarding tests that the rule-engine correctly dispatches requests to the next caller func TestForwarding(t *testing.T) { js := "" ui := &dummyUI{make([]string, 0)} @@ -434,7 +434,7 @@ func dummyTx(value hexutil.Big) *core.SignTxRequest { Gas: gas, }, Callinfo: []apitypes.ValidationInfo{ - {Typ: "Warning", Message: "All your base are bellong to us"}, + {Typ: "Warning", Message: "All your base are belong to us"}, }, Meta: core.Metadata{Remote: "remoteip", Local: "localip", Scheme: "inproc"}, } @@ -536,7 +536,7 @@ func (d *dontCallMe) OnApprovedTx(tx ethapi.SignTransactionResult) { d.t.Fatalf("Did not expect next-handler to be called") } -//TestContextIsCleared tests that the rule-engine does not retain variables over several requests. +// TestContextIsCleared tests that the rule-engine does not retain variables over several requests. // if it does, that would be bad since developers may rely on that to store data, // instead of using the disk-based data storage func TestContextIsCleared(t *testing.T) { diff --git a/signer/storage/aes_gcm_storage.go b/signer/storage/aes_gcm_storage.go index f09bfa7d4..928d643dd 100644 --- a/signer/storage/aes_gcm_storage.go +++ b/signer/storage/aes_gcm_storage.go @@ -143,7 +143,7 @@ func (s *AESEncryptedStorage) writeEncryptedStorage(creds map[string]storedCrede // encrypt encrypts plaintext with the given key, with additional data // The 'additionalData' is used to place the (plaintext) KV-store key into the V, -// to prevent the possibility to alter a K, or swap two entries in the KV store with eachother. +// to prevent the possibility to alter a K, or swap two entries in the KV store with each other. func encrypt(key []byte, plaintext []byte, additionalData []byte) ([]byte, []byte, error) { block, err := aes.NewCipher(key) if err != nil { diff --git a/trie/hasher.go b/trie/hasher.go index 9e17d639f..183e96c22 100644 --- a/trie/hasher.go +++ b/trie/hasher.go @@ -191,7 +191,7 @@ func (h *hasher) hashData(data []byte) hashNode { } // proofHash is used to construct trie proofs, and returns the 'collapsed' -// node (for later RLP encoding) aswell as the hashed node -- unless the +// node (for later RLP encoding) as well as the hashed node -- unless the // node is smaller than 32 bytes, in which case it will be returned as is. // This method does not do anything on value- or hash-nodes. func (h *hasher) proofHash(original node) (collapsed, hashed node) { diff --git a/trie/proof_test.go b/trie/proof_test.go index 8db035256..61667b20a 100644 --- a/trie/proof_test.go +++ b/trie/proof_test.go @@ -205,7 +205,7 @@ func TestRangeProofWithNonExistentProof(t *testing.T) { proof := memorydb.New() // Short circuit if the decreased key is same with the previous key - first := decreseKey(common.CopyBytes(entries[start].k)) + first := decreaseKey(common.CopyBytes(entries[start].k)) if start != 0 && bytes.Equal(first, entries[start-1].k) { continue } @@ -214,7 +214,7 @@ func TestRangeProofWithNonExistentProof(t *testing.T) { continue } // Short circuit if the increased key is same with the next key - last := increseKey(common.CopyBytes(entries[end-1].k)) + last := increaseKey(common.CopyBytes(entries[end-1].k)) if end != len(entries) && bytes.Equal(last, entries[end].k) { continue } @@ -274,7 +274,7 @@ func TestRangeProofWithInvalidNonExistentProof(t *testing.T) { // Case 1 start, end := 100, 200 - first := decreseKey(common.CopyBytes(entries[start].k)) + first := decreaseKey(common.CopyBytes(entries[start].k)) proof := memorydb.New() if err := trie.Prove(first, 0, proof); err != nil { @@ -297,7 +297,7 @@ func TestRangeProofWithInvalidNonExistentProof(t *testing.T) { // Case 2 start, end = 100, 200 - last := increseKey(common.CopyBytes(entries[end-1].k)) + last := increaseKey(common.CopyBytes(entries[end-1].k)) proof = memorydb.New() if err := trie.Prove(entries[start].k, 0, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) @@ -343,7 +343,7 @@ func TestOneElementRangeProof(t *testing.T) { // One element with left non-existent edge proof start = 1000 - first := decreseKey(common.CopyBytes(entries[start].k)) + first := decreaseKey(common.CopyBytes(entries[start].k)) proof = memorydb.New() if err := trie.Prove(first, 0, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) @@ -358,7 +358,7 @@ func TestOneElementRangeProof(t *testing.T) { // One element with right non-existent edge proof start = 1000 - last := increseKey(common.CopyBytes(entries[start].k)) + last := increaseKey(common.CopyBytes(entries[start].k)) proof = memorydb.New() if err := trie.Prove(entries[start].k, 0, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) @@ -373,7 +373,7 @@ func TestOneElementRangeProof(t *testing.T) { // One element with two non-existent edge proofs start = 1000 - first, last = decreseKey(common.CopyBytes(entries[start].k)), increseKey(common.CopyBytes(entries[start].k)) + first, last = decreaseKey(common.CopyBytes(entries[start].k)), increaseKey(common.CopyBytes(entries[start].k)) proof = memorydb.New() if err := trie.Prove(first, 0, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) @@ -641,9 +641,9 @@ func TestSameSideProofs(t *testing.T) { sort.Sort(entries) pos := 1000 - first := decreseKey(common.CopyBytes(entries[pos].k)) - first = decreseKey(first) - last := decreseKey(common.CopyBytes(entries[pos].k)) + first := decreaseKey(common.CopyBytes(entries[pos].k)) + first = decreaseKey(first) + last := decreaseKey(common.CopyBytes(entries[pos].k)) proof := memorydb.New() if err := trie.Prove(first, 0, proof); err != nil { @@ -657,9 +657,9 @@ func TestSameSideProofs(t *testing.T) { t.Fatalf("Expected error, got nil") } - first = increseKey(common.CopyBytes(entries[pos].k)) - last = increseKey(common.CopyBytes(entries[pos].k)) - last = increseKey(last) + first = increaseKey(common.CopyBytes(entries[pos].k)) + last = increaseKey(common.CopyBytes(entries[pos].k)) + last = increaseKey(last) proof = memorydb.New() if err := trie.Prove(first, 0, proof); err != nil { @@ -765,7 +765,7 @@ func TestEmptyRangeProof(t *testing.T) { } for _, c := range cases { proof := memorydb.New() - first := increseKey(common.CopyBytes(entries[c.pos].k)) + first := increaseKey(common.CopyBytes(entries[c.pos].k)) if err := trie.Prove(first, 0, proof); err != nil { t.Fatalf("Failed to prove the first node %v", err) } @@ -904,7 +904,7 @@ func mutateByte(b []byte) { } } -func increseKey(key []byte) []byte { +func increaseKey(key []byte) []byte { for i := len(key) - 1; i >= 0; i-- { key[i]++ if key[i] != 0x0 { @@ -914,7 +914,7 @@ func increseKey(key []byte) []byte { return key } -func decreseKey(key []byte) []byte { +func decreaseKey(key []byte) []byte { for i := len(key) - 1; i >= 0; i-- { key[i]-- if key[i] != 0xff { diff --git a/trie/secure_trie_test.go b/trie/secure_trie_test.go index 05bddb21b..862c3a3ec 100644 --- a/trie/secure_trie_test.go +++ b/trie/secure_trie_test.go @@ -121,7 +121,7 @@ func TestStateTrieConcurrency(t *testing.T) { for i := 0; i < threads; i++ { tries[i] = trie.Copy() } - // Start a batch of goroutines interactng with the trie + // Start a batch of goroutines interacting with the trie pend := new(sync.WaitGroup) pend.Add(threads) for i := 0; i < threads; i++ { From fa1305f8bf1b1e6c4eadf4d12c9dd8ce7ddc4c6c Mon Sep 17 00:00:00 2001 From: ucwong Date: Fri, 19 Aug 2022 14:01:09 +0800 Subject: [PATCH 47/59] internal/ethapi: fix comment typo (#25548) --- internal/ethapi/transaction_args.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/ethapi/transaction_args.go b/internal/ethapi/transaction_args.go index 787ac6577..e07248db5 100644 --- a/internal/ethapi/transaction_args.go +++ b/internal/ethapi/transaction_args.go @@ -238,7 +238,7 @@ func (args *TransactionArgs) ToMessage(globalGasCap uint64, baseFee *big.Int) (t gasPrice = args.GasPrice.ToInt() gasFeeCap, gasTipCap = gasPrice, gasPrice } else { - // User specified 1559 gas feilds (or none), use those + // User specified 1559 gas fields (or none), use those gasFeeCap = new(big.Int) if args.MaxFeePerGas != nil { gasFeeCap = args.MaxFeePerGas.ToInt() From 32e849061559d2c15d8c9f8752317b3df7ed267d Mon Sep 17 00:00:00 2001 From: ucwong Date: Fri, 19 Aug 2022 14:01:43 +0800 Subject: [PATCH 48/59] accounts/abi/bind/backends: typo fix (#25549) --- accounts/abi/bind/backends/simulated.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index cd14afa14..e03f2e120 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -609,7 +609,7 @@ func (b *SimulatedBackend) callContract(ctx context.Context, call ethereum.CallM // User specified the legacy gas field, convert to 1559 gas typing call.GasFeeCap, call.GasTipCap = call.GasPrice, call.GasPrice } else { - // User specified 1559 gas feilds (or none), use those + // User specified 1559 gas fields (or none), use those if call.GasFeeCap == nil { call.GasFeeCap = new(big.Int) } From 656dc8cc0034d3dd57b5e1baecb788d9c4e9929d Mon Sep 17 00:00:00 2001 From: Justin Traglia <95511699+jtraglia@users.noreply.github.com> Date: Fri, 19 Aug 2022 01:02:47 -0500 Subject: [PATCH 49/59] eth, les: unlock downloader peerSet if there's an error (#25546) Unlock peerSet if there's an error in the downloader --- eth/downloader/peer.go | 1 + les/downloader/peer.go | 1 + 2 files changed, 2 insertions(+) diff --git a/eth/downloader/peer.go b/eth/downloader/peer.go index d74d23e74..6b8269495 100644 --- a/eth/downloader/peer.go +++ b/eth/downloader/peer.go @@ -237,6 +237,7 @@ func (ps *peerSet) Register(p *peerConnection) error { } p.rates = msgrate.NewTracker(ps.rates.MeanCapacities(), ps.rates.MedianRoundTrip()) if err := ps.rates.Track(p.id, p.rates); err != nil { + ps.lock.Unlock() return err } ps.peers[p.id] = p diff --git a/les/downloader/peer.go b/les/downloader/peer.go index 5a92e9cf9..c2161e2da 100644 --- a/les/downloader/peer.go +++ b/les/downloader/peer.go @@ -350,6 +350,7 @@ func (ps *peerSet) Register(p *peerConnection) error { } p.rates = msgrate.NewTracker(ps.rates.MeanCapacities(), ps.rates.MedianRoundTrip()) if err := ps.rates.Track(p.id, p.rates); err != nil { + ps.lock.Unlock() return err } ps.peers[p.id] = p From 9762ddf8b0e9d471600c99f158479912f4870c52 Mon Sep 17 00:00:00 2001 From: Justin Traglia <95511699+jtraglia@users.noreply.github.com> Date: Fri, 19 Aug 2022 01:03:45 -0500 Subject: [PATCH 50/59] cmd/geth: parse uint64 value with ParseUint instead of Atoi (#25545) Parse uint64 value with ParseUint instead of Atoi --- cmd/geth/chaincmd.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index 878637506..a3016c4b0 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -384,12 +384,12 @@ func parseDumpConfig(ctx *cli.Context, stack *node.Node) (*state.DumpConfig, eth return nil, nil, common.Hash{}, fmt.Errorf("block %x not found", hash) } } else { - number, err := strconv.Atoi(arg) + number, err := strconv.ParseUint(arg, 10, 64) if err != nil { return nil, nil, common.Hash{}, err } - if hash := rawdb.ReadCanonicalHash(db, uint64(number)); hash != (common.Hash{}) { - header = rawdb.ReadHeader(db, hash, uint64(number)) + if hash := rawdb.ReadCanonicalHash(db, number); hash != (common.Hash{}) { + header = rawdb.ReadHeader(db, hash, number) } else { return nil, nil, common.Hash{}, fmt.Errorf("header for block %d not found", number) } From 77308cd6fceb99e276c1e2753d92ff82c6f1a962 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Fri, 19 Aug 2022 10:37:53 +0200 Subject: [PATCH 51/59] consensus/beacon: check ttd reached on pos blocks (#25552) * consensus/beacon: check ttd reached on pos blocks * consensus/beacon: check ttd reached on pos blocks * consensus/beacon: check ttd reached on pos blocks --- consensus/beacon/consensus.go | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/consensus/beacon/consensus.go b/consensus/beacon/consensus.go index 0397b026f..949c8ad81 100644 --- a/consensus/beacon/consensus.go +++ b/consensus/beacon/consensus.go @@ -114,8 +114,16 @@ func (beacon *Beacon) VerifyHeaders(chain consensus.ChainHeaderReader, headers [ } } - // All the headers have passed the transition point, use new rules. if len(preHeaders) == 0 { + // All the headers are pos headers. Verify that the parent block reached total terminal difficulty. + if reached, _ := IsTTDReached(chain, headers[0].ParentHash, headers[0].Number.Uint64()-1); !reached { + // TTD not reached for the first block, mark subsequent with invalid terminal block + results := make(chan error, len(headers)) + for i := 0; i < len(headers); i++ { + results <- consensus.ErrInvalidTerminalBlock + } + return make(chan struct{}), results + } return beacon.verifyHeaders(chain, headers, nil) } From 36874b63a1b56eed2b8e4b47ccea0337920b84b8 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi <1591639+s1na@users.noreply.github.com> Date: Fri, 19 Aug 2022 11:14:59 +0200 Subject: [PATCH 52/59] eth/filters: add global block logs cache (#25459) This adds a cache for block logs which is shared by all filters. The cache size of is configurable using the `--cache.blocklogs` flag. Co-authored-by: Felix Lange --- accounts/abi/bind/backends/simulated.go | 31 ++++---- cmd/geth/config.go | 11 ++- cmd/geth/main.go | 1 + cmd/utils/flags.go | 34 +++++++-- eth/api_backend.go | 14 +--- eth/backend.go | 5 -- eth/ethconfig/config.go | 4 + eth/ethconfig/gen_config.go | 6 ++ eth/filters/api.go | 20 ++--- eth/filters/bench_test.go | 16 ++-- eth/filters/filter.go | 97 +++++++++++-------------- eth/filters/filter_system.go | 85 +++++++++++++++++++++- eth/filters/filter_system_test.go | 77 +++++++++----------- eth/filters/filter_test.go | 20 ++--- ethclient/gethclient/gethclient_test.go | 7 ++ graphql/graphql.go | 39 ++++++++-- graphql/graphql_test.go | 9 ++- graphql/service.go | 9 ++- internal/ethapi/backend.go | 14 ++-- les/api_backend.go | 7 +- les/client.go | 4 - 21 files changed, 310 insertions(+), 200 deletions(-) diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index e03f2e120..0ce752103 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -68,7 +68,8 @@ type SimulatedBackend struct { pendingState *state.StateDB // Currently pending state that will be the active on request pendingReceipts types.Receipts // Currently receipts for the pending block - events *filters.EventSystem // Event system for filtering log events live + events *filters.EventSystem // for filtering log events live + filterSystem *filters.FilterSystem // for filtering database logs config *params.ChainConfig } @@ -86,7 +87,11 @@ func NewSimulatedBackendWithDatabase(database ethdb.Database, alloc core.Genesis blockchain: blockchain, config: genesis.Config, } - backend.events = filters.NewEventSystem(&filterBackend{database, blockchain, backend}, false) + + filterBackend := &filterBackend{database, blockchain, backend} + backend.filterSystem = filters.NewFilterSystem(filterBackend, filters.Config{}) + backend.events = filters.NewEventSystem(backend.filterSystem, false) + backend.rollback(blockchain.CurrentBlock()) return backend } @@ -689,7 +694,7 @@ func (b *SimulatedBackend) FilterLogs(ctx context.Context, query ethereum.Filter var filter *filters.Filter if query.BlockHash != nil { // Block filter requested, construct a single-shot filter - filter = filters.NewBlockFilter(&filterBackend{b.database, b.blockchain, b}, *query.BlockHash, query.Addresses, query.Topics) + filter = b.filterSystem.NewBlockFilter(*query.BlockHash, query.Addresses, query.Topics) } else { // Initialize unset filter boundaries to run from genesis to chain head from := int64(0) @@ -701,7 +706,7 @@ func (b *SimulatedBackend) FilterLogs(ctx context.Context, query ethereum.Filter to = query.ToBlock.Int64() } // Construct the range filter - filter = filters.NewRangeFilter(&filterBackend{b.database, b.blockchain, b}, from, to, query.Addresses, query.Topics) + filter = b.filterSystem.NewRangeFilter(from, to, query.Addresses, query.Topics) } // Run the filter and return all the logs logs, err := filter.Logs(ctx) @@ -827,7 +832,8 @@ type filterBackend struct { backend *SimulatedBackend } -func (fb *filterBackend) ChainDb() ethdb.Database { return fb.db } +func (fb *filterBackend) ChainDb() ethdb.Database { return fb.db } + func (fb *filterBackend) EventMux() *event.TypeMux { panic("not supported") } func (fb *filterBackend) HeaderByNumber(ctx context.Context, block rpc.BlockNumber) (*types.Header, error) { @@ -853,19 +859,8 @@ func (fb *filterBackend) GetReceipts(ctx context.Context, hash common.Hash) (typ return rawdb.ReadReceipts(fb.db, hash, *number, fb.bc.Config()), nil } -func (fb *filterBackend) GetLogs(ctx context.Context, hash common.Hash) ([][]*types.Log, error) { - number := rawdb.ReadHeaderNumber(fb.db, hash) - if number == nil { - return nil, nil - } - receipts := rawdb.ReadReceipts(fb.db, hash, *number, fb.bc.Config()) - if receipts == nil { - return nil, nil - } - logs := make([][]*types.Log, len(receipts)) - for i, receipt := range receipts { - logs[i] = receipt.Logs - } +func (fb *filterBackend) GetLogs(ctx context.Context, hash common.Hash, number uint64) ([][]*types.Log, error) { + logs := rawdb.ReadLogs(fb.db, hash, number, fb.bc.Config()) return logs, nil } diff --git a/cmd/geth/config.go b/cmd/geth/config.go index 2562de8ae..30565fda6 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -163,7 +163,9 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) { override := ctx.Bool(utils.OverrideTerminalTotalDifficultyPassed.Name) cfg.Eth.OverrideTerminalTotalDifficultyPassed = &override } + backend, eth := utils.RegisterEthService(stack, &cfg.Eth) + // Warn users to migrate if they have a legacy freezer format. if eth != nil && !ctx.IsSet(utils.IgnoreLegacyReceiptsFlag.Name) { firstIdx := uint64(0) @@ -181,10 +183,15 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) { utils.Fatalf("Database has receipts with a legacy format. Please run `geth db freezer-migrate`.") } } - // Configure GraphQL if requested + + // Configure log filter RPC API. + filterSystem := utils.RegisterFilterAPI(stack, backend, &cfg.Eth) + + // Configure GraphQL if requested. if ctx.IsSet(utils.GraphQLEnabledFlag.Name) { - utils.RegisterGraphQLService(stack, backend, cfg.Node) + utils.RegisterGraphQLService(stack, backend, filterSystem, &cfg.Node) } + // Add the Ethereum Stats daemon if requested. if cfg.Ethstats.URL != "" { utils.RegisterEthStatsService(stack, backend, cfg.Ethstats.URL) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index c0f636fb2..b9e3ed31e 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -117,6 +117,7 @@ var ( utils.CacheSnapshotFlag, utils.CacheNoPrefetchFlag, utils.CachePreimagesFlag, + utils.CacheLogSizeFlag, utils.FDLimitFlag, utils.ListenPortFlag, utils.DiscoveryPortFlag, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index efea93492..9e9519334 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -43,6 +43,7 @@ import ( ethcatalyst "github.com/ethereum/go-ethereum/eth/catalyst" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/ethconfig" + "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/eth/gasprice" "github.com/ethereum/go-ethereum/eth/tracers" "github.com/ethereum/go-ethereum/ethdb" @@ -64,6 +65,7 @@ import ( "github.com/ethereum/go-ethereum/p2p/nat" "github.com/ethereum/go-ethereum/p2p/netutil" "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/rpc" pcsclite "github.com/gballet/go-libpcsclite" gopsutil "github.com/shirou/gopsutil/mem" "github.com/urfave/cli/v2" @@ -491,6 +493,12 @@ var ( Usage: "Enable recording the SHA3/keccak preimages of trie keys", Category: flags.PerfCategory, } + CacheLogSizeFlag = &cli.IntFlag{ + Name: "cache.blocklogs", + Usage: "Size (in number of blocks) of the log cache for filtering", + Category: flags.PerfCategory, + Value: ethconfig.Defaults.FilterLogCacheSize, + } FDLimitFlag = &cli.IntFlag{ Name: "fdlimit", Usage: "Raise the open file descriptor resource limit (default = system fd limit)", @@ -1808,6 +1816,9 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { if ctx.IsSet(CacheFlag.Name) || ctx.IsSet(CacheSnapshotFlag.Name) { cfg.SnapshotCache = ctx.Int(CacheFlag.Name) * ctx.Int(CacheSnapshotFlag.Name) / 100 } + if ctx.IsSet(CacheLogSizeFlag.Name) { + cfg.FilterLogCacheSize = ctx.Int(CacheLogSizeFlag.Name) + } if !ctx.Bool(SnapshotFlag.Name) { // If snap-sync is requested, this flag is also required if cfg.SyncMode == downloader.SnapSync { @@ -2005,21 +2016,34 @@ func RegisterEthService(stack *node.Node, cfg *ethconfig.Config) (ethapi.Backend return backend.APIBackend, backend } -// RegisterEthStatsService configures the Ethereum Stats daemon and adds it to -// the given node. +// RegisterEthStatsService configures the Ethereum Stats daemon and adds it to the node. func RegisterEthStatsService(stack *node.Node, backend ethapi.Backend, url string) { if err := ethstats.New(stack, backend, backend.Engine(), url); err != nil { Fatalf("Failed to register the Ethereum Stats service: %v", err) } } -// RegisterGraphQLService is a utility function to construct a new service and register it against a node. -func RegisterGraphQLService(stack *node.Node, backend ethapi.Backend, cfg node.Config) { - if err := graphql.New(stack, backend, cfg.GraphQLCors, cfg.GraphQLVirtualHosts); err != nil { +// RegisterGraphQLService adds the GraphQL API to the node. +func RegisterGraphQLService(stack *node.Node, backend ethapi.Backend, filterSystem *filters.FilterSystem, cfg *node.Config) { + err := graphql.New(stack, backend, filterSystem, cfg.GraphQLCors, cfg.GraphQLVirtualHosts) + if err != nil { Fatalf("Failed to register the GraphQL service: %v", err) } } +// RegisterFilterAPI adds the eth log filtering RPC API to the node. +func RegisterFilterAPI(stack *node.Node, backend ethapi.Backend, ethcfg *ethconfig.Config) *filters.FilterSystem { + isLightClient := ethcfg.SyncMode == downloader.LightSync + filterSystem := filters.NewFilterSystem(backend, filters.Config{ + LogCacheSize: ethcfg.FilterLogCacheSize, + }) + stack.RegisterAPIs([]rpc.API{{ + Namespace: "eth", + Service: filters.NewFilterAPI(filterSystem, isLightClient), + }}) + return filterSystem +} + func SetupMetrics(ctx *cli.Context) { if metrics.Enabled { log.Info("Enabling metrics collection") diff --git a/eth/api_backend.go b/eth/api_backend.go index 1d8ba8ea5..00ecacc31 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -19,7 +19,6 @@ package eth import ( "context" "errors" - "fmt" "math/big" "time" @@ -202,17 +201,8 @@ func (b *EthAPIBackend) GetReceipts(ctx context.Context, hash common.Hash) (type return b.eth.blockchain.GetReceiptsByHash(hash), nil } -func (b *EthAPIBackend) GetLogs(ctx context.Context, hash common.Hash) ([][]*types.Log, error) { - db := b.eth.ChainDb() - number := rawdb.ReadHeaderNumber(db, hash) - if number == nil { - return nil, fmt.Errorf("failed to get block number for hash %#x", hash) - } - logs := rawdb.ReadLogs(db, hash, *number, b.eth.blockchain.Config()) - if logs == nil { - return nil, fmt.Errorf("failed to get logs for block #%d (0x%s)", *number, hash.TerminalString()) - } - return logs, nil +func (b *EthAPIBackend) GetLogs(ctx context.Context, hash common.Hash, number uint64) ([][]*types.Log, error) { + return rawdb.ReadLogs(b.eth.chainDb, hash, number, b.ChainConfig()), nil } func (b *EthAPIBackend) GetTd(ctx context.Context, hash common.Hash) *big.Int { diff --git a/eth/backend.go b/eth/backend.go index ebe7001c7..778207636 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -25,7 +25,6 @@ import ( "strings" "sync" "sync/atomic" - "time" "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/common" @@ -41,7 +40,6 @@ import ( "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/ethconfig" - "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/eth/gasprice" "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/eth/protocols/snap" @@ -315,9 +313,6 @@ func (s *Ethereum) APIs() []rpc.API { }, { Namespace: "eth", Service: downloader.NewDownloaderAPI(s.handler.downloader, s.eventMux), - }, { - Namespace: "eth", - Service: filters.NewFilterAPI(s.APIBackend, false, 5*time.Minute), }, { Namespace: "admin", Service: NewAdminAPI(s), diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index d1323de7b..569036642 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -83,6 +83,7 @@ var Defaults = Config{ TrieDirtyCache: 256, TrieTimeout: 60 * time.Minute, SnapshotCache: 102, + FilterLogCacheSize: 32, Miner: miner.Config{ GasCeil: 30000000, GasPrice: big.NewInt(params.GWei), @@ -171,6 +172,9 @@ type Config struct { SnapshotCache int Preimages bool + // This is the number of blocks for which logs will be cached in the filter system. + FilterLogCacheSize int + // Mining options Miner miner.Config diff --git a/eth/ethconfig/gen_config.go b/eth/ethconfig/gen_config.go index a6528c8df..9c7a04364 100644 --- a/eth/ethconfig/gen_config.go +++ b/eth/ethconfig/gen_config.go @@ -48,6 +48,7 @@ func (c Config) MarshalTOML() (interface{}, error) { TrieTimeout time.Duration SnapshotCache int Preimages bool + FilterLogCacheSize int Miner miner.Config Ethash ethash.Config TxPool core.TxPoolConfig @@ -93,6 +94,7 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.TrieTimeout = c.TrieTimeout enc.SnapshotCache = c.SnapshotCache enc.Preimages = c.Preimages + enc.FilterLogCacheSize = c.FilterLogCacheSize enc.Miner = c.Miner enc.Ethash = c.Ethash enc.TxPool = c.TxPool @@ -142,6 +144,7 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { TrieTimeout *time.Duration SnapshotCache *int Preimages *bool + FilterLogCacheSize *int Miner *miner.Config Ethash *ethash.Config TxPool *core.TxPoolConfig @@ -250,6 +253,9 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.Preimages != nil { c.Preimages = *dec.Preimages } + if dec.FilterLogCacheSize != nil { + c.FilterLogCacheSize = *dec.FilterLogCacheSize + } if dec.Miner != nil { c.Miner = *dec.Miner } diff --git a/eth/filters/api.go b/eth/filters/api.go index 3b8933d0a..43e63d5ba 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -46,7 +46,7 @@ type filter struct { // FilterAPI offers support to create and manage filters. This will allow external clients to retrieve various // information related to the Ethereum protocol such als blocks, transactions and logs. type FilterAPI struct { - backend Backend + sys *FilterSystem events *EventSystem filtersMu sync.Mutex filters map[rpc.ID]*filter @@ -54,14 +54,14 @@ type FilterAPI struct { } // NewFilterAPI returns a new FilterAPI instance. -func NewFilterAPI(backend Backend, lightMode bool, timeout time.Duration) *FilterAPI { +func NewFilterAPI(system *FilterSystem, lightMode bool) *FilterAPI { api := &FilterAPI{ - backend: backend, - events: NewEventSystem(backend, lightMode), + sys: system, + events: NewEventSystem(system, lightMode), filters: make(map[rpc.ID]*filter), - timeout: timeout, + timeout: system.cfg.Timeout, } - go api.timeoutLoop(timeout) + go api.timeoutLoop(system.cfg.Timeout) return api } @@ -320,7 +320,7 @@ func (api *FilterAPI) GetLogs(ctx context.Context, crit FilterCriteria) ([]*type var filter *Filter if crit.BlockHash != nil { // Block filter requested, construct a single-shot filter - filter = NewBlockFilter(api.backend, *crit.BlockHash, crit.Addresses, crit.Topics) + filter = api.sys.NewBlockFilter(*crit.BlockHash, crit.Addresses, crit.Topics) } else { // Convert the RPC block numbers into internal representations begin := rpc.LatestBlockNumber.Int64() @@ -332,7 +332,7 @@ func (api *FilterAPI) GetLogs(ctx context.Context, crit FilterCriteria) ([]*type end = crit.ToBlock.Int64() } // Construct the range filter - filter = NewRangeFilter(api.backend, begin, end, crit.Addresses, crit.Topics) + filter = api.sys.NewRangeFilter(begin, end, crit.Addresses, crit.Topics) } // Run the filter and return all the logs logs, err := filter.Logs(ctx) @@ -371,7 +371,7 @@ func (api *FilterAPI) GetFilterLogs(ctx context.Context, id rpc.ID) ([]*types.Lo var filter *Filter if f.crit.BlockHash != nil { // Block filter requested, construct a single-shot filter - filter = NewBlockFilter(api.backend, *f.crit.BlockHash, f.crit.Addresses, f.crit.Topics) + filter = api.sys.NewBlockFilter(*f.crit.BlockHash, f.crit.Addresses, f.crit.Topics) } else { // Convert the RPC block numbers into internal representations begin := rpc.LatestBlockNumber.Int64() @@ -383,7 +383,7 @@ func (api *FilterAPI) GetFilterLogs(ctx context.Context, id rpc.ID) ([]*types.Lo end = f.crit.ToBlock.Int64() } // Construct the range filter - filter = NewRangeFilter(api.backend, begin, end, f.crit.Addresses, f.crit.Topics) + filter = api.sys.NewRangeFilter(begin, end, f.crit.Addresses, f.crit.Topics) } // Run the filter and return all the logs logs, err := filter.Logs(ctx) diff --git a/eth/filters/bench_test.go b/eth/filters/bench_test.go index 694d73735..73b96b77a 100644 --- a/eth/filters/bench_test.go +++ b/eth/filters/bench_test.go @@ -122,22 +122,27 @@ func benchmarkBloomBits(b *testing.B, sectionSize uint64) { b.Log("Running filter benchmarks...") start = time.Now() - var backend *testBackend + var ( + backend *testBackend + sys *FilterSystem + ) for i := 0; i < benchFilterCnt; i++ { if i%20 == 0 { db.Close() db, _ = rawdb.NewLevelDBDatabase(benchDataDir, 128, 1024, "", false) backend = &testBackend{db: db, sections: cnt} + sys = NewFilterSystem(backend, Config{}) } var addr common.Address addr[0] = byte(i) addr[1] = byte(i / 256) - filter := NewRangeFilter(backend, 0, int64(cnt*sectionSize-1), []common.Address{addr}, nil) + filter := sys.NewRangeFilter(0, int64(cnt*sectionSize-1), []common.Address{addr}, nil) if _, err := filter.Logs(context.Background()); err != nil { - b.Error("filter.Find error:", err) + b.Error("filter.Logs error:", err) } } + d = time.Since(start) b.Log("Finished running filter benchmarks") b.Log(" ", d, "total ", d/time.Duration(benchFilterCnt), "per address", d*time.Duration(1000000)/time.Duration(benchFilterCnt*cnt*sectionSize), "per million blocks") @@ -171,10 +176,11 @@ func BenchmarkNoBloomBits(b *testing.B) { clearBloomBits(db) + _, sys := newTestFilterSystem(b, db, Config{}) + b.Log("Running filter benchmarks...") start := time.Now() - backend := &testBackend{db: db} - filter := NewRangeFilter(backend, 0, int64(*headNum), []common.Address{{}}, nil) + filter := sys.NewRangeFilter(0, int64(*headNum), []common.Address{{}}, nil) filter.Logs(context.Background()) d := time.Since(start) b.Log("Finished running filter benchmarks") diff --git a/eth/filters/filter.go b/eth/filters/filter.go index 9ff7ab7f5..0a70c9ece 100644 --- a/eth/filters/filter.go +++ b/eth/filters/filter.go @@ -22,37 +22,15 @@ import ( "math/big" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/bloombits" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/rpc" ) -type Backend interface { - ChainDb() ethdb.Database - HeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*types.Header, error) - HeaderByHash(ctx context.Context, blockHash common.Hash) (*types.Header, error) - GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error) - GetLogs(ctx context.Context, blockHash common.Hash) ([][]*types.Log, error) - PendingBlockAndReceipts() (*types.Block, types.Receipts) - - SubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription - SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription - SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription - SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription - SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription - - BloomStatus() (uint64, uint64) - ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) -} - // Filter can be used to retrieve and filter logs. type Filter struct { - backend Backend + sys *FilterSystem - db ethdb.Database addresses []common.Address topics [][]common.Hash @@ -64,7 +42,7 @@ type Filter struct { // NewRangeFilter creates a new filter which uses a bloom filter on blocks to // figure out whether a particular block is interesting or not. -func NewRangeFilter(backend Backend, begin, end int64, addresses []common.Address, topics [][]common.Hash) *Filter { +func (sys *FilterSystem) NewRangeFilter(begin, end int64, addresses []common.Address, topics [][]common.Hash) *Filter { // Flatten the address and topic filter clauses into a single bloombits filter // system. Since the bloombits are not positional, nil topics are permitted, // which get flattened into a nil byte slice. @@ -83,10 +61,10 @@ func NewRangeFilter(backend Backend, begin, end int64, addresses []common.Addres } filters = append(filters, filter) } - size, _ := backend.BloomStatus() + size, _ := sys.backend.BloomStatus() // Create a generic filter and convert it into a range filter - filter := newFilter(backend, addresses, topics) + filter := newFilter(sys, addresses, topics) filter.matcher = bloombits.NewMatcher(size, filters) filter.begin = begin @@ -97,21 +75,20 @@ func NewRangeFilter(backend Backend, begin, end int64, addresses []common.Addres // NewBlockFilter creates a new filter which directly inspects the contents of // a block to figure out whether it is interesting or not. -func NewBlockFilter(backend Backend, block common.Hash, addresses []common.Address, topics [][]common.Hash) *Filter { +func (sys *FilterSystem) NewBlockFilter(block common.Hash, addresses []common.Address, topics [][]common.Hash) *Filter { // Create a generic filter and convert it into a block filter - filter := newFilter(backend, addresses, topics) + filter := newFilter(sys, addresses, topics) filter.block = block return filter } // newFilter creates a generic filter that can either filter based on a block hash, // or based on range queries. The search criteria needs to be explicitly set. -func newFilter(backend Backend, addresses []common.Address, topics [][]common.Hash) *Filter { +func newFilter(sys *FilterSystem, addresses []common.Address, topics [][]common.Hash) *Filter { return &Filter{ - backend: backend, + sys: sys, addresses: addresses, topics: topics, - db: backend.ChainDb(), } } @@ -120,14 +97,14 @@ func newFilter(backend Backend, addresses []common.Address, topics [][]common.Ha func (f *Filter) Logs(ctx context.Context) ([]*types.Log, error) { // If we're doing singleton block filtering, execute and return if f.block != (common.Hash{}) { - header, err := f.backend.HeaderByHash(ctx, f.block) + header, err := f.sys.backend.HeaderByHash(ctx, f.block) if err != nil { return nil, err } if header == nil { return nil, errors.New("unknown block") } - return f.blockLogs(ctx, header) + return f.blockLogs(ctx, header, false) } // Short-cut if all we care about is pending logs if f.begin == rpc.PendingBlockNumber.Int64() { @@ -137,7 +114,7 @@ func (f *Filter) Logs(ctx context.Context) ([]*types.Log, error) { return f.pendingLogs() } // Figure out the limits of the filter range - header, _ := f.backend.HeaderByNumber(ctx, rpc.LatestBlockNumber) + header, _ := f.sys.backend.HeaderByNumber(ctx, rpc.LatestBlockNumber) if header == nil { return nil, nil } @@ -156,7 +133,7 @@ func (f *Filter) Logs(ctx context.Context) ([]*types.Log, error) { var ( logs []*types.Log err error - size, sections = f.backend.BloomStatus() + size, sections = f.sys.backend.BloomStatus() ) if indexed := sections * size; indexed > uint64(f.begin) { if indexed > end { @@ -192,7 +169,7 @@ func (f *Filter) indexedLogs(ctx context.Context, end uint64) ([]*types.Log, err } defer session.Close() - f.backend.ServiceFilter(ctx, session) + f.sys.backend.ServiceFilter(ctx, session) // Iterate over the matches until exhausted or context closed var logs []*types.Log @@ -211,11 +188,11 @@ func (f *Filter) indexedLogs(ctx context.Context, end uint64) ([]*types.Log, err f.begin = int64(number) + 1 // Retrieve the suggested block and pull any truly matching logs - header, err := f.backend.HeaderByNumber(ctx, rpc.BlockNumber(number)) + header, err := f.sys.backend.HeaderByNumber(ctx, rpc.BlockNumber(number)) if header == nil || err != nil { return logs, err } - found, err := f.checkMatches(ctx, header) + found, err := f.blockLogs(ctx, header, true) if err != nil { return logs, err } @@ -233,11 +210,11 @@ func (f *Filter) unindexedLogs(ctx context.Context, end uint64) ([]*types.Log, e var logs []*types.Log for ; f.begin <= int64(end); f.begin++ { - header, err := f.backend.HeaderByNumber(ctx, rpc.BlockNumber(f.begin)) + header, err := f.sys.backend.HeaderByNumber(ctx, rpc.BlockNumber(f.begin)) if header == nil || err != nil { return logs, err } - found, err := f.blockLogs(ctx, header) + found, err := f.blockLogs(ctx, header, false) if err != nil { return logs, err } @@ -247,34 +224,34 @@ func (f *Filter) unindexedLogs(ctx context.Context, end uint64) ([]*types.Log, e } // blockLogs returns the logs matching the filter criteria within a single block. -func (f *Filter) blockLogs(ctx context.Context, header *types.Header) (logs []*types.Log, err error) { - if bloomFilter(header.Bloom, f.addresses, f.topics) { - found, err := f.checkMatches(ctx, header) +func (f *Filter) blockLogs(ctx context.Context, header *types.Header, skipBloom bool) ([]*types.Log, error) { + // Fast track: no filtering criteria + if len(f.addresses) == 0 && len(f.topics) == 0 { + list, err := f.sys.cachedGetLogs(ctx, header.Hash(), header.Number.Uint64()) if err != nil { - return logs, err + return nil, err } - logs = append(logs, found...) + return flatten(list), nil + } else if skipBloom || bloomFilter(header.Bloom, f.addresses, f.topics) { + return f.checkMatches(ctx, header) } - return logs, nil + return nil, nil } // checkMatches checks if the receipts belonging to the given header contain any log events that // match the filter criteria. This function is called when the bloom filter signals a potential match. -func (f *Filter) checkMatches(ctx context.Context, header *types.Header) (logs []*types.Log, err error) { - // Get the logs of the block - logsList, err := f.backend.GetLogs(ctx, header.Hash()) +func (f *Filter) checkMatches(ctx context.Context, header *types.Header) ([]*types.Log, error) { + logsList, err := f.sys.cachedGetLogs(ctx, header.Hash(), header.Number.Uint64()) if err != nil { return nil, err } - var unfiltered []*types.Log - for _, logs := range logsList { - unfiltered = append(unfiltered, logs...) - } - logs = filterLogs(unfiltered, nil, nil, f.addresses, f.topics) + + unfiltered := flatten(logsList) + logs := filterLogs(unfiltered, nil, nil, f.addresses, f.topics) if len(logs) > 0 { // We have matching logs, check if we need to resolve full logs via the light client if logs[0].TxHash == (common.Hash{}) { - receipts, err := f.backend.GetReceipts(ctx, header.Hash()) + receipts, err := f.sys.backend.GetReceipts(ctx, header.Hash()) if err != nil { return nil, err } @@ -291,7 +268,7 @@ func (f *Filter) checkMatches(ctx context.Context, header *types.Header) (logs [ // pendingLogs returns the logs matching the filter criteria within the pending block. func (f *Filter) pendingLogs() ([]*types.Log, error) { - block, receipts := f.backend.PendingBlockAndReceipts() + block, receipts := f.sys.backend.PendingBlockAndReceipts() if bloomFilter(block.Bloom(), f.addresses, f.topics) { var unfiltered []*types.Log for _, r := range receipts { @@ -376,3 +353,11 @@ func bloomFilter(bloom types.Bloom, addresses []common.Address, topics [][]commo } return true } + +func flatten(list [][]*types.Log) []*types.Log { + var flat []*types.Log + for _, logs := range list { + flat = append(flat, logs...) + } + return flat +} diff --git a/eth/filters/filter_system.go b/eth/filters/filter_system.go index c1a1b408b..79a9b089f 100644 --- a/eth/filters/filter_system.go +++ b/eth/filters/filter_system.go @@ -27,13 +27,90 @@ import ( "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/bloombits" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rpc" + lru "github.com/hashicorp/golang-lru" ) +// Config represents the configuration of the filter system. +type Config struct { + LogCacheSize int // maximum number of cached blocks (default: 32) + Timeout time.Duration // how long filters stay active (default: 5min) +} + +func (cfg Config) withDefaults() Config { + if cfg.Timeout == 0 { + cfg.Timeout = 5 * time.Minute + } + if cfg.LogCacheSize == 0 { + cfg.LogCacheSize = 32 + } + return cfg +} + +type Backend interface { + ChainDb() ethdb.Database + HeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*types.Header, error) + HeaderByHash(ctx context.Context, blockHash common.Hash) (*types.Header, error) + GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error) + GetLogs(ctx context.Context, blockHash common.Hash, number uint64) ([][]*types.Log, error) + PendingBlockAndReceipts() (*types.Block, types.Receipts) + + SubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription + SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription + SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription + SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription + SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription + + BloomStatus() (uint64, uint64) + ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) +} + +// FilterSystem holds resources shared by all filters. +type FilterSystem struct { + backend Backend + logsCache *lru.Cache + cfg *Config +} + +// NewFilterSystem creates a filter system. +func NewFilterSystem(backend Backend, config Config) *FilterSystem { + config = config.withDefaults() + + cache, err := lru.New(config.LogCacheSize) + if err != nil { + panic(err) + } + return &FilterSystem{ + backend: backend, + logsCache: cache, + cfg: &config, + } +} + +// cachedGetLogs loads block logs from the backend and caches the result. +func (sys *FilterSystem) cachedGetLogs(ctx context.Context, blockHash common.Hash, number uint64) ([][]*types.Log, error) { + cached, ok := sys.logsCache.Get(blockHash) + if ok { + return cached.([][]*types.Log), nil + } + + logs, err := sys.backend.GetLogs(ctx, blockHash, number) + if err != nil { + return nil, err + } + if logs == nil { + return nil, fmt.Errorf("failed to get logs for block #%d (0x%s)", number, blockHash.TerminalString()) + } + sys.logsCache.Add(blockHash, logs) + return logs, nil +} + // Type determines the kind of filter and is used to put the filter in to // the correct bucket when added. type Type byte @@ -84,6 +161,7 @@ type subscription struct { // subscription which match the subscription criteria. type EventSystem struct { backend Backend + sys *FilterSystem lightMode bool lastHead *types.Header @@ -110,9 +188,10 @@ type EventSystem struct { // // The returned manager has a loop that needs to be stopped with the Stop function // or by stopping the given mux. -func NewEventSystem(backend Backend, lightMode bool) *EventSystem { +func NewEventSystem(sys *FilterSystem, lightMode bool) *EventSystem { m := &EventSystem{ - backend: backend, + sys: sys, + backend: sys.backend, lightMode: lightMode, install: make(chan *subscription), uninstall: make(chan *subscription), @@ -405,7 +484,7 @@ func (es *EventSystem) lightFilterLogs(header *types.Header, addresses []common. // Get the logs of the block ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() - logsList, err := es.backend.GetLogs(ctx, header.Hash()) + logsList, err := es.sys.cachedGetLogs(ctx, header.Hash(), header.Number.Uint64()) if err != nil { return nil } diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index c7fc4331b..51bda29b4 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -39,10 +39,6 @@ import ( "github.com/ethereum/go-ethereum/rpc" ) -var ( - deadline = 5 * time.Minute -) - type testBackend struct { db ethdb.Database sections uint64 @@ -91,17 +87,8 @@ func (b *testBackend) GetReceipts(ctx context.Context, hash common.Hash) (types. return nil, nil } -func (b *testBackend) GetLogs(ctx context.Context, hash common.Hash) ([][]*types.Log, error) { - number := rawdb.ReadHeaderNumber(b.db, hash) - if number == nil { - return nil, nil - } - receipts := rawdb.ReadReceipts(b.db, hash, *number, params.TestChainConfig) - - logs := make([][]*types.Log, len(receipts)) - for i, receipt := range receipts { - logs[i] = receipt.Logs - } +func (b *testBackend) GetLogs(ctx context.Context, hash common.Hash, number uint64) ([][]*types.Log, error) { + logs := rawdb.ReadLogs(b.db, hash, number, params.TestChainConfig) return logs, nil } @@ -160,6 +147,12 @@ func (b *testBackend) ServiceFilter(ctx context.Context, session *bloombits.Matc }() } +func newTestFilterSystem(t testing.TB, db ethdb.Database, cfg Config) (*testBackend, *FilterSystem) { + backend := &testBackend{db: db} + sys := NewFilterSystem(backend, cfg) + return backend, sys +} + // TestBlockSubscription tests if a block subscription returns block hashes for posted chain events. // It creates multiple subscriptions: // - one at the start and should receive all posted chain events and a second (blockHashes) @@ -169,12 +162,12 @@ func TestBlockSubscription(t *testing.T) { t.Parallel() var ( - db = rawdb.NewMemoryDatabase() - backend = &testBackend{db: db} - api = NewFilterAPI(backend, false, deadline) - genesis = (&core.Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(db) - chain, _ = core.GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), db, 10, func(i int, gen *core.BlockGen) {}) - chainEvents = []core.ChainEvent{} + db = rawdb.NewMemoryDatabase() + backend, sys = newTestFilterSystem(t, db, Config{}) + api = NewFilterAPI(sys, false) + genesis = (&core.Genesis{BaseFee: big.NewInt(params.InitialBaseFee)}).MustCommit(db) + chain, _ = core.GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), db, 10, func(i int, gen *core.BlockGen) {}) + chainEvents = []core.ChainEvent{} ) for _, blk := range chain { @@ -221,9 +214,9 @@ func TestPendingTxFilter(t *testing.T) { t.Parallel() var ( - db = rawdb.NewMemoryDatabase() - backend = &testBackend{db: db} - api = NewFilterAPI(backend, false, deadline) + db = rawdb.NewMemoryDatabase() + backend, sys = newTestFilterSystem(t, db, Config{}) + api = NewFilterAPI(sys, false) transactions = []*types.Transaction{ types.NewTransaction(0, common.HexToAddress("0xb794f5ea0ba39494ce83a213fffba74279579268"), new(big.Int), 0, new(big.Int), nil), @@ -276,9 +269,9 @@ func TestPendingTxFilter(t *testing.T) { // If not it must return an error. func TestLogFilterCreation(t *testing.T) { var ( - db = rawdb.NewMemoryDatabase() - backend = &testBackend{db: db} - api = NewFilterAPI(backend, false, deadline) + db = rawdb.NewMemoryDatabase() + _, sys = newTestFilterSystem(t, db, Config{}) + api = NewFilterAPI(sys, false) testCases = []struct { crit FilterCriteria @@ -323,9 +316,9 @@ func TestInvalidLogFilterCreation(t *testing.T) { t.Parallel() var ( - db = rawdb.NewMemoryDatabase() - backend = &testBackend{db: db} - api = NewFilterAPI(backend, false, deadline) + db = rawdb.NewMemoryDatabase() + _, sys = newTestFilterSystem(t, db, Config{}) + api = NewFilterAPI(sys, false) ) // different situations where log filter creation should fail. @@ -346,8 +339,8 @@ func TestInvalidLogFilterCreation(t *testing.T) { func TestInvalidGetLogsRequest(t *testing.T) { var ( db = rawdb.NewMemoryDatabase() - backend = &testBackend{db: db} - api = NewFilterAPI(backend, false, deadline) + _, sys = newTestFilterSystem(t, db, Config{}) + api = NewFilterAPI(sys, false) blockHash = common.HexToHash("0x1111111111111111111111111111111111111111111111111111111111111111") ) @@ -370,9 +363,9 @@ func TestLogFilter(t *testing.T) { t.Parallel() var ( - db = rawdb.NewMemoryDatabase() - backend = &testBackend{db: db} - api = NewFilterAPI(backend, false, deadline) + db = rawdb.NewMemoryDatabase() + backend, sys = newTestFilterSystem(t, db, Config{}) + api = NewFilterAPI(sys, false) firstAddr = common.HexToAddress("0x1111111111111111111111111111111111111111") secondAddr = common.HexToAddress("0x2222222222222222222222222222222222222222") @@ -484,9 +477,9 @@ func TestPendingLogsSubscription(t *testing.T) { t.Parallel() var ( - db = rawdb.NewMemoryDatabase() - backend = &testBackend{db: db} - api = NewFilterAPI(backend, false, deadline) + db = rawdb.NewMemoryDatabase() + backend, sys = newTestFilterSystem(t, db, Config{}) + api = NewFilterAPI(sys, false) firstAddr = common.HexToAddress("0x1111111111111111111111111111111111111111") secondAddr = common.HexToAddress("0x2222222222222222222222222222222222222222") @@ -668,10 +661,10 @@ func TestPendingTxFilterDeadlock(t *testing.T) { timeout := 100 * time.Millisecond var ( - db = rawdb.NewMemoryDatabase() - backend = &testBackend{db: db} - api = NewFilterAPI(backend, false, timeout) - done = make(chan struct{}) + db = rawdb.NewMemoryDatabase() + backend, sys = newTestFilterSystem(t, db, Config{Timeout: timeout}) + api = NewFilterAPI(sys, false) + done = make(chan struct{}) ) go func() { diff --git a/eth/filters/filter_test.go b/eth/filters/filter_test.go index 59fdde7e8..2c1f7cadf 100644 --- a/eth/filters/filter_test.go +++ b/eth/filters/filter_test.go @@ -44,7 +44,7 @@ func BenchmarkFilters(b *testing.B) { var ( db, _ = rawdb.NewLevelDBDatabase(dir, 0, 0, "", false) - backend = &testBackend{db: db} + _, sys = newTestFilterSystem(b, db, Config{}) key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr1 = crypto.PubkeyToAddress(key1.PublicKey) addr2 = common.BytesToAddress([]byte("jeff")) @@ -89,7 +89,7 @@ func BenchmarkFilters(b *testing.B) { } b.ResetTimer() - filter := NewRangeFilter(backend, 0, -1, []common.Address{addr1, addr2, addr3, addr4}, nil) + filter := sys.NewRangeFilter(0, -1, []common.Address{addr1, addr2, addr3, addr4}, nil) for i := 0; i < b.N; i++ { logs, _ := filter.Logs(context.Background()) @@ -104,7 +104,7 @@ func TestFilters(t *testing.T) { var ( db, _ = rawdb.NewLevelDBDatabase(dir, 0, 0, "", false) - backend = &testBackend{db: db} + _, sys = newTestFilterSystem(t, db, Config{}) key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") addr = crypto.PubkeyToAddress(key1.PublicKey) @@ -175,14 +175,14 @@ func TestFilters(t *testing.T) { rawdb.WriteReceipts(db, block.Hash(), block.NumberU64(), receipts[i]) } - filter := NewRangeFilter(backend, 0, -1, []common.Address{addr}, [][]common.Hash{{hash1, hash2, hash3, hash4}}) + filter := sys.NewRangeFilter(0, -1, []common.Address{addr}, [][]common.Hash{{hash1, hash2, hash3, hash4}}) logs, _ := filter.Logs(context.Background()) if len(logs) != 4 { t.Error("expected 4 log, got", len(logs)) } - filter = NewRangeFilter(backend, 900, 999, []common.Address{addr}, [][]common.Hash{{hash3}}) + filter = sys.NewRangeFilter(900, 999, []common.Address{addr}, [][]common.Hash{{hash3}}) logs, _ = filter.Logs(context.Background()) if len(logs) != 1 { t.Error("expected 1 log, got", len(logs)) @@ -191,7 +191,7 @@ func TestFilters(t *testing.T) { t.Errorf("expected log[0].Topics[0] to be %x, got %x", hash3, logs[0].Topics[0]) } - filter = NewRangeFilter(backend, 990, -1, []common.Address{addr}, [][]common.Hash{{hash3}}) + filter = sys.NewRangeFilter(990, -1, []common.Address{addr}, [][]common.Hash{{hash3}}) logs, _ = filter.Logs(context.Background()) if len(logs) != 1 { t.Error("expected 1 log, got", len(logs)) @@ -200,7 +200,7 @@ func TestFilters(t *testing.T) { t.Errorf("expected log[0].Topics[0] to be %x, got %x", hash3, logs[0].Topics[0]) } - filter = NewRangeFilter(backend, 1, 10, nil, [][]common.Hash{{hash1, hash2}}) + filter = sys.NewRangeFilter(1, 10, nil, [][]common.Hash{{hash1, hash2}}) logs, _ = filter.Logs(context.Background()) if len(logs) != 2 { @@ -208,7 +208,7 @@ func TestFilters(t *testing.T) { } failHash := common.BytesToHash([]byte("fail")) - filter = NewRangeFilter(backend, 0, -1, nil, [][]common.Hash{{failHash}}) + filter = sys.NewRangeFilter(0, -1, nil, [][]common.Hash{{failHash}}) logs, _ = filter.Logs(context.Background()) if len(logs) != 0 { @@ -216,14 +216,14 @@ func TestFilters(t *testing.T) { } failAddr := common.BytesToAddress([]byte("failmenow")) - filter = NewRangeFilter(backend, 0, -1, []common.Address{failAddr}, nil) + filter = sys.NewRangeFilter(0, -1, []common.Address{failAddr}, nil) logs, _ = filter.Logs(context.Background()) if len(logs) != 0 { t.Error("expected 0 log, got", len(logs)) } - filter = NewRangeFilter(backend, 0, -1, nil, [][]common.Hash{{failHash}, {hash1}}) + filter = sys.NewRangeFilter(0, -1, nil, [][]common.Hash{{failHash}, {hash1}}) logs, _ = filter.Logs(context.Background()) if len(logs) != 0 { diff --git a/ethclient/gethclient/gethclient_test.go b/ethclient/gethclient/gethclient_test.go index b78d11c32..a0f4eaaf5 100644 --- a/ethclient/gethclient/gethclient_test.go +++ b/ethclient/gethclient/gethclient_test.go @@ -31,6 +31,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/eth/ethconfig" + "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/ethclient" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/params" @@ -60,6 +61,12 @@ func newTestBackend(t *testing.T) (*node.Node, []*types.Block) { if err != nil { t.Fatalf("can't create new ethereum service: %v", err) } + filterSystem := filters.NewFilterSystem(ethservice.APIBackend, filters.Config{}) + n.RegisterAPIs([]rpc.API{{ + Namespace: "eth", + Service: filters.NewFilterAPI(filterSystem, false), + }}) + // Import the test chain. if err := n.Start(); err != nil { t.Fatalf("can't start test node: %v", err) diff --git a/graphql/graphql.go b/graphql/graphql.go index 0949c3480..97b460c20 100644 --- a/graphql/graphql.go +++ b/graphql/graphql.go @@ -450,12 +450,36 @@ func (t *Transaction) CreatedContract(ctx context.Context, args BlockNumberArgs) } func (t *Transaction) Logs(ctx context.Context) (*[]*Log, error) { - receipt, err := t.getReceipt(ctx) - if err != nil || receipt == nil { + if _, err := t.resolve(ctx); err != nil { return nil, err } - ret := make([]*Log, 0, len(receipt.Logs)) - for _, log := range receipt.Logs { + if t.block == nil { + return nil, nil + } + if _, ok := t.block.numberOrHash.Hash(); !ok { + header, err := t.r.backend.HeaderByNumberOrHash(ctx, *t.block.numberOrHash) + if err != nil { + return nil, err + } + hash := header.Hash() + t.block.numberOrHash.BlockHash = &hash + } + return t.getLogs(ctx) +} + +// getLogs returns log objects for the given tx. +// Assumes block hash is resolved. +func (t *Transaction) getLogs(ctx context.Context) (*[]*Log, error) { + var ( + hash, _ = t.block.numberOrHash.Hash() + filter = t.r.filterSystem.NewBlockFilter(hash, nil, nil) + logs, err = filter.Logs(ctx) + ) + if err != nil { + return nil, err + } + ret := make([]*Log, 0, len(logs)) + for _, log := range logs { ret = append(ret, &Log{ r: t.r, transaction: t, @@ -978,7 +1002,7 @@ func (b *Block) Logs(ctx context.Context, args struct{ Filter BlockFilterCriteri hash = header.Hash() } // Construct the range filter - filter := filters.NewBlockFilter(b.r.backend, hash, addresses, topics) + filter := b.r.filterSystem.NewBlockFilter(hash, addresses, topics) // Run the filter and return all the logs return runFilter(ctx, b.r, filter) @@ -1137,7 +1161,8 @@ func (p *Pending) EstimateGas(ctx context.Context, args struct { // Resolver is the top-level object in the GraphQL hierarchy. type Resolver struct { - backend ethapi.Backend + backend ethapi.Backend + filterSystem *filters.FilterSystem } func (r *Resolver) Block(ctx context.Context, args struct { @@ -1284,7 +1309,7 @@ func (r *Resolver) Logs(ctx context.Context, args struct{ Filter FilterCriteria topics = *args.Filter.Topics } // Construct the range filter - filter := filters.NewRangeFilter(r.backend, begin, end, addresses, topics) + filter := r.filterSystem.NewRangeFilter(begin, end, addresses, topics) return runFilter(ctx, r, filter) } diff --git a/graphql/graphql_test.go b/graphql/graphql_test.go index 4b7f7bf96..d55f4e063 100644 --- a/graphql/graphql_test.go +++ b/graphql/graphql_test.go @@ -33,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/eth/ethconfig" + "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/params" @@ -50,7 +51,7 @@ func TestBuildSchema(t *testing.T) { } defer stack.Close() // Make sure the schema can be parsed and matched up to the object model. - if err := newHandler(stack, nil, []string{}, []string{}); err != nil { + if err := newHandler(stack, nil, nil, []string{}, []string{}); err != nil { t.Errorf("Could not construct GraphQL handler: %v", err) } } @@ -263,7 +264,8 @@ func createGQLService(t *testing.T, stack *node.Node) { t.Fatalf("could not create import blocks: %v", err) } // create gql service - err = New(stack, ethBackend.APIBackend, []string{}, []string{}) + filterSystem := filters.NewFilterSystem(ethBackend.APIBackend, filters.Config{}) + err = New(stack, ethBackend.APIBackend, filterSystem, []string{}, []string{}) if err != nil { t.Fatalf("could not create graphql service: %v", err) } @@ -348,7 +350,8 @@ func createGQLServiceWithTransactions(t *testing.T, stack *node.Node) { t.Fatalf("could not create import blocks: %v", err) } // create gql service - err = New(stack, ethBackend.APIBackend, []string{}, []string{}) + filterSystem := filters.NewFilterSystem(ethBackend.APIBackend, filters.Config{}) + err = New(stack, ethBackend.APIBackend, filterSystem, []string{}, []string{}) if err != nil { t.Fatalf("could not create graphql service: %v", err) } diff --git a/graphql/service.go b/graphql/service.go index 1a2ffaa94..019026bc7 100644 --- a/graphql/service.go +++ b/graphql/service.go @@ -20,6 +20,7 @@ import ( "encoding/json" "net/http" + "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/internal/ethapi" "github.com/ethereum/go-ethereum/node" "github.com/graph-gophers/graphql-go" @@ -55,14 +56,14 @@ func (h handler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } // New constructs a new GraphQL service instance. -func New(stack *node.Node, backend ethapi.Backend, cors, vhosts []string) error { - return newHandler(stack, backend, cors, vhosts) +func New(stack *node.Node, backend ethapi.Backend, filterSystem *filters.FilterSystem, cors, vhosts []string) error { + return newHandler(stack, backend, filterSystem, cors, vhosts) } // newHandler returns a new `http.Handler` that will answer GraphQL queries. // It additionally exports an interactive query browser on the / endpoint. -func newHandler(stack *node.Node, backend ethapi.Backend, cors, vhosts []string) error { - q := Resolver{backend} +func newHandler(stack *node.Node, backend ethapi.Backend, filterSystem *filters.FilterSystem, cors, vhosts []string) error { + q := Resolver{backend, filterSystem} s, err := graphql.ParseSchema(schema, &q) if err != nil { diff --git a/internal/ethapi/backend.go b/internal/ethapi/backend.go index d13547f23..5b4ceb631 100644 --- a/internal/ethapi/backend.go +++ b/internal/ethapi/backend.go @@ -27,10 +27,10 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/bloombits" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/params" @@ -84,16 +84,12 @@ type Backend interface { TxPoolContentFrom(addr common.Address) (types.Transactions, types.Transactions) SubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription - // Filter API - BloomStatus() (uint64, uint64) - GetLogs(ctx context.Context, blockHash common.Hash) ([][]*types.Log, error) - ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) - SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription - SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription - SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription - ChainConfig() *params.ChainConfig Engine() consensus.Engine + + // eth/filters needs to be initialized from this backend type, so methods needed by + // it must also be included here. + filters.Backend } func GetAPIs(apiBackend Backend) []rpc.API { diff --git a/les/api_backend.go b/les/api_backend.go index 11a9ca128..5b4213134 100644 --- a/les/api_backend.go +++ b/les/api_backend.go @@ -168,11 +168,8 @@ func (b *LesApiBackend) GetReceipts(ctx context.Context, hash common.Hash) (type return nil, nil } -func (b *LesApiBackend) GetLogs(ctx context.Context, hash common.Hash) ([][]*types.Log, error) { - if number := rawdb.ReadHeaderNumber(b.eth.chainDb, hash); number != nil { - return light.GetBlockLogs(ctx, b.eth.odr, hash, *number) - } - return nil, nil +func (b *LesApiBackend) GetLogs(ctx context.Context, hash common.Hash, number uint64) ([][]*types.Log, error) { + return light.GetBlockLogs(ctx, b.eth.odr, hash, number) } func (b *LesApiBackend) GetTd(ctx context.Context, hash common.Hash) *big.Int { diff --git a/les/client.go b/les/client.go index 7caaf2c18..6504fe2af 100644 --- a/les/client.go +++ b/les/client.go @@ -32,7 +32,6 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth/ethconfig" - "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/eth/gasprice" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/internal/ethapi" @@ -298,9 +297,6 @@ func (s *LightEthereum) APIs() []rpc.API { }, { Namespace: "eth", Service: downloader.NewDownloaderAPI(s.handler.downloader, s.eventMux), - }, { - Namespace: "eth", - Service: filters.NewFilterAPI(s.ApiBackend, true, 5*time.Minute), }, { Namespace: "net", Service: s.netRPCService, From 08658806268f4c570c682e87681881ac35123fe9 Mon Sep 17 00:00:00 2001 From: zhiqiangxu <652732310@qq.com> Date: Fri, 19 Aug 2022 18:15:04 +0800 Subject: [PATCH 53/59] accounts/abi: fix set function (#25477) * accounts/abi: fix set function * don't break things * update test --- accounts/abi/reflect.go | 2 +- accounts/abi/unpack_test.go | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/accounts/abi/reflect.go b/accounts/abi/reflect.go index eb21bb264..7917fa980 100644 --- a/accounts/abi/reflect.go +++ b/accounts/abi/reflect.go @@ -99,7 +99,7 @@ func mustArrayToByteSlice(value reflect.Value) reflect.Value { func set(dst, src reflect.Value) error { dstType, srcType := dst.Type(), src.Type() switch { - case dstType.Kind() == reflect.Interface && dst.Elem().IsValid(): + case dstType.Kind() == reflect.Interface && dst.Elem().IsValid() && (dst.Elem().Type().Kind() == reflect.Ptr || dst.Elem().CanSet()): return set(dst.Elem(), src) case dstType.Kind() == reflect.Ptr && dstType.Elem() != reflect.TypeOf(big.Int{}): return set(dst.Elem(), src) diff --git a/accounts/abi/unpack_test.go b/accounts/abi/unpack_test.go index ae3565c71..363e0cd59 100644 --- a/accounts/abi/unpack_test.go +++ b/accounts/abi/unpack_test.go @@ -352,6 +352,11 @@ func TestMethodMultiReturn(t *testing.T) { &[]interface{}{&expected.Int, &expected.String}, "", "Can unpack into a slice", + }, { + &[]interface{}{&bigint, ""}, + &[]interface{}{&expected.Int, expected.String}, + "", + "Can unpack into a slice without indirection", }, { &[2]interface{}{&bigint, new(string)}, &[2]interface{}{&expected.Int, &expected.String}, From ac7ad811b4d2ba0b93f9272e2487253026620b48 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Fri, 19 Aug 2022 14:48:49 +0200 Subject: [PATCH 54/59] internal/ethapi: fix build regression (#25555) --- internal/ethapi/transaction_args_test.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/internal/ethapi/transaction_args_test.go b/internal/ethapi/transaction_args_test.go index 92f009aa8..28dc561c3 100644 --- a/internal/ethapi/transaction_args_test.go +++ b/internal/ethapi/transaction_args_test.go @@ -298,6 +298,9 @@ func (b *backendMock) PendingBlockAndReceipts() (*types.Block, types.Receipts) { func (b *backendMock) GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) { return nil, nil } +func (b *backendMock) GetLogs(ctx context.Context, blockHash common.Hash, number uint64) ([][]*types.Log, error) { + return nil, nil +} func (b *backendMock) GetTd(ctx context.Context, hash common.Hash) *big.Int { return nil } func (b *backendMock) GetEVM(ctx context.Context, msg core.Message, state *state.StateDB, header *types.Header, vmConfig *vm.Config) (*vm.EVM, func() error, error) { return nil, nil, nil @@ -325,11 +328,8 @@ func (b *backendMock) TxPoolContent() (map[common.Address]types.Transactions, ma func (b *backendMock) TxPoolContentFrom(addr common.Address) (types.Transactions, types.Transactions) { return nil, nil } -func (b *backendMock) SubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription { return nil } -func (b *backendMock) BloomStatus() (uint64, uint64) { return 0, 0 } -func (b *backendMock) GetLogs(ctx context.Context, blockHash common.Hash) ([][]*types.Log, error) { - return nil, nil -} +func (b *backendMock) SubscribeNewTxsEvent(chan<- core.NewTxsEvent) event.Subscription { return nil } +func (b *backendMock) BloomStatus() (uint64, uint64) { return 0, 0 } func (b *backendMock) ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) {} func (b *backendMock) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription { return nil } func (b *backendMock) SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription { From 0ce494b60cd00d70f1f9f2dd0b9bfbd76204168a Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Fri, 19 Aug 2022 15:59:36 +0200 Subject: [PATCH 55/59] eth/fetcher: don't spend too much time on transaction inclusion (#25524) * eth/fetcher: introduce some lag in tx fetching * eth/fetcher: change conditions a bit * eth/fetcher: use per-batch quota check * eth/fetcher: fix some comments * eth/fetcher: address review concerns * eth/fetcher: fix panic + add warn log * eth/fetcher: fix log * eth/fetcher: fix log * cmd/devp2p/internal/ethtest: fix ignorign tx announcements from prev. tests * cmd/devp2p/internal/ethtest: fix TestLargeTxRequest This increases the number of tx relay messages the test waits for. Since go-ethereum now processes incoming txs in smaller batches, the announcement messages it sends are also smaller. Co-authored-by: Felix Lange --- cmd/devp2p/internal/ethtest/helpers.go | 6 +- cmd/devp2p/internal/ethtest/suite.go | 4 + cmd/devp2p/internal/ethtest/transaction.go | 6 +- eth/fetcher/tx_fetcher.go | 100 +++++++++++++-------- 4 files changed, 73 insertions(+), 43 deletions(-) diff --git a/cmd/devp2p/internal/ethtest/helpers.go b/cmd/devp2p/internal/ethtest/helpers.go index eeeb4f93c..b57649ade 100644 --- a/cmd/devp2p/internal/ethtest/helpers.go +++ b/cmd/devp2p/internal/ethtest/helpers.go @@ -357,9 +357,13 @@ func (s *Suite) waitAnnounce(conn *Conn, blockAnnouncement *NewBlock) error { return fmt.Errorf("wrong block hash in announcement: expected %v, got %v", blockAnnouncement.Block.Hash(), hashes[0].Hash) } return nil + + // ignore tx announcements from previous tests case *NewPooledTransactionHashes: - // ignore tx announcements from previous tests continue + case *Transactions: + continue + default: return fmt.Errorf("unexpected: %s", pretty.Sdump(msg)) } diff --git a/cmd/devp2p/internal/ethtest/suite.go b/cmd/devp2p/internal/ethtest/suite.go index 7059b4ba7..4497478d7 100644 --- a/cmd/devp2p/internal/ethtest/suite.go +++ b/cmd/devp2p/internal/ethtest/suite.go @@ -544,9 +544,13 @@ func (s *Suite) TestNewPooledTxs(t *utesting.T) { t.Fatalf("unexpected number of txs requested: wanted %d, got %d", len(hashes), len(msg.GetPooledTransactionsPacket)) } return + // ignore propagated txs from previous tests case *NewPooledTransactionHashes: continue + case *Transactions: + continue + // ignore block announcements from previous tests case *NewBlockHashes: continue diff --git a/cmd/devp2p/internal/ethtest/transaction.go b/cmd/devp2p/internal/ethtest/transaction.go index c4748bf8f..baa55bd49 100644 --- a/cmd/devp2p/internal/ethtest/transaction.go +++ b/cmd/devp2p/internal/ethtest/transaction.go @@ -29,7 +29,7 @@ import ( "github.com/ethereum/go-ethereum/params" ) -//var faucetAddr = common.HexToAddress("0x71562b71999873DB5b286dF957af199Ec94617F7") +// var faucetAddr = common.HexToAddress("0x71562b71999873DB5b286dF957af199Ec94617F7") var faucetKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") func (s *Suite) sendSuccessfulTxs(t *utesting.T) error { @@ -192,10 +192,10 @@ func sendMultipleSuccessfulTxs(t *utesting.T, s *Suite, txs []*types.Transaction nonce = txs[len(txs)-1].Nonce() // Wait for the transaction announcement(s) and make sure all sent txs are being propagated. - // all txs should be announced within 3 announcements. + // all txs should be announced within a couple announcements. recvHashes := make([]common.Hash, 0) - for i := 0; i < 3; i++ { + for i := 0; i < 20; i++ { switch msg := recvConn.readAndServe(s.chain, timeout).(type) { case *Transactions: for _, tx := range *msg { diff --git a/eth/fetcher/tx_fetcher.go b/eth/fetcher/tx_fetcher.go index 035e0c2ec..7c8f16df5 100644 --- a/eth/fetcher/tx_fetcher.go +++ b/eth/fetcher/tx_fetcher.go @@ -262,57 +262,79 @@ func (f *TxFetcher) Notify(peer string, hashes []common.Hash) error { // direct request replies. The differentiation is important so the fetcher can // re-schedule missing transactions as soon as possible. func (f *TxFetcher) Enqueue(peer string, txs []*types.Transaction, direct bool) error { - // Keep track of all the propagated transactions - if direct { - txReplyInMeter.Mark(int64(len(txs))) - } else { - txBroadcastInMeter.Mark(int64(len(txs))) + var ( + inMeter = txReplyInMeter + knownMeter = txReplyKnownMeter + underpricedMeter = txReplyUnderpricedMeter + otherRejectMeter = txReplyOtherRejectMeter + ) + if !direct { + inMeter = txBroadcastInMeter + knownMeter = txBroadcastKnownMeter + underpricedMeter = txBroadcastUnderpricedMeter + otherRejectMeter = txBroadcastOtherRejectMeter } + // Keep track of all the propagated transactions + inMeter.Mark(int64(len(txs))) + // Push all the transactions into the pool, tracking underpriced ones to avoid // re-requesting them and dropping the peer in case of malicious transfers. var ( - added = make([]common.Hash, 0, len(txs)) - duplicate int64 - underpriced int64 - otherreject int64 + added = make([]common.Hash, 0, len(txs)) + delay time.Duration ) - errs := f.addTxs(txs) - for i, err := range errs { - // Track the transaction hash if the price is too low for us. - // Avoid re-request this transaction when we receive another - // announcement. - if errors.Is(err, core.ErrUnderpriced) || errors.Is(err, core.ErrReplaceUnderpriced) { - for f.underpriced.Cardinality() >= maxTxUnderpricedSetSize { - f.underpriced.Pop() + // proceed in batches + for i := 0; i < len(txs); i += 128 { + end := i + 128 + if end > len(txs) { + end = len(txs) + } + var ( + duplicate int64 + underpriced int64 + otherreject int64 + ) + batch := txs[i:end] + for j, err := range f.addTxs(batch) { + // Track the transaction hash if the price is too low for us. + // Avoid re-request this transaction when we receive another + // announcement. + if errors.Is(err, core.ErrUnderpriced) || errors.Is(err, core.ErrReplaceUnderpriced) { + for f.underpriced.Cardinality() >= maxTxUnderpricedSetSize { + f.underpriced.Pop() + } + f.underpriced.Add(batch[j].Hash()) } - f.underpriced.Add(txs[i].Hash()) + // Track a few interesting failure types + switch { + case err == nil: // Noop, but need to handle to not count these + + case errors.Is(err, core.ErrAlreadyKnown): + duplicate++ + + case errors.Is(err, core.ErrUnderpriced) || errors.Is(err, core.ErrReplaceUnderpriced): + underpriced++ + + default: + otherreject++ + } + added = append(added, batch[j].Hash()) } - // Track a few interesting failure types - switch { - case err == nil: // Noop, but need to handle to not count these + knownMeter.Mark(duplicate) + underpricedMeter.Mark(underpriced) + otherRejectMeter.Mark(otherreject) - case errors.Is(err, core.ErrAlreadyKnown): - duplicate++ - - case errors.Is(err, core.ErrUnderpriced) || errors.Is(err, core.ErrReplaceUnderpriced): - underpriced++ - - default: - otherreject++ + // If 'other reject' is >25% of the deliveries in any batch, abort. Either we are + // out of sync with the chain or the peer is griefing us. + if otherreject > 128/4 { + delay = 200 * time.Millisecond + log.Warn("Peer delivering useless transactions", "peer", peer, "ignored", len(txs)-end) + break } - added = append(added, txs[i].Hash()) - } - if direct { - txReplyKnownMeter.Mark(duplicate) - txReplyUnderpricedMeter.Mark(underpriced) - txReplyOtherRejectMeter.Mark(otherreject) - } else { - txBroadcastKnownMeter.Mark(duplicate) - txBroadcastUnderpricedMeter.Mark(underpriced) - txBroadcastOtherRejectMeter.Mark(otherreject) } select { case f.cleanup <- &txDelivery{origin: peer, hashes: added, direct: direct}: + time.Sleep(delay) return nil case <-f.quit: return errTerminated From 02418c2fa965e61f5bc1e66e1063482eb5293e5d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 22 Aug 2022 10:14:56 +0300 Subject: [PATCH 56/59] Revert "eth/fetcher: don't spend too much time on transaction inclusion" (#25567) Revert "eth/fetcher: don't spend too much time on transaction inclusion (#25524)" This reverts commit 0ce494b60cd00d70f1f9f2dd0b9bfbd76204168a. --- cmd/devp2p/internal/ethtest/helpers.go | 6 +- cmd/devp2p/internal/ethtest/suite.go | 4 - cmd/devp2p/internal/ethtest/transaction.go | 6 +- eth/fetcher/tx_fetcher.go | 100 ++++++++------------- 4 files changed, 43 insertions(+), 73 deletions(-) diff --git a/cmd/devp2p/internal/ethtest/helpers.go b/cmd/devp2p/internal/ethtest/helpers.go index b57649ade..eeeb4f93c 100644 --- a/cmd/devp2p/internal/ethtest/helpers.go +++ b/cmd/devp2p/internal/ethtest/helpers.go @@ -357,13 +357,9 @@ func (s *Suite) waitAnnounce(conn *Conn, blockAnnouncement *NewBlock) error { return fmt.Errorf("wrong block hash in announcement: expected %v, got %v", blockAnnouncement.Block.Hash(), hashes[0].Hash) } return nil - - // ignore tx announcements from previous tests case *NewPooledTransactionHashes: + // ignore tx announcements from previous tests continue - case *Transactions: - continue - default: return fmt.Errorf("unexpected: %s", pretty.Sdump(msg)) } diff --git a/cmd/devp2p/internal/ethtest/suite.go b/cmd/devp2p/internal/ethtest/suite.go index 4497478d7..7059b4ba7 100644 --- a/cmd/devp2p/internal/ethtest/suite.go +++ b/cmd/devp2p/internal/ethtest/suite.go @@ -544,13 +544,9 @@ func (s *Suite) TestNewPooledTxs(t *utesting.T) { t.Fatalf("unexpected number of txs requested: wanted %d, got %d", len(hashes), len(msg.GetPooledTransactionsPacket)) } return - // ignore propagated txs from previous tests case *NewPooledTransactionHashes: continue - case *Transactions: - continue - // ignore block announcements from previous tests case *NewBlockHashes: continue diff --git a/cmd/devp2p/internal/ethtest/transaction.go b/cmd/devp2p/internal/ethtest/transaction.go index baa55bd49..c4748bf8f 100644 --- a/cmd/devp2p/internal/ethtest/transaction.go +++ b/cmd/devp2p/internal/ethtest/transaction.go @@ -29,7 +29,7 @@ import ( "github.com/ethereum/go-ethereum/params" ) -// var faucetAddr = common.HexToAddress("0x71562b71999873DB5b286dF957af199Ec94617F7") +//var faucetAddr = common.HexToAddress("0x71562b71999873DB5b286dF957af199Ec94617F7") var faucetKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") func (s *Suite) sendSuccessfulTxs(t *utesting.T) error { @@ -192,10 +192,10 @@ func sendMultipleSuccessfulTxs(t *utesting.T, s *Suite, txs []*types.Transaction nonce = txs[len(txs)-1].Nonce() // Wait for the transaction announcement(s) and make sure all sent txs are being propagated. - // all txs should be announced within a couple announcements. + // all txs should be announced within 3 announcements. recvHashes := make([]common.Hash, 0) - for i := 0; i < 20; i++ { + for i := 0; i < 3; i++ { switch msg := recvConn.readAndServe(s.chain, timeout).(type) { case *Transactions: for _, tx := range *msg { diff --git a/eth/fetcher/tx_fetcher.go b/eth/fetcher/tx_fetcher.go index 7c8f16df5..035e0c2ec 100644 --- a/eth/fetcher/tx_fetcher.go +++ b/eth/fetcher/tx_fetcher.go @@ -262,79 +262,57 @@ func (f *TxFetcher) Notify(peer string, hashes []common.Hash) error { // direct request replies. The differentiation is important so the fetcher can // re-schedule missing transactions as soon as possible. func (f *TxFetcher) Enqueue(peer string, txs []*types.Transaction, direct bool) error { - var ( - inMeter = txReplyInMeter - knownMeter = txReplyKnownMeter - underpricedMeter = txReplyUnderpricedMeter - otherRejectMeter = txReplyOtherRejectMeter - ) - if !direct { - inMeter = txBroadcastInMeter - knownMeter = txBroadcastKnownMeter - underpricedMeter = txBroadcastUnderpricedMeter - otherRejectMeter = txBroadcastOtherRejectMeter - } // Keep track of all the propagated transactions - inMeter.Mark(int64(len(txs))) - + if direct { + txReplyInMeter.Mark(int64(len(txs))) + } else { + txBroadcastInMeter.Mark(int64(len(txs))) + } // Push all the transactions into the pool, tracking underpriced ones to avoid // re-requesting them and dropping the peer in case of malicious transfers. var ( - added = make([]common.Hash, 0, len(txs)) - delay time.Duration + added = make([]common.Hash, 0, len(txs)) + duplicate int64 + underpriced int64 + otherreject int64 ) - // proceed in batches - for i := 0; i < len(txs); i += 128 { - end := i + 128 - if end > len(txs) { - end = len(txs) - } - var ( - duplicate int64 - underpriced int64 - otherreject int64 - ) - batch := txs[i:end] - for j, err := range f.addTxs(batch) { - // Track the transaction hash if the price is too low for us. - // Avoid re-request this transaction when we receive another - // announcement. - if errors.Is(err, core.ErrUnderpriced) || errors.Is(err, core.ErrReplaceUnderpriced) { - for f.underpriced.Cardinality() >= maxTxUnderpricedSetSize { - f.underpriced.Pop() - } - f.underpriced.Add(batch[j].Hash()) + errs := f.addTxs(txs) + for i, err := range errs { + // Track the transaction hash if the price is too low for us. + // Avoid re-request this transaction when we receive another + // announcement. + if errors.Is(err, core.ErrUnderpriced) || errors.Is(err, core.ErrReplaceUnderpriced) { + for f.underpriced.Cardinality() >= maxTxUnderpricedSetSize { + f.underpriced.Pop() } - // Track a few interesting failure types - switch { - case err == nil: // Noop, but need to handle to not count these - - case errors.Is(err, core.ErrAlreadyKnown): - duplicate++ - - case errors.Is(err, core.ErrUnderpriced) || errors.Is(err, core.ErrReplaceUnderpriced): - underpriced++ - - default: - otherreject++ - } - added = append(added, batch[j].Hash()) + f.underpriced.Add(txs[i].Hash()) } - knownMeter.Mark(duplicate) - underpricedMeter.Mark(underpriced) - otherRejectMeter.Mark(otherreject) + // Track a few interesting failure types + switch { + case err == nil: // Noop, but need to handle to not count these - // If 'other reject' is >25% of the deliveries in any batch, abort. Either we are - // out of sync with the chain or the peer is griefing us. - if otherreject > 128/4 { - delay = 200 * time.Millisecond - log.Warn("Peer delivering useless transactions", "peer", peer, "ignored", len(txs)-end) - break + case errors.Is(err, core.ErrAlreadyKnown): + duplicate++ + + case errors.Is(err, core.ErrUnderpriced) || errors.Is(err, core.ErrReplaceUnderpriced): + underpriced++ + + default: + otherreject++ } + added = append(added, txs[i].Hash()) + } + if direct { + txReplyKnownMeter.Mark(duplicate) + txReplyUnderpricedMeter.Mark(underpriced) + txReplyOtherRejectMeter.Mark(otherreject) + } else { + txBroadcastKnownMeter.Mark(duplicate) + txBroadcastUnderpricedMeter.Mark(underpriced) + txBroadcastOtherRejectMeter.Mark(otherreject) } select { case f.cleanup <- &txDelivery{origin: peer, hashes: added, direct: direct}: - time.Sleep(delay) return nil case <-f.quit: return errTerminated From 395f3d4bf689c199e93e05af57ebff09b10f1c9a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 22 Aug 2022 11:27:39 +0300 Subject: [PATCH 57/59] eth/catalyst: warn less frequently if no beacon client is available (#25569) * eth/catalyst: warn less frequently if no beacon client is available * eth/catalyst: tweak warning frequency a bit * eth/catalyst: some more tweaks * Update api.go Co-authored-by: Felix Lange --- eth/catalyst/api.go | 39 ++++++++++++++++++++++++++++----------- 1 file changed, 28 insertions(+), 11 deletions(-) diff --git a/eth/catalyst/api.go b/eth/catalyst/api.go index 358529459..b159f34e6 100644 --- a/eth/catalyst/api.go +++ b/eth/catalyst/api.go @@ -61,10 +61,23 @@ const ( // have lead to some bad ancestor block. It's just an OOM protection. invalidTipsetsCap = 512 - // beaconUpdateTimeout is the max time allowed for a beacon client to signal - // use (from the last heartbeat) before it's consifered offline and the user + // beaconUpdateStartupTimeout is the time to wait for a beacon client to get + // attached before starting to issue warnings. + beaconUpdateStartupTimeout = 30 * time.Second + + // beaconUpdateExchangeTimeout is the max time allowed for a beacon client to + // do a transition config exchange before it's considered offline and the user // is warned. - beaconUpdateTimeout = 30 * time.Second + beaconUpdateExchangeTimeout = 2 * time.Minute + + // beaconUpdateConsensusTimeout is the max time allowed for a beacon client + // to send a consensus update before it's considered offline and the user is + // warned. + beaconUpdateConsensusTimeout = 30 * time.Second + + // beaconUpdateWarnFrequency is the frequency at which to warn the user that + // the beacon client is offline. + beaconUpdateWarnFrequency = 5 * time.Minute ) type ConsensusAPI struct { @@ -545,9 +558,9 @@ func (api *ConsensusAPI) invalid(err error, latestValid *types.Header) beacon.Pa // // TODO(karalabe): Spin this goroutine down somehow func (api *ConsensusAPI) heartbeat() { - // Sleep a bit more on startup since there's obviously no beacon client yet + // Sleep a bit on startup since there's obviously no beacon client yet // attached, so no need to print scary warnings to the user. - time.Sleep(beaconUpdateTimeout) + time.Sleep(beaconUpdateStartupTimeout) var ( offlineLogged time.Time @@ -576,9 +589,9 @@ func (api *ConsensusAPI) heartbeat() { // If there have been no updates for the past while, warn the user // that the beacon client is probably offline if api.eth.BlockChain().Config().TerminalTotalDifficultyPassed || api.eth.Merger().TDDReached() { - if time.Since(lastForkchoiceUpdate) > beaconUpdateTimeout && time.Since(lastNewPayloadUpdate) > beaconUpdateTimeout { - if time.Since(lastTransitionUpdate) > beaconUpdateTimeout { - if time.Since(offlineLogged) > beaconUpdateTimeout { + if time.Since(lastForkchoiceUpdate) > beaconUpdateConsensusTimeout && time.Since(lastNewPayloadUpdate) > beaconUpdateConsensusTimeout { + if time.Since(lastTransitionUpdate) > beaconUpdateExchangeTimeout { + if time.Since(offlineLogged) > beaconUpdateWarnFrequency { if lastTransitionUpdate.IsZero() { log.Warn("Post-merge network, but no beacon client seen. Please launch one to follow the chain!") } else { @@ -588,7 +601,7 @@ func (api *ConsensusAPI) heartbeat() { } continue } - if time.Since(offlineLogged) > beaconUpdateTimeout { + if time.Since(offlineLogged) > beaconUpdateWarnFrequency { if lastForkchoiceUpdate.IsZero() && lastNewPayloadUpdate.IsZero() { log.Warn("Beacon client online, but never received consensus updates. Please ensure your beacon client is operational to follow the chain!") } else { @@ -597,10 +610,12 @@ func (api *ConsensusAPI) heartbeat() { offlineLogged = time.Now() } continue + } else { + offlineLogged = time.Time{} } } else { - if time.Since(lastTransitionUpdate) > beaconUpdateTimeout { - if time.Since(offlineLogged) > beaconUpdateTimeout { + if time.Since(lastTransitionUpdate) > beaconUpdateExchangeTimeout { + if time.Since(offlineLogged) > beaconUpdateWarnFrequency { // Retrieve the last few blocks and make a rough estimate as // to when the merge transition should happen var ( @@ -654,6 +669,8 @@ func (api *ConsensusAPI) heartbeat() { offlineLogged = time.Now() } continue + } else { + offlineLogged = time.Time{} } } } From 2de49b04e56cf07f011d6d91f9d5c08847aabe8e Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Mon, 22 Aug 2022 10:36:39 +0200 Subject: [PATCH 58/59] params: release go-ethereum v1.10.22 --- params/version.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/params/version.go b/params/version.go index 258be5d5d..5105f3578 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 10 // Minor version component of the current release - VersionPatch = 22 // Patch version component of the current release - VersionMeta = "unstable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 10 // Minor version component of the current release + VersionPatch = 22 // Patch version component of the current release + VersionMeta = "stable" // Version metadata to append to the version string ) // Version holds the textual version string. @@ -41,9 +41,9 @@ var VersionWithMeta = func() string { return v }() -// ArchiveVersion holds the textual version string used for Geth archives. -// e.g. "1.8.11-dea1ce05" for stable releases, or -// "1.8.13-unstable-21c059b6" for unstable releases +// ArchiveVersion holds the textual version string used for Geth archives. e.g. +// "1.8.11-dea1ce05" for stable releases, or "1.8.13-unstable-21c059b6" for unstable +// releases. func ArchiveVersion(gitCommit string) string { vsn := Version if VersionMeta != "stable" { From 967626b0759e0d0ebec4d09a00244db2d94adf61 Mon Sep 17 00:00:00 2001 From: philip-morlier Date: Mon, 22 Aug 2022 11:48:26 -0700 Subject: [PATCH 59/59] Updates to reflect comments on original merge. --- core/rawdb/plugin_hooks.go | 114 +++++++++--------- go.mod | 2 - .../wrappers/backendwrapper/backendwrapper.go | 5 +- 3 files changed, 60 insertions(+), 61 deletions(-) diff --git a/core/rawdb/plugin_hooks.go b/core/rawdb/plugin_hooks.go index 017bcd7cf..05a0b36d3 100644 --- a/core/rawdb/plugin_hooks.go +++ b/core/rawdb/plugin_hooks.go @@ -4,7 +4,7 @@ package rawdb import ( "github.com/ethereum/go-ethereum/plugins" "github.com/ethereum/go-ethereum/log" - // "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/rlp" "sync" ) @@ -57,64 +57,64 @@ func PluginCommitUpdate(pl *plugins.PluginLoader, num uint64) { fn(i, update) } } - _ = pl.Lookup("AppendAncient", func(item interface{}) bool { + appendAncientFnList := pl.Lookup("AppendAncient", func(item interface{}) bool { _, ok := item.(func(number uint64, hash, header, body, receipts, td []byte)) - if ok { log.Warn("PlugEth's AppendAncient is deprecated. Please update to ModifyAncients.") } + if ok { log.Warn("PluGeth's AppendAncient is deprecated. Please update to ModifyAncients.") } return ok }) - // if len(appendAncientFnList) > 0 { - // var ( - // hash []byte - // header []byte - // body []byte - // receipts []byte - // td []byte - // ) - // if hashi, ok := update[freezerHashTable]; ok { - // switch v := hashi.(type) { - // case []byte: - // hash = v - // default: - // hash, _ = rlp.EncodeToBytes(v) - // } - // } - // if headeri, ok := update[freezerHeaderTable]; ok { - // switch v := headeri.(type) { - // case []byte: - // header = v - // default: - // header, _ = rlp.EncodeToBytes(v) - // } - // } - // if bodyi, ok := update[freezerBodiesTable]; ok { - // switch v := bodyi.(type) { - // case []byte: - // body = v - // default: - // body, _ = rlp.EncodeToBytes(v) - // } - // } - // if receiptsi, ok := update[freezerReceiptTable]; ok { - // switch v := receiptsi.(type) { - // case []byte: - // receipts = v - // default: - // receipts, _ = rlp.EncodeToBytes(v) - // } - // } - // if tdi, ok := update[freezerDifficultyTable]; ok { - // switch v := tdi.(type) { - // case []byte: - // td = v - // default: - // td, _ = rlp.EncodeToBytes(v) - // } - // } - // for _, fni := range appendAncientFnList { - // if fn, ok := fni.(func(number uint64, hash, header, body, receipts, td []byte)); ok { - // fn(i, hash, header, body, receipts, td) - // } - // } - // } + if len(appendAncientFnList) > 0 { + var ( + hash []byte + header []byte + body []byte + receipts []byte + td []byte + ) + if hashi, ok := update[chainFreezerHashTable]; ok { + switch v := hashi.(type) { + case []byte: + hash = v + default: + hash, _ = rlp.EncodeToBytes(v) + } + } + if headeri, ok := update[chainFreezerHeaderTable]; ok { + switch v := headeri.(type) { + case []byte: + header = v + default: + header, _ = rlp.EncodeToBytes(v) + } + } + if bodyi, ok := update[chainFreezerBodiesTable]; ok { + switch v := bodyi.(type) { + case []byte: + body = v + default: + body, _ = rlp.EncodeToBytes(v) + } + } + if receiptsi, ok := update[chainFreezerReceiptTable]; ok { + switch v := receiptsi.(type) { + case []byte: + receipts = v + default: + receipts, _ = rlp.EncodeToBytes(v) + } + } + if tdi, ok := update[chainFreezerDifficultyTable]; ok { + switch v := tdi.(type) { + case []byte: + td = v + default: + td, _ = rlp.EncodeToBytes(v) + } + } + for _, fni := range appendAncientFnList { + if fn, ok := fni.(func(number uint64, hash, header, body, receipts, td []byte)); ok { + fn(i, hash, header, body, receipts, td) + } + } + } } } diff --git a/go.mod b/go.mod index cf0ba4ff2..1c9a8dbe1 100644 --- a/go.mod +++ b/go.mod @@ -105,5 +105,3 @@ require ( gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) - -replace github.com/openrelayxyz/plugeth-utils v0.0.18 => ../plugeth-utils diff --git a/plugins/wrappers/backendwrapper/backendwrapper.go b/plugins/wrappers/backendwrapper/backendwrapper.go index a88ed8cac..0027ac223 100644 --- a/plugins/wrappers/backendwrapper/backendwrapper.go +++ b/plugins/wrappers/backendwrapper/backendwrapper.go @@ -173,8 +173,9 @@ func (b *Backend) TxPoolContent() (map[core.Address][][]byte, map[core.Address][ func (b *Backend) BloomStatus() (uint64, uint64) { return b.b.BloomStatus() } -func (b *Backend) GetLogs(ctx context.Context, blockHash core.Hash, number uint64) ([][]byte, error) { - logs, err := b.b.GetLogs(ctx, common.Hash(blockHash), number) +func (b *Backend) GetLogs(ctx context.Context, blockHash core.Hash) ([][]byte, error) { + header, _ := b.b.HeaderByHash(ctx, common.Hash(blockHash)) + logs, err := b.b.GetLogs(ctx, common.Hash(blockHash), uint64(header.Number.Uint64())) if err != nil { return nil, err }