From ea4bc2dbff949932050994d5213d9a41cf4cd6ee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 24 Aug 2021 10:08:08 +0300 Subject: [PATCH 01/73] params: begin Geth v1.10.9 release cycle --- params/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/params/version.go b/params/version.go index 8d222ac04..dd629a100 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 10 // Minor version component of the current release - VersionPatch = 8 // Patch version component of the current release - VersionMeta = "stable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 10 // Minor version component of the current release + VersionPatch = 9 // Patch version component of the current release + VersionMeta = "unstable" // Version metadata to append to the version string ) // Version holds the textual version string. From 79bb9300c1b9fbc00701bd27d5f465df99752870 Mon Sep 17 00:00:00 2001 From: chuwt Date: Tue, 24 Aug 2021 16:51:28 +0800 Subject: [PATCH 02/73] trie, les: add missing calls to Ticker.Stop (#23415) --- les/pruner.go | 1 + trie/sync_bloom.go | 2 ++ 2 files changed, 3 insertions(+) diff --git a/les/pruner.go b/les/pruner.go index 622e64868..a1bd51d86 100644 --- a/les/pruner.go +++ b/les/pruner.go @@ -62,6 +62,7 @@ func (p *pruner) loop() { // cleanTicker is the ticker used to trigger a history clean 2 times a day. var cleanTicker = time.NewTicker(12 * time.Hour) + defer cleanTicker.Stop() // pruning finds the sections that have been processed by all indexers // and deletes all historical chain data. diff --git a/trie/sync_bloom.go b/trie/sync_bloom.go index 49986fcf0..91e5e6711 100644 --- a/trie/sync_bloom.go +++ b/trie/sync_bloom.go @@ -129,6 +129,8 @@ func (b *SyncBloom) init(database ethdb.Iteratee) { func (b *SyncBloom) meter() { // check every second tick := time.NewTicker(1 * time.Second) + defer tick.Stop() + for { select { case <-tick.C: From 8dbf261fd9fc4cd6eca64ebeafee17826e608094 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Tue, 24 Aug 2021 12:22:56 +0200 Subject: [PATCH 03/73] p2p, p2p/enode: fix data races (#23434) In p2p/dial.go, conn.flags was accessed without using sync/atomic. This race is fixed by removing the access. In p2p/enode/iter_test.go, a similar race is resolved by writing the field atomically. Co-authored-by: Felix Lange --- p2p/dial.go | 6 +++--- p2p/enode/iter_test.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/p2p/dial.go b/p2p/dial.go index 83ced3cb3..0d70e6f4a 100644 --- a/p2p/dial.go +++ b/p2p/dial.go @@ -107,7 +107,7 @@ type dialScheduler struct { // Everything below here belongs to loop and // should only be accessed by code on the loop goroutine. dialing map[enode.ID]*dialTask // active tasks - peers map[enode.ID]connFlag // all connected peers + peers map[enode.ID]struct{} // all connected peers dialPeers int // current number of dialed peers // The static map tracks all static dial tasks. The subset of usable static dial tasks @@ -166,7 +166,7 @@ func newDialScheduler(config dialConfig, it enode.Iterator, setupFunc dialSetupF setupFunc: setupFunc, dialing: make(map[enode.ID]*dialTask), static: make(map[enode.ID]*dialTask), - peers: make(map[enode.ID]connFlag), + peers: make(map[enode.ID]struct{}), doneCh: make(chan *dialTask), nodesIn: make(chan *enode.Node), addStaticCh: make(chan *enode.Node), @@ -259,7 +259,7 @@ loop: d.dialPeers++ } id := c.node.ID() - d.peers[id] = c.flags + d.peers[id] = struct{}{} // Remove from static pool because the node is now connected. task := d.static[id] if task != nil && task.staticPoolIndex >= 0 { diff --git a/p2p/enode/iter_test.go b/p2p/enode/iter_test.go index 6009661f3..5014346af 100644 --- a/p2p/enode/iter_test.go +++ b/p2p/enode/iter_test.go @@ -268,7 +268,7 @@ func (s *genIter) Node() *Node { } func (s *genIter) Close() { - s.index = ^uint32(0) + atomic.StoreUint32(&s.index, ^uint32(0)) } func testNode(id, seq uint64) *Node { From 8e0771c21825e9621f29d6c4018038b4505a59f4 Mon Sep 17 00:00:00 2001 From: Pedro Gomes Date: Tue, 24 Aug 2021 12:32:19 +0100 Subject: [PATCH 04/73] core/bloombits: avoid crash when storing errors of different type (#23437) This fixes a rare crash which could occur when two different errors happened in the same bloombits.MatcherSession. --- core/bloombits/matcher.go | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/core/bloombits/matcher.go b/core/bloombits/matcher.go index 927232be0..f2a8bda17 100644 --- a/core/bloombits/matcher.go +++ b/core/bloombits/matcher.go @@ -510,8 +510,9 @@ type MatcherSession struct { closer sync.Once // Sync object to ensure we only ever close once quit chan struct{} // Quit channel to request pipeline termination - ctx context.Context // Context used by the light client to abort filtering - err atomic.Value // Global error to track retrieval failures deep in the chain + ctx context.Context // Context used by the light client to abort filtering + err error // Global error to track retrieval failures deep in the chain + errLock sync.Mutex pend sync.WaitGroup } @@ -529,10 +530,10 @@ func (s *MatcherSession) Close() { // Error returns any failure encountered during the matching session. func (s *MatcherSession) Error() error { - if err := s.err.Load(); err != nil { - return err.(error) - } - return nil + s.errLock.Lock() + defer s.errLock.Unlock() + + return s.err } // allocateRetrieval assigns a bloom bit index to a client process that can either @@ -630,7 +631,9 @@ func (s *MatcherSession) Multiplex(batch int, wait time.Duration, mux chan chan result := <-request if result.Error != nil { - s.err.Store(result.Error) + s.errLock.Lock() + s.err = result.Error + s.errLock.Unlock() s.Close() } s.deliverSections(result.Bit, result.Sections, result.Bitsets) From 5c66bab3b8e6368e66f6bc300ac6816ebd1f2c83 Mon Sep 17 00:00:00 2001 From: ligi Date: Tue, 24 Aug 2021 13:56:31 +0200 Subject: [PATCH 05/73] tests/testdata: make submodule shallow (#23412) Before: 316M .git/modules/tests/ After: 171M .git/modules/tests/ --- .gitmodules | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitmodules b/.gitmodules index 32bdb3b6e..90d1be0a3 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,3 +1,4 @@ [submodule "tests"] path = tests/testdata url = https://github.com/ethereum/tests + shallow = true From 0b4097748016395b14ecdc316d2095eb7c968f2a Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Tue, 24 Aug 2021 13:57:05 +0200 Subject: [PATCH 06/73] core/vm: fix typo in comment (#23450) --- core/vm/gas_table.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/vm/gas_table.go b/core/vm/gas_table.go index 944b6cf0a..19d2198af 100644 --- a/core/vm/gas_table.go +++ b/core/vm/gas_table.go @@ -60,7 +60,7 @@ func memoryGasCost(mem *Memory, newMemSize uint64) (uint64, error) { // as argument: // CALLDATACOPY (stack position 2) // CODECOPY (stack position 2) -// EXTCODECOPY (stack poition 3) +// EXTCODECOPY (stack position 3) // RETURNDATACOPY (stack position 2) func memoryCopierGas(stackpos int) gasFunc { return func(evm *EVM, contract *Contract, stack *Stack, mem *Memory, memorySize uint64) (uint64, error) { From a0a4a153e98f21b014e7ad289816514751d86568 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 24 Aug 2021 13:59:15 +0200 Subject: [PATCH 07/73] core/types: add benchmarks for rlp encoding/decoding (#23190) Co-authored-by: Felix Lange Co-authored-by: Sina Mahmoodi <1591639+s1na@users.noreply.github.com> --- core/types/types_test.go | 148 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 148 insertions(+) create mode 100644 core/types/types_test.go diff --git a/core/types/types_test.go b/core/types/types_test.go new file mode 100644 index 000000000..1fb386d5d --- /dev/null +++ b/core/types/types_test.go @@ -0,0 +1,148 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package types + +import ( + "math/big" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/rlp" +) + +type devnull struct{ len int } + +func (d *devnull) Write(p []byte) (int, error) { + d.len += len(p) + return len(p), nil +} + +func BenchmarkEncodeRLP(b *testing.B) { + benchRLP(b, true) +} + +func BenchmarkDecodeRLP(b *testing.B) { + benchRLP(b, false) +} + +func benchRLP(b *testing.B, encode bool) { + key, _ := crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + to := common.HexToAddress("0x00000000000000000000000000000000deadbeef") + signer := NewLondonSigner(big.NewInt(1337)) + for _, tc := range []struct { + name string + obj interface{} + }{ + { + "legacy-header", + &Header{ + Difficulty: big.NewInt(10000000000), + Number: big.NewInt(1000), + GasLimit: 8_000_000, + GasUsed: 8_000_000, + Time: 555, + Extra: make([]byte, 32), + }, + }, + { + "london-header", + &Header{ + Difficulty: big.NewInt(10000000000), + Number: big.NewInt(1000), + GasLimit: 8_000_000, + GasUsed: 8_000_000, + Time: 555, + Extra: make([]byte, 32), + BaseFee: big.NewInt(10000000000), + }, + }, + { + "receipt-for-storage", + &ReceiptForStorage{ + Status: ReceiptStatusSuccessful, + CumulativeGasUsed: 0x888888888, + Logs: make([]*Log, 0), + }, + }, + { + "receipt-full", + &Receipt{ + Status: ReceiptStatusSuccessful, + CumulativeGasUsed: 0x888888888, + Logs: make([]*Log, 0), + }, + }, + { + "legacy-transaction", + MustSignNewTx(key, signer, + &LegacyTx{ + Nonce: 1, + GasPrice: big.NewInt(500), + Gas: 1000000, + To: &to, + Value: big.NewInt(1), + }), + }, + { + "access-transaction", + MustSignNewTx(key, signer, + &AccessListTx{ + Nonce: 1, + GasPrice: big.NewInt(500), + Gas: 1000000, + To: &to, + Value: big.NewInt(1), + }), + }, + { + "1559-transaction", + MustSignNewTx(key, signer, + &DynamicFeeTx{ + Nonce: 1, + Gas: 1000000, + To: &to, + Value: big.NewInt(1), + GasTipCap: big.NewInt(500), + GasFeeCap: big.NewInt(500), + }), + }, + } { + if encode { + b.Run(tc.name, func(b *testing.B) { + b.ReportAllocs() + var null = &devnull{} + for i := 0; i < b.N; i++ { + rlp.Encode(null, tc.obj) + } + b.SetBytes(int64(null.len / b.N)) + }) + } else { + data, _ := rlp.EncodeToBytes(tc.obj) + // Test decoding + b.Run(tc.name, func(b *testing.B) { + b.ReportAllocs() + for i := 0; i < b.N; i++ { + if err := rlp.DecodeBytes(data, tc.obj); err != nil { + b.Fatal(err) + } + } + b.SetBytes(int64(len(data))) + }) + } + } +} From 85126c4eb99ab4b2ccd74c1ee66d2aba0b54a767 Mon Sep 17 00:00:00 2001 From: Zachinquarantine Date: Tue, 24 Aug 2021 08:10:52 -0400 Subject: [PATCH 08/73] node: add comment about --nousb being deprecated (#23439) Co-authored-by: Felix Lange --- node/config.go | 1 + 1 file changed, 1 insertion(+) diff --git a/node/config.go b/node/config.go index ef1da15d7..b7e99eb7b 100644 --- a/node/config.go +++ b/node/config.go @@ -93,6 +93,7 @@ type Config struct { InsecureUnlockAllowed bool `toml:",omitempty"` // NoUSB disables hardware wallet monitoring and connectivity. + // Deprecated: USB monitoring is disabled by default and must be enabled explicitly. NoUSB bool `toml:",omitempty"` // USB enables hardware wallet monitoring and connectivity. From 5cee33eb7234756939b1ef8b23e990b672a03dff Mon Sep 17 00:00:00 2001 From: baptiste-b-pegasys <85155432+baptiste-b-pegasys@users.noreply.github.com> Date: Tue, 24 Aug 2021 14:27:15 +0200 Subject: [PATCH 09/73] tests/fuzzers: fix go vet warning about ReadByte (#23380) --- tests/fuzzers/trie/trie-fuzzer.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/fuzzers/trie/trie-fuzzer.go b/tests/fuzzers/trie/trie-fuzzer.go index 762ab5f34..d0b614c9a 100644 --- a/tests/fuzzers/trie/trie-fuzzer.go +++ b/tests/fuzzers/trie/trie-fuzzer.go @@ -69,7 +69,7 @@ func newDataSource(input []byte) *dataSource { input, bytes.NewReader(input), } } -func (ds *dataSource) ReadByte() byte { +func (ds *dataSource) readByte() byte { if b, err := ds.reader.ReadByte(); err != nil { return 0 } else { @@ -89,22 +89,22 @@ func Generate(input []byte) randTest { r := newDataSource(input) genKey := func() []byte { - if len(allKeys) < 2 || r.ReadByte() < 0x0f { + if len(allKeys) < 2 || r.readByte() < 0x0f { // new key - key := make([]byte, r.ReadByte()%50) + key := make([]byte, r.readByte()%50) r.Read(key) allKeys = append(allKeys, key) return key } // use existing key - return allKeys[int(r.ReadByte())%len(allKeys)] + return allKeys[int(r.readByte())%len(allKeys)] } var steps randTest for i := 0; !r.Ended(); i++ { - step := randTestStep{op: int(r.ReadByte()) % opMax} + step := randTestStep{op: int(r.readByte()) % opMax} switch step.op { case opUpdate: step.key = genKey() From d705f5a5543402a1d505bdc411d264cd4f883402 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 24 Aug 2021 20:48:36 +0200 Subject: [PATCH 10/73] core: make txpool reject too sudden changes (#23095) * core: make txpool reject too sudden changes * core: add some metrics to txpool --- core/tx_pool.go | 26 ++++++++++++++++++++++++++ core/tx_pool_test.go | 14 +++++++------- 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/core/tx_pool.go b/core/tx_pool.go index f6228df2f..ee56dae88 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -111,6 +111,14 @@ var ( invalidTxMeter = metrics.NewRegisteredMeter("txpool/invalid", nil) underpricedTxMeter = metrics.NewRegisteredMeter("txpool/underpriced", nil) overflowedTxMeter = metrics.NewRegisteredMeter("txpool/overflowed", nil) + // throttleTxMeter counts how many transactions are rejected due to too-many-changes between + // txpool reorgs. + throttleTxMeter = metrics.NewRegisteredMeter("txpool/throttle", nil) + // reorgDurationTimer measures how long time a txpool reorg takes. + reorgDurationTimer = metrics.NewRegisteredTimer("txpool/reorgtime", nil) + // dropBetweenReorgHistogram counts how many drops we experience between two reorg runs. It is expected + // that this number is pretty low, since txpool reorgs happen very frequently. + dropBetweenReorgHistogram = metrics.NewRegisteredHistogram("txpool/dropbetweenreorg", nil, metrics.NewExpDecaySample(1028, 0.015)) pendingGauge = metrics.NewRegisteredGauge("txpool/pending", nil) queuedGauge = metrics.NewRegisteredGauge("txpool/queued", nil) @@ -256,6 +264,8 @@ type TxPool struct { reorgDoneCh chan chan struct{} reorgShutdownCh chan struct{} // requests shutdown of scheduleReorgLoop wg sync.WaitGroup // tracks loop, scheduleReorgLoop + + changesSinceReorg int // A counter for how many drops we've performed in-between reorg. } type txpoolResetRequest struct { @@ -663,6 +673,15 @@ func (pool *TxPool) add(tx *types.Transaction, local bool) (replaced bool, err e underpricedTxMeter.Mark(1) return false, ErrUnderpriced } + // We're about to replace a transaction. The reorg does a more thorough + // analysis of what to remove and how, but it runs async. We don't want to + // do too many replacements between reorg-runs, so we cap the number of + // replacements to 25% of the slots + if pool.changesSinceReorg > int(pool.config.GlobalSlots/4) { + throttleTxMeter.Mark(1) + return false, ErrTxPoolOverflow + } + // New transaction is better than our worse ones, make room for it. // If it's a local transaction, forcibly discard all available transactions. // Otherwise if we can't make enough room for new one, abort the operation. @@ -674,6 +693,8 @@ func (pool *TxPool) add(tx *types.Transaction, local bool) (replaced bool, err e overflowedTxMeter.Mark(1) return false, ErrTxPoolOverflow } + // Bump the counter of rejections-since-reorg + pool.changesSinceReorg += len(drop) // Kick out the underpriced remote transactions. for _, tx := range drop { log.Trace("Discarding freshly underpriced transaction", "hash", tx.Hash(), "gasTipCap", tx.GasTipCap(), "gasFeeCap", tx.GasFeeCap()) @@ -1114,6 +1135,9 @@ func (pool *TxPool) scheduleReorgLoop() { // runReorg runs reset and promoteExecutables on behalf of scheduleReorgLoop. func (pool *TxPool) runReorg(done chan struct{}, reset *txpoolResetRequest, dirtyAccounts *accountSet, events map[common.Address]*txSortedMap) { + defer func(t0 time.Time) { + reorgDurationTimer.Update(time.Since(t0)) + }(time.Now()) defer close(done) var promoteAddrs []common.Address @@ -1163,6 +1187,8 @@ func (pool *TxPool) runReorg(done chan struct{}, reset *txpoolResetRequest, dirt highestPending := list.LastElement() pool.pendingNonces.set(addr, highestPending.Nonce()+1) } + dropBetweenReorgHistogram.Update(int64(pool.changesSinceReorg)) + pool.changesSinceReorg = 0 // Reset change counter pool.mu.Unlock() // Notify subsystems for newly added transactions diff --git a/core/tx_pool_test.go b/core/tx_pool_test.go index e02096fe2..f86f64bf2 100644 --- a/core/tx_pool_test.go +++ b/core/tx_pool_test.go @@ -1946,20 +1946,20 @@ func TestDualHeapEviction(t *testing.T) { } add := func(urgent bool) { - txs := make([]*types.Transaction, 20) - for i := range txs { + for i := 0; i < 20; i++ { + var tx *types.Transaction // Create a test accounts and fund it key, _ := crypto.GenerateKey() testAddBalance(pool, crypto.PubkeyToAddress(key.PublicKey), big.NewInt(1000000000000)) if urgent { - txs[i] = dynamicFeeTx(0, 100000, big.NewInt(int64(baseFee+1+i)), big.NewInt(int64(1+i)), key) - highTip = txs[i] + tx = dynamicFeeTx(0, 100000, big.NewInt(int64(baseFee+1+i)), big.NewInt(int64(1+i)), key) + highTip = tx } else { - txs[i] = dynamicFeeTx(0, 100000, big.NewInt(int64(baseFee+200+i)), big.NewInt(1), key) - highCap = txs[i] + tx = dynamicFeeTx(0, 100000, big.NewInt(int64(baseFee+200+i)), big.NewInt(1), key) + highCap = tx } + pool.AddRemotesSync([]*types.Transaction{tx}) } - pool.AddRemotes(txs) pending, queued := pool.Stats() if pending+queued != 20 { t.Fatalf("transaction count mismatch: have %d, want %d", pending+queued, 10) From 0db0b277545383d952132f126d4a8edd05138e6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 24 Aug 2021 21:52:58 +0300 Subject: [PATCH 11/73] Revert "Revert "eth: drop eth/65, the last non-reqid protocol version" (#23426)" (#23456) This reverts commit c368f728c19e7fd7a9613513edda68ffcb503af0. --- cmd/devp2p/internal/ethtest/suite_test.go | 2 +- eth/downloader/downloader.go | 4 +- eth/downloader/downloader_test.go | 95 +---------- eth/downloader/peer.go | 8 +- eth/handler.go | 5 +- eth/handler_eth_test.go | 38 +++-- eth/protocols/eth/handler.go | 47 ++---- eth/protocols/eth/handler_test.go | 134 ++++++--------- eth/protocols/eth/handlers.go | 107 ------------ eth/protocols/eth/handshake_test.go | 1 - eth/protocols/eth/peer.go | 194 +++++++--------------- eth/protocols/eth/protocol.go | 34 ++-- eth/sync.go | 102 +----------- eth/sync_test.go | 1 - les/client_handler.go | 2 +- 15 files changed, 172 insertions(+), 602 deletions(-) diff --git a/cmd/devp2p/internal/ethtest/suite_test.go b/cmd/devp2p/internal/ethtest/suite_test.go index 6e3217151..6d14404e6 100644 --- a/cmd/devp2p/internal/ethtest/suite_test.go +++ b/cmd/devp2p/internal/ethtest/suite_test.go @@ -45,7 +45,7 @@ func TestEthSuite(t *testing.T) { if err != nil { t.Fatalf("could not create new test suite: %v", err) } - for _, test := range suite.AllEthTests() { + for _, test := range suite.Eth66Tests() { t.Run(test.Name, func(t *testing.T) { result := utesting.RunTAP([]utesting.Test{{Name: test.Name, Fn: test.Fn}}, os.Stdout) if result[0].Failed { diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index 1767506a3..a6bf87acb 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -448,8 +448,8 @@ func (d *Downloader) syncWithPeer(p *peerConnection, hash common.Hash, td *big.I d.mux.Post(DoneEvent{latest}) } }() - if p.version < eth.ETH65 { - return fmt.Errorf("%w: advertized %d < required %d", errTooOld, p.version, eth.ETH65) + if p.version < eth.ETH66 { + return fmt.Errorf("%w: advertized %d < required %d", errTooOld, p.version, eth.ETH66) } mode := d.getMode() diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index 794160993..17cd3630c 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -522,10 +522,6 @@ func assertOwnForkedChain(t *testing.T, tester *downloadTester, common int, leng } } -func TestCanonicalSynchronisation65Full(t *testing.T) { testCanonSync(t, eth.ETH65, FullSync) } -func TestCanonicalSynchronisation65Fast(t *testing.T) { testCanonSync(t, eth.ETH65, FastSync) } -func TestCanonicalSynchronisation65Light(t *testing.T) { testCanonSync(t, eth.ETH65, LightSync) } - func TestCanonicalSynchronisation66Full(t *testing.T) { testCanonSync(t, eth.ETH66, FullSync) } func TestCanonicalSynchronisation66Fast(t *testing.T) { testCanonSync(t, eth.ETH66, FastSync) } func TestCanonicalSynchronisation66Light(t *testing.T) { testCanonSync(t, eth.ETH66, LightSync) } @@ -549,9 +545,6 @@ func testCanonSync(t *testing.T, protocol uint, mode SyncMode) { // Tests that if a large batch of blocks are being downloaded, it is throttled // until the cached blocks are retrieved. -func TestThrottling65Full(t *testing.T) { testThrottling(t, eth.ETH65, FullSync) } -func TestThrottling65Fast(t *testing.T) { testThrottling(t, eth.ETH65, FastSync) } - func TestThrottling66Full(t *testing.T) { testThrottling(t, eth.ETH66, FullSync) } func TestThrottling66Fast(t *testing.T) { testThrottling(t, eth.ETH66, FastSync) } @@ -634,10 +627,6 @@ func testThrottling(t *testing.T, protocol uint, mode SyncMode) { // Tests that simple synchronization against a forked chain works correctly. In // this test common ancestor lookup should *not* be short circuited, and a full // binary search should be executed. -func TestForkedSync65Full(t *testing.T) { testForkedSync(t, eth.ETH65, FullSync) } -func TestForkedSync65Fast(t *testing.T) { testForkedSync(t, eth.ETH65, FastSync) } -func TestForkedSync65Light(t *testing.T) { testForkedSync(t, eth.ETH65, LightSync) } - func TestForkedSync66Full(t *testing.T) { testForkedSync(t, eth.ETH66, FullSync) } func TestForkedSync66Fast(t *testing.T) { testForkedSync(t, eth.ETH66, FastSync) } func TestForkedSync66Light(t *testing.T) { testForkedSync(t, eth.ETH66, LightSync) } @@ -667,10 +656,6 @@ func testForkedSync(t *testing.T, protocol uint, mode SyncMode) { // Tests that synchronising against a much shorter but much heavyer fork works // corrently and is not dropped. -func TestHeavyForkedSync65Full(t *testing.T) { testHeavyForkedSync(t, eth.ETH65, FullSync) } -func TestHeavyForkedSync65Fast(t *testing.T) { testHeavyForkedSync(t, eth.ETH65, FastSync) } -func TestHeavyForkedSync65Light(t *testing.T) { testHeavyForkedSync(t, eth.ETH65, LightSync) } - func TestHeavyForkedSync66Full(t *testing.T) { testHeavyForkedSync(t, eth.ETH66, FullSync) } func TestHeavyForkedSync66Fast(t *testing.T) { testHeavyForkedSync(t, eth.ETH66, FastSync) } func TestHeavyForkedSync66Light(t *testing.T) { testHeavyForkedSync(t, eth.ETH66, LightSync) } @@ -702,10 +687,6 @@ func testHeavyForkedSync(t *testing.T, protocol uint, mode SyncMode) { // Tests that chain forks are contained within a certain interval of the current // chain head, ensuring that malicious peers cannot waste resources by feeding // long dead chains. -func TestBoundedForkedSync65Full(t *testing.T) { testBoundedForkedSync(t, eth.ETH65, FullSync) } -func TestBoundedForkedSync65Fast(t *testing.T) { testBoundedForkedSync(t, eth.ETH65, FastSync) } -func TestBoundedForkedSync65Light(t *testing.T) { testBoundedForkedSync(t, eth.ETH65, LightSync) } - func TestBoundedForkedSync66Full(t *testing.T) { testBoundedForkedSync(t, eth.ETH66, FullSync) } func TestBoundedForkedSync66Fast(t *testing.T) { testBoundedForkedSync(t, eth.ETH66, FastSync) } func TestBoundedForkedSync66Light(t *testing.T) { testBoundedForkedSync(t, eth.ETH66, LightSync) } @@ -736,16 +717,6 @@ func testBoundedForkedSync(t *testing.T, protocol uint, mode SyncMode) { // Tests that chain forks are contained within a certain interval of the current // chain head for short but heavy forks too. These are a bit special because they // take different ancestor lookup paths. -func TestBoundedHeavyForkedSync65Full(t *testing.T) { - testBoundedHeavyForkedSync(t, eth.ETH65, FullSync) -} -func TestBoundedHeavyForkedSync65Fast(t *testing.T) { - testBoundedHeavyForkedSync(t, eth.ETH65, FastSync) -} -func TestBoundedHeavyForkedSync65Light(t *testing.T) { - testBoundedHeavyForkedSync(t, eth.ETH65, LightSync) -} - func TestBoundedHeavyForkedSync66Full(t *testing.T) { testBoundedHeavyForkedSync(t, eth.ETH66, FullSync) } @@ -800,10 +771,6 @@ func TestInactiveDownloader63(t *testing.T) { } // Tests that a canceled download wipes all previously accumulated state. -func TestCancel65Full(t *testing.T) { testCancel(t, eth.ETH65, FullSync) } -func TestCancel65Fast(t *testing.T) { testCancel(t, eth.ETH65, FastSync) } -func TestCancel65Light(t *testing.T) { testCancel(t, eth.ETH65, LightSync) } - func TestCancel66Full(t *testing.T) { testCancel(t, eth.ETH66, FullSync) } func TestCancel66Fast(t *testing.T) { testCancel(t, eth.ETH66, FastSync) } func TestCancel66Light(t *testing.T) { testCancel(t, eth.ETH66, LightSync) } @@ -833,10 +800,6 @@ func testCancel(t *testing.T, protocol uint, mode SyncMode) { } // Tests that synchronisation from multiple peers works as intended (multi thread sanity test). -func TestMultiSynchronisation65Full(t *testing.T) { testMultiSynchronisation(t, eth.ETH65, FullSync) } -func TestMultiSynchronisation65Fast(t *testing.T) { testMultiSynchronisation(t, eth.ETH65, FastSync) } -func TestMultiSynchronisation65Light(t *testing.T) { testMultiSynchronisation(t, eth.ETH65, LightSync) } - func TestMultiSynchronisation66Full(t *testing.T) { testMultiSynchronisation(t, eth.ETH66, FullSync) } func TestMultiSynchronisation66Fast(t *testing.T) { testMultiSynchronisation(t, eth.ETH66, FastSync) } func TestMultiSynchronisation66Light(t *testing.T) { testMultiSynchronisation(t, eth.ETH66, LightSync) } @@ -863,10 +826,6 @@ func testMultiSynchronisation(t *testing.T, protocol uint, mode SyncMode) { // Tests that synchronisations behave well in multi-version protocol environments // and not wreak havoc on other nodes in the network. -func TestMultiProtoSynchronisation65Full(t *testing.T) { testMultiProtoSync(t, eth.ETH65, FullSync) } -func TestMultiProtoSynchronisation65Fast(t *testing.T) { testMultiProtoSync(t, eth.ETH65, FastSync) } -func TestMultiProtoSynchronisation65Light(t *testing.T) { testMultiProtoSync(t, eth.ETH65, LightSync) } - func TestMultiProtoSynchronisation66Full(t *testing.T) { testMultiProtoSync(t, eth.ETH66, FullSync) } func TestMultiProtoSynchronisation66Fast(t *testing.T) { testMultiProtoSync(t, eth.ETH66, FastSync) } func TestMultiProtoSynchronisation66Light(t *testing.T) { testMultiProtoSync(t, eth.ETH66, LightSync) } @@ -881,8 +840,8 @@ func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { chain := testChainBase.shorten(blockCacheMaxItems - 15) // Create peers of every type - tester.newPeer("peer 65", eth.ETH65, chain) tester.newPeer("peer 66", eth.ETH66, chain) + //tester.newPeer("peer 65", eth.ETH67, chain) // Synchronise with the requested peer and make sure all blocks were retrieved if err := tester.sync(fmt.Sprintf("peer %d", protocol), nil, mode); err != nil { @@ -891,7 +850,7 @@ func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { assertOwnChain(t, tester, chain.len()) // Check that no peers have been dropped off - for _, version := range []int{65, 66} { + for _, version := range []int{66} { peer := fmt.Sprintf("peer %d", version) if _, ok := tester.peers[peer]; !ok { t.Errorf("%s dropped", peer) @@ -901,10 +860,6 @@ func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { // Tests that if a block is empty (e.g. header only), no body request should be // made, and instead the header should be assembled into a whole block in itself. -func TestEmptyShortCircuit65Full(t *testing.T) { testEmptyShortCircuit(t, eth.ETH65, FullSync) } -func TestEmptyShortCircuit65Fast(t *testing.T) { testEmptyShortCircuit(t, eth.ETH65, FastSync) } -func TestEmptyShortCircuit65Light(t *testing.T) { testEmptyShortCircuit(t, eth.ETH65, LightSync) } - func TestEmptyShortCircuit66Full(t *testing.T) { testEmptyShortCircuit(t, eth.ETH66, FullSync) } func TestEmptyShortCircuit66Fast(t *testing.T) { testEmptyShortCircuit(t, eth.ETH66, FastSync) } func TestEmptyShortCircuit66Light(t *testing.T) { testEmptyShortCircuit(t, eth.ETH66, LightSync) } @@ -955,10 +910,6 @@ func testEmptyShortCircuit(t *testing.T, protocol uint, mode SyncMode) { // Tests that headers are enqueued continuously, preventing malicious nodes from // stalling the downloader by feeding gapped header chains. -func TestMissingHeaderAttack65Full(t *testing.T) { testMissingHeaderAttack(t, eth.ETH65, FullSync) } -func TestMissingHeaderAttack65Fast(t *testing.T) { testMissingHeaderAttack(t, eth.ETH65, FastSync) } -func TestMissingHeaderAttack65Light(t *testing.T) { testMissingHeaderAttack(t, eth.ETH65, LightSync) } - func TestMissingHeaderAttack66Full(t *testing.T) { testMissingHeaderAttack(t, eth.ETH66, FullSync) } func TestMissingHeaderAttack66Fast(t *testing.T) { testMissingHeaderAttack(t, eth.ETH66, FastSync) } func TestMissingHeaderAttack66Light(t *testing.T) { testMissingHeaderAttack(t, eth.ETH66, LightSync) } @@ -987,10 +938,6 @@ func testMissingHeaderAttack(t *testing.T, protocol uint, mode SyncMode) { // Tests that if requested headers are shifted (i.e. first is missing), the queue // detects the invalid numbering. -func TestShiftedHeaderAttack65Full(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH65, FullSync) } -func TestShiftedHeaderAttack65Fast(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH65, FastSync) } -func TestShiftedHeaderAttack65Light(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH65, LightSync) } - func TestShiftedHeaderAttack66Full(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH66, FullSync) } func TestShiftedHeaderAttack66Fast(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH66, FastSync) } func TestShiftedHeaderAttack66Light(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH66, LightSync) } @@ -1024,7 +971,6 @@ func testShiftedHeaderAttack(t *testing.T, protocol uint, mode SyncMode) { // Tests that upon detecting an invalid header, the recent ones are rolled back // for various failure scenarios. Afterwards a full sync is attempted to make // sure no state was corrupted. -func TestInvalidHeaderRollback65Fast(t *testing.T) { testInvalidHeaderRollback(t, eth.ETH65, FastSync) } func TestInvalidHeaderRollback66Fast(t *testing.T) { testInvalidHeaderRollback(t, eth.ETH66, FastSync) } func testInvalidHeaderRollback(t *testing.T, protocol uint, mode SyncMode) { @@ -1115,16 +1061,6 @@ func testInvalidHeaderRollback(t *testing.T, protocol uint, mode SyncMode) { // Tests that a peer advertising a high TD doesn't get to stall the downloader // afterwards by not sending any useful hashes. -func TestHighTDStarvationAttack65Full(t *testing.T) { - testHighTDStarvationAttack(t, eth.ETH65, FullSync) -} -func TestHighTDStarvationAttack65Fast(t *testing.T) { - testHighTDStarvationAttack(t, eth.ETH65, FastSync) -} -func TestHighTDStarvationAttack65Light(t *testing.T) { - testHighTDStarvationAttack(t, eth.ETH65, LightSync) -} - func TestHighTDStarvationAttack66Full(t *testing.T) { testHighTDStarvationAttack(t, eth.ETH66, FullSync) } @@ -1149,7 +1085,6 @@ func testHighTDStarvationAttack(t *testing.T, protocol uint, mode SyncMode) { } // Tests that misbehaving peers are disconnected, whilst behaving ones are not. -func TestBlockHeaderAttackerDropping65(t *testing.T) { testBlockHeaderAttackerDropping(t, eth.ETH65) } func TestBlockHeaderAttackerDropping66(t *testing.T) { testBlockHeaderAttackerDropping(t, eth.ETH66) } func testBlockHeaderAttackerDropping(t *testing.T, protocol uint) { @@ -1202,10 +1137,6 @@ func testBlockHeaderAttackerDropping(t *testing.T, protocol uint) { // Tests that synchronisation progress (origin block number, current block number // and highest block number) is tracked and updated correctly. -func TestSyncProgress65Full(t *testing.T) { testSyncProgress(t, eth.ETH65, FullSync) } -func TestSyncProgress65Fast(t *testing.T) { testSyncProgress(t, eth.ETH65, FastSync) } -func TestSyncProgress65Light(t *testing.T) { testSyncProgress(t, eth.ETH65, LightSync) } - func TestSyncProgress66Full(t *testing.T) { testSyncProgress(t, eth.ETH66, FullSync) } func TestSyncProgress66Fast(t *testing.T) { testSyncProgress(t, eth.ETH66, FastSync) } func TestSyncProgress66Light(t *testing.T) { testSyncProgress(t, eth.ETH66, LightSync) } @@ -1286,10 +1217,6 @@ func checkProgress(t *testing.T, d *Downloader, stage string, want ethereum.Sync // Tests that synchronisation progress (origin block number and highest block // number) is tracked and updated correctly in case of a fork (or manual head // revertal). -func TestForkedSyncProgress65Full(t *testing.T) { testForkedSyncProgress(t, eth.ETH65, FullSync) } -func TestForkedSyncProgress65Fast(t *testing.T) { testForkedSyncProgress(t, eth.ETH65, FastSync) } -func TestForkedSyncProgress65Light(t *testing.T) { testForkedSyncProgress(t, eth.ETH65, LightSync) } - func TestForkedSyncProgress66Full(t *testing.T) { testForkedSyncProgress(t, eth.ETH66, FullSync) } func TestForkedSyncProgress66Fast(t *testing.T) { testForkedSyncProgress(t, eth.ETH66, FastSync) } func TestForkedSyncProgress66Light(t *testing.T) { testForkedSyncProgress(t, eth.ETH66, LightSync) } @@ -1362,10 +1289,6 @@ func testForkedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { // Tests that if synchronisation is aborted due to some failure, then the progress // origin is not updated in the next sync cycle, as it should be considered the // continuation of the previous sync and not a new instance. -func TestFailedSyncProgress65Full(t *testing.T) { testFailedSyncProgress(t, eth.ETH65, FullSync) } -func TestFailedSyncProgress65Fast(t *testing.T) { testFailedSyncProgress(t, eth.ETH65, FastSync) } -func TestFailedSyncProgress65Light(t *testing.T) { testFailedSyncProgress(t, eth.ETH65, LightSync) } - func TestFailedSyncProgress66Full(t *testing.T) { testFailedSyncProgress(t, eth.ETH66, FullSync) } func TestFailedSyncProgress66Fast(t *testing.T) { testFailedSyncProgress(t, eth.ETH66, FastSync) } func TestFailedSyncProgress66Light(t *testing.T) { testFailedSyncProgress(t, eth.ETH66, LightSync) } @@ -1435,10 +1358,6 @@ func testFailedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { // Tests that if an attacker fakes a chain height, after the attack is detected, // the progress height is successfully reduced at the next sync invocation. -func TestFakedSyncProgress65Full(t *testing.T) { testFakedSyncProgress(t, eth.ETH65, FullSync) } -func TestFakedSyncProgress65Fast(t *testing.T) { testFakedSyncProgress(t, eth.ETH65, FastSync) } -func TestFakedSyncProgress65Light(t *testing.T) { testFakedSyncProgress(t, eth.ETH65, LightSync) } - func TestFakedSyncProgress66Full(t *testing.T) { testFakedSyncProgress(t, eth.ETH66, FullSync) } func TestFakedSyncProgress66Fast(t *testing.T) { testFakedSyncProgress(t, eth.ETH66, FastSync) } func TestFakedSyncProgress66Light(t *testing.T) { testFakedSyncProgress(t, eth.ETH66, LightSync) } @@ -1512,10 +1431,6 @@ func testFakedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { // This test reproduces an issue where unexpected deliveries would // block indefinitely if they arrived at the right time. -func TestDeliverHeadersHang65Full(t *testing.T) { testDeliverHeadersHang(t, eth.ETH65, FullSync) } -func TestDeliverHeadersHang65Fast(t *testing.T) { testDeliverHeadersHang(t, eth.ETH65, FastSync) } -func TestDeliverHeadersHang65Light(t *testing.T) { testDeliverHeadersHang(t, eth.ETH65, LightSync) } - func TestDeliverHeadersHang66Full(t *testing.T) { testDeliverHeadersHang(t, eth.ETH66, FullSync) } func TestDeliverHeadersHang66Fast(t *testing.T) { testDeliverHeadersHang(t, eth.ETH66, FastSync) } func TestDeliverHeadersHang66Light(t *testing.T) { testDeliverHeadersHang(t, eth.ETH66, LightSync) } @@ -1673,12 +1588,6 @@ func TestRemoteHeaderRequestSpan(t *testing.T) { // Tests that peers below a pre-configured checkpoint block are prevented from // being fast-synced from, avoiding potential cheap eclipse attacks. -func TestCheckpointEnforcement65Full(t *testing.T) { testCheckpointEnforcement(t, eth.ETH65, FullSync) } -func TestCheckpointEnforcement65Fast(t *testing.T) { testCheckpointEnforcement(t, eth.ETH65, FastSync) } -func TestCheckpointEnforcement65Light(t *testing.T) { - testCheckpointEnforcement(t, eth.ETH65, LightSync) -} - func TestCheckpointEnforcement66Full(t *testing.T) { testCheckpointEnforcement(t, eth.ETH66, FullSync) } func TestCheckpointEnforcement66Fast(t *testing.T) { testCheckpointEnforcement(t, eth.ETH66, FastSync) } func TestCheckpointEnforcement66Light(t *testing.T) { diff --git a/eth/downloader/peer.go b/eth/downloader/peer.go index 066a36631..863294832 100644 --- a/eth/downloader/peer.go +++ b/eth/downloader/peer.go @@ -413,7 +413,7 @@ func (ps *peerSet) HeaderIdlePeers() ([]*peerConnection, int) { throughput := func(p *peerConnection) int { return p.rates.Capacity(eth.BlockHeadersMsg, time.Second) } - return ps.idlePeers(eth.ETH65, eth.ETH66, idle, throughput) + return ps.idlePeers(eth.ETH66, eth.ETH66, idle, throughput) } // BodyIdlePeers retrieves a flat list of all the currently body-idle peers within @@ -425,7 +425,7 @@ func (ps *peerSet) BodyIdlePeers() ([]*peerConnection, int) { throughput := func(p *peerConnection) int { return p.rates.Capacity(eth.BlockBodiesMsg, time.Second) } - return ps.idlePeers(eth.ETH65, eth.ETH66, idle, throughput) + return ps.idlePeers(eth.ETH66, eth.ETH66, idle, throughput) } // ReceiptIdlePeers retrieves a flat list of all the currently receipt-idle peers @@ -437,7 +437,7 @@ func (ps *peerSet) ReceiptIdlePeers() ([]*peerConnection, int) { throughput := func(p *peerConnection) int { return p.rates.Capacity(eth.ReceiptsMsg, time.Second) } - return ps.idlePeers(eth.ETH65, eth.ETH66, idle, throughput) + return ps.idlePeers(eth.ETH66, eth.ETH66, idle, throughput) } // NodeDataIdlePeers retrieves a flat list of all the currently node-data-idle @@ -449,7 +449,7 @@ func (ps *peerSet) NodeDataIdlePeers() ([]*peerConnection, int) { throughput := func(p *peerConnection) int { return p.rates.Capacity(eth.NodeDataMsg, time.Second) } - return ps.idlePeers(eth.ETH65, eth.ETH66, idle, throughput) + return ps.idlePeers(eth.ETH66, eth.ETH66, idle, throughput) } // idlePeers retrieves a flat list of all currently idle peers satisfying the diff --git a/eth/handler.go b/eth/handler.go index aff4871af..06a8088bf 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -117,7 +117,6 @@ type handler struct { whitelist map[uint64]common.Hash // channels for fetcher, syncer, txsyncLoop - txsyncCh chan *txsync quitSync chan struct{} chainSync *chainSyncer @@ -140,7 +139,6 @@ func newHandler(config *handlerConfig) (*handler, error) { chain: config.Chain, peers: newPeerSet(), whitelist: config.Whitelist, - txsyncCh: make(chan *txsync), quitSync: make(chan struct{}), } if config.Sync == downloader.FullSync { @@ -408,9 +406,8 @@ func (h *handler) Start(maxPeers int) { go h.minedBroadcastLoop() // start sync handlers - h.wg.Add(2) + h.wg.Add(1) go h.chainSync.loop() - go h.txsyncLoop64() // TODO(karalabe): Legacy initial tx echange, drop with eth/64. } func (h *handler) Stop() { diff --git a/eth/handler_eth_test.go b/eth/handler_eth_test.go index 038de4699..039091244 100644 --- a/eth/handler_eth_test.go +++ b/eth/handler_eth_test.go @@ -80,7 +80,6 @@ func (h *testEthHandler) Handle(peer *eth.Peer, packet eth.Packet) error { // Tests that peers are correctly accepted (or rejected) based on the advertised // fork IDs in the protocol handshake. -func TestForkIDSplit65(t *testing.T) { testForkIDSplit(t, eth.ETH65) } func TestForkIDSplit66(t *testing.T) { testForkIDSplit(t, eth.ETH66) } func testForkIDSplit(t *testing.T, protocol uint) { @@ -236,7 +235,6 @@ func testForkIDSplit(t *testing.T, protocol uint) { } // Tests that received transactions are added to the local pool. -func TestRecvTransactions65(t *testing.T) { testRecvTransactions(t, eth.ETH65) } func TestRecvTransactions66(t *testing.T) { testRecvTransactions(t, eth.ETH66) } func testRecvTransactions(t *testing.T, protocol uint) { @@ -294,7 +292,6 @@ func testRecvTransactions(t *testing.T, protocol uint) { } // This test checks that pending transactions are sent. -func TestSendTransactions65(t *testing.T) { testSendTransactions(t, eth.ETH65) } func TestSendTransactions66(t *testing.T) { testSendTransactions(t, eth.ETH66) } func testSendTransactions(t *testing.T, protocol uint) { @@ -306,7 +303,7 @@ func testSendTransactions(t *testing.T, protocol uint) { insert := make([]*types.Transaction, 100) for nonce := range insert { - tx := types.NewTransaction(uint64(nonce), common.Address{}, big.NewInt(0), 100000, big.NewInt(0), make([]byte, txsyncPackSize/10)) + tx := types.NewTransaction(uint64(nonce), common.Address{}, big.NewInt(0), 100000, big.NewInt(0), make([]byte, 10240)) tx, _ = types.SignTx(tx, types.HomesteadSigner{}, testKey) insert[nonce] = tx @@ -380,7 +377,6 @@ func testSendTransactions(t *testing.T, protocol uint) { // Tests that transactions get propagated to all attached peers, either via direct // broadcasts or via announcements/retrievals. -func TestTransactionPropagation65(t *testing.T) { testTransactionPropagation(t, eth.ETH65) } func TestTransactionPropagation66(t *testing.T) { testTransactionPropagation(t, eth.ETH66) } func testTransactionPropagation(t *testing.T, protocol uint) { @@ -521,8 +517,8 @@ func testCheckpointChallenge(t *testing.T, syncmode downloader.SyncMode, checkpo defer p2pLocal.Close() defer p2pRemote.Close() - local := eth.NewPeer(eth.ETH65, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pLocal), p2pLocal, handler.txpool) - remote := eth.NewPeer(eth.ETH65, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pRemote), p2pRemote, handler.txpool) + local := eth.NewPeer(eth.ETH66, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pLocal), p2pLocal, handler.txpool) + remote := eth.NewPeer(eth.ETH66, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pRemote), p2pRemote, handler.txpool) defer local.Close() defer remote.Close() @@ -543,30 +539,39 @@ func testCheckpointChallenge(t *testing.T, syncmode downloader.SyncMode, checkpo if err := remote.Handshake(1, td, head.Hash(), genesis.Hash(), forkid.NewIDWithChain(handler.chain), forkid.NewFilter(handler.chain)); err != nil { t.Fatalf("failed to run protocol handshake") } - // Connect a new peer and check that we receive the checkpoint challenge. if checkpoint { - if err := remote.ExpectRequestHeadersByNumber(response.Number.Uint64(), 1, 0, false); err != nil { - t.Fatalf("challenge mismatch: %v", err) + msg, err := p2pRemote.ReadMsg() + if err != nil { + t.Fatalf("failed to read checkpoint challenge: %v", err) + } + request := new(eth.GetBlockHeadersPacket66) + if err := msg.Decode(request); err != nil { + t.Fatalf("failed to decode checkpoint challenge: %v", err) + } + query := request.GetBlockHeadersPacket + if query.Origin.Number != response.Number.Uint64() || query.Amount != 1 || query.Skip != 0 || query.Reverse { + t.Fatalf("challenge mismatch: have [%d, %d, %d, %v] want [%d, %d, %d, %v]", + query.Origin.Number, query.Amount, query.Skip, query.Reverse, + response.Number.Uint64(), 1, 0, false) } // Create a block to reply to the challenge if no timeout is simulated. if !timeout { if empty { - if err := remote.SendBlockHeaders([]*types.Header{}); err != nil { + if err := remote.ReplyBlockHeaders(request.RequestId, []*types.Header{}); err != nil { t.Fatalf("failed to answer challenge: %v", err) } } else if match { - if err := remote.SendBlockHeaders([]*types.Header{response}); err != nil { + if err := remote.ReplyBlockHeaders(request.RequestId, []*types.Header{response}); err != nil { t.Fatalf("failed to answer challenge: %v", err) } } else { - if err := remote.SendBlockHeaders([]*types.Header{{Number: response.Number}}); err != nil { + if err := remote.ReplyBlockHeaders(request.RequestId, []*types.Header{{Number: response.Number}}); err != nil { t.Fatalf("failed to answer challenge: %v", err) } } } } - // Wait until the test timeout passes to ensure proper cleanup time.Sleep(syncChallengeTimeout + 300*time.Millisecond) @@ -619,8 +624,8 @@ func testBroadcastBlock(t *testing.T, peers, bcasts int) { defer sourcePipe.Close() defer sinkPipe.Close() - sourcePeer := eth.NewPeer(eth.ETH65, p2p.NewPeerPipe(enode.ID{byte(i)}, "", nil, sourcePipe), sourcePipe, nil) - sinkPeer := eth.NewPeer(eth.ETH65, p2p.NewPeerPipe(enode.ID{0}, "", nil, sinkPipe), sinkPipe, nil) + sourcePeer := eth.NewPeer(eth.ETH66, p2p.NewPeerPipe(enode.ID{byte(i)}, "", nil, sourcePipe), sourcePipe, nil) + sinkPeer := eth.NewPeer(eth.ETH66, p2p.NewPeerPipe(enode.ID{0}, "", nil, sinkPipe), sinkPipe, nil) defer sourcePeer.Close() defer sinkPeer.Close() @@ -671,7 +676,6 @@ func testBroadcastBlock(t *testing.T, peers, bcasts int) { // Tests that a propagated malformed block (uncles or transactions don't match // with the hashes in the header) gets discarded and not broadcast forward. -func TestBroadcastMalformedBlock65(t *testing.T) { testBroadcastMalformedBlock(t, eth.ETH65) } func TestBroadcastMalformedBlock66(t *testing.T) { testBroadcastMalformedBlock(t, eth.ETH66) } func testBroadcastMalformedBlock(t *testing.T, protocol uint) { diff --git a/eth/protocols/eth/handler.go b/eth/protocols/eth/handler.go index 6bbaa2f55..828930014 100644 --- a/eth/protocols/eth/handler.go +++ b/eth/protocols/eth/handler.go @@ -171,39 +171,21 @@ type Decoder interface { Time() time.Time } -var eth65 = map[uint64]msgHandler{ - GetBlockHeadersMsg: handleGetBlockHeaders, - BlockHeadersMsg: handleBlockHeaders, - GetBlockBodiesMsg: handleGetBlockBodies, - BlockBodiesMsg: handleBlockBodies, - GetNodeDataMsg: handleGetNodeData, - NodeDataMsg: handleNodeData, - GetReceiptsMsg: handleGetReceipts, - ReceiptsMsg: handleReceipts, - NewBlockHashesMsg: handleNewBlockhashes, - NewBlockMsg: handleNewBlock, - TransactionsMsg: handleTransactions, - NewPooledTransactionHashesMsg: handleNewPooledTransactionHashes, - GetPooledTransactionsMsg: handleGetPooledTransactions, - PooledTransactionsMsg: handlePooledTransactions, -} - var eth66 = map[uint64]msgHandler{ NewBlockHashesMsg: handleNewBlockhashes, NewBlockMsg: handleNewBlock, TransactionsMsg: handleTransactions, NewPooledTransactionHashesMsg: handleNewPooledTransactionHashes, - // eth66 messages with request-id - GetBlockHeadersMsg: handleGetBlockHeaders66, - BlockHeadersMsg: handleBlockHeaders66, - GetBlockBodiesMsg: handleGetBlockBodies66, - BlockBodiesMsg: handleBlockBodies66, - GetNodeDataMsg: handleGetNodeData66, - NodeDataMsg: handleNodeData66, - GetReceiptsMsg: handleGetReceipts66, - ReceiptsMsg: handleReceipts66, - GetPooledTransactionsMsg: handleGetPooledTransactions66, - PooledTransactionsMsg: handlePooledTransactions66, + GetBlockHeadersMsg: handleGetBlockHeaders66, + BlockHeadersMsg: handleBlockHeaders66, + GetBlockBodiesMsg: handleGetBlockBodies66, + BlockBodiesMsg: handleBlockBodies66, + GetNodeDataMsg: handleGetNodeData66, + NodeDataMsg: handleNodeData66, + GetReceiptsMsg: handleGetReceipts66, + ReceiptsMsg: handleReceipts66, + GetPooledTransactionsMsg: handleGetPooledTransactions66, + PooledTransactionsMsg: handlePooledTransactions66, } // handleMessage is invoked whenever an inbound message is received from a remote @@ -219,10 +201,11 @@ func handleMessage(backend Backend, peer *Peer) error { } defer msg.Discard() - var handlers = eth65 - if peer.Version() >= ETH66 { - handlers = eth66 - } + var handlers = eth66 + //if peer.Version() >= ETH67 { // Left in as a sample when new protocol is added + // handlers = eth67 + //} + // Track the amount of time it takes to serve the request and run the handler if metrics.Enabled { h := fmt.Sprintf("%s/%s/%d/%#02x", p2p.HandleHistName, ProtocolName, peer.Version(), msg.Code) diff --git a/eth/protocols/eth/handler_test.go b/eth/protocols/eth/handler_test.go index 473be3f9b..809f17e36 100644 --- a/eth/protocols/eth/handler_test.go +++ b/eth/protocols/eth/handler_test.go @@ -110,7 +110,6 @@ func (b *testBackend) Handle(*Peer, Packet) error { } // Tests that block headers can be retrieved from a remote chain based on user queries. -func TestGetBlockHeaders65(t *testing.T) { testGetBlockHeaders(t, ETH65) } func TestGetBlockHeaders66(t *testing.T) { testGetBlockHeaders(t, ETH66) } func testGetBlockHeaders(t *testing.T, protocol uint) { @@ -254,44 +253,30 @@ func testGetBlockHeaders(t *testing.T, protocol uint) { headers = append(headers, backend.chain.GetBlockByHash(hash).Header()) } // Send the hash request and verify the response - if protocol <= ETH65 { - p2p.Send(peer.app, GetBlockHeadersMsg, tt.query) - if err := p2p.ExpectMsg(peer.app, BlockHeadersMsg, headers); err != nil { - t.Errorf("test %d: headers mismatch: %v", i, err) - } - } else { - p2p.Send(peer.app, GetBlockHeadersMsg, GetBlockHeadersPacket66{ - RequestId: 123, - GetBlockHeadersPacket: tt.query, - }) - if err := p2p.ExpectMsg(peer.app, BlockHeadersMsg, BlockHeadersPacket66{ - RequestId: 123, - BlockHeadersPacket: headers, - }); err != nil { - t.Errorf("test %d: headers mismatch: %v", i, err) - } + p2p.Send(peer.app, GetBlockHeadersMsg, GetBlockHeadersPacket66{ + RequestId: 123, + GetBlockHeadersPacket: tt.query, + }) + if err := p2p.ExpectMsg(peer.app, BlockHeadersMsg, BlockHeadersPacket66{ + RequestId: 123, + BlockHeadersPacket: headers, + }); err != nil { + t.Errorf("test %d: headers mismatch: %v", i, err) } // If the test used number origins, repeat with hashes as the too if tt.query.Origin.Hash == (common.Hash{}) { if origin := backend.chain.GetBlockByNumber(tt.query.Origin.Number); origin != nil { tt.query.Origin.Hash, tt.query.Origin.Number = origin.Hash(), 0 - if protocol <= ETH65 { - p2p.Send(peer.app, GetBlockHeadersMsg, tt.query) - if err := p2p.ExpectMsg(peer.app, BlockHeadersMsg, headers); err != nil { - t.Errorf("test %d: headers mismatch: %v", i, err) - } - } else { - p2p.Send(peer.app, GetBlockHeadersMsg, GetBlockHeadersPacket66{ - RequestId: 456, - GetBlockHeadersPacket: tt.query, - }) - if err := p2p.ExpectMsg(peer.app, BlockHeadersMsg, BlockHeadersPacket66{ - RequestId: 456, - BlockHeadersPacket: headers, - }); err != nil { - t.Errorf("test %d: headers mismatch: %v", i, err) - } + p2p.Send(peer.app, GetBlockHeadersMsg, GetBlockHeadersPacket66{ + RequestId: 456, + GetBlockHeadersPacket: tt.query, + }) + if err := p2p.ExpectMsg(peer.app, BlockHeadersMsg, BlockHeadersPacket66{ + RequestId: 456, + BlockHeadersPacket: headers, + }); err != nil { + t.Errorf("test %d: headers mismatch: %v", i, err) } } } @@ -299,7 +284,6 @@ func testGetBlockHeaders(t *testing.T, protocol uint) { } // Tests that block contents can be retrieved from a remote chain based on their hashes. -func TestGetBlockBodies65(t *testing.T) { testGetBlockBodies(t, ETH65) } func TestGetBlockBodies66(t *testing.T) { testGetBlockBodies(t, ETH66) } func testGetBlockBodies(t *testing.T, protocol uint) { @@ -369,28 +353,20 @@ func testGetBlockBodies(t *testing.T, protocol uint) { } } // Send the hash request and verify the response - if protocol <= ETH65 { - p2p.Send(peer.app, GetBlockBodiesMsg, hashes) - if err := p2p.ExpectMsg(peer.app, BlockBodiesMsg, bodies); err != nil { - t.Errorf("test %d: bodies mismatch: %v", i, err) - } - } else { - p2p.Send(peer.app, GetBlockBodiesMsg, GetBlockBodiesPacket66{ - RequestId: 123, - GetBlockBodiesPacket: hashes, - }) - if err := p2p.ExpectMsg(peer.app, BlockBodiesMsg, BlockBodiesPacket66{ - RequestId: 123, - BlockBodiesPacket: bodies, - }); err != nil { - t.Errorf("test %d: bodies mismatch: %v", i, err) - } + p2p.Send(peer.app, GetBlockBodiesMsg, GetBlockBodiesPacket66{ + RequestId: 123, + GetBlockBodiesPacket: hashes, + }) + if err := p2p.ExpectMsg(peer.app, BlockBodiesMsg, BlockBodiesPacket66{ + RequestId: 123, + BlockBodiesPacket: bodies, + }); err != nil { + t.Errorf("test %d: bodies mismatch: %v", i, err) } } } // Tests that the state trie nodes can be retrieved based on hashes. -func TestGetNodeData65(t *testing.T) { testGetNodeData(t, ETH65) } func TestGetNodeData66(t *testing.T) { testGetNodeData(t, ETH66) } func testGetNodeData(t *testing.T, protocol uint) { @@ -449,14 +425,10 @@ func testGetNodeData(t *testing.T, protocol uint) { } it.Release() - if protocol <= ETH65 { - p2p.Send(peer.app, GetNodeDataMsg, hashes) - } else { - p2p.Send(peer.app, GetNodeDataMsg, GetNodeDataPacket66{ - RequestId: 123, - GetNodeDataPacket: hashes, - }) - } + p2p.Send(peer.app, GetNodeDataMsg, GetNodeDataPacket66{ + RequestId: 123, + GetNodeDataPacket: hashes, + }) msg, err := peer.app.ReadMsg() if err != nil { t.Fatalf("failed to read node data response: %v", err) @@ -464,18 +436,14 @@ func testGetNodeData(t *testing.T, protocol uint) { if msg.Code != NodeDataMsg { t.Fatalf("response packet code mismatch: have %x, want %x", msg.Code, NodeDataMsg) } - var data [][]byte - if protocol <= ETH65 { - if err := msg.Decode(&data); err != nil { - t.Fatalf("failed to decode response node data: %v", err) - } - } else { - var res NodeDataPacket66 - if err := msg.Decode(&res); err != nil { - t.Fatalf("failed to decode response node data: %v", err) - } - data = res.NodeDataPacket + var ( + data [][]byte + res NodeDataPacket66 + ) + if err := msg.Decode(&res); err != nil { + t.Fatalf("failed to decode response node data: %v", err) } + data = res.NodeDataPacket // Verify that all hashes correspond to the requested data, and reconstruct a state tree for i, want := range hashes { if hash := crypto.Keccak256Hash(data[i]); hash != want { @@ -506,7 +474,6 @@ func testGetNodeData(t *testing.T, protocol uint) { } // Tests that the transaction receipts can be retrieved based on hashes. -func TestGetBlockReceipts65(t *testing.T) { testGetBlockReceipts(t, ETH65) } func TestGetBlockReceipts66(t *testing.T) { testGetBlockReceipts(t, ETH66) } func testGetBlockReceipts(t *testing.T, protocol uint) { @@ -566,21 +533,14 @@ func testGetBlockReceipts(t *testing.T, protocol uint) { receipts = append(receipts, backend.chain.GetReceiptsByHash(block.Hash())) } // Send the hash request and verify the response - if protocol <= ETH65 { - p2p.Send(peer.app, GetReceiptsMsg, hashes) - if err := p2p.ExpectMsg(peer.app, ReceiptsMsg, receipts); err != nil { - t.Errorf("receipts mismatch: %v", err) - } - } else { - p2p.Send(peer.app, GetReceiptsMsg, GetReceiptsPacket66{ - RequestId: 123, - GetReceiptsPacket: hashes, - }) - if err := p2p.ExpectMsg(peer.app, ReceiptsMsg, ReceiptsPacket66{ - RequestId: 123, - ReceiptsPacket: receipts, - }); err != nil { - t.Errorf("receipts mismatch: %v", err) - } + p2p.Send(peer.app, GetReceiptsMsg, GetReceiptsPacket66{ + RequestId: 123, + GetReceiptsPacket: hashes, + }) + if err := p2p.ExpectMsg(peer.app, ReceiptsMsg, ReceiptsPacket66{ + RequestId: 123, + ReceiptsPacket: receipts, + }); err != nil { + t.Errorf("receipts mismatch: %v", err) } } diff --git a/eth/protocols/eth/handlers.go b/eth/protocols/eth/handlers.go index d7d993a23..e54838cbc 100644 --- a/eth/protocols/eth/handlers.go +++ b/eth/protocols/eth/handlers.go @@ -27,17 +27,6 @@ import ( "github.com/ethereum/go-ethereum/trie" ) -// handleGetBlockHeaders handles Block header query, collect the requested headers and reply -func handleGetBlockHeaders(backend Backend, msg Decoder, peer *Peer) error { - // Decode the complex header query - var query GetBlockHeadersPacket - if err := msg.Decode(&query); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - response := answerGetBlockHeadersQuery(backend, &query, peer) - return peer.SendBlockHeaders(response) -} - // handleGetBlockHeaders66 is the eth/66 version of handleGetBlockHeaders func handleGetBlockHeaders66(backend Backend, msg Decoder, peer *Peer) error { // Decode the complex header query @@ -135,16 +124,6 @@ func answerGetBlockHeadersQuery(backend Backend, query *GetBlockHeadersPacket, p return headers } -func handleGetBlockBodies(backend Backend, msg Decoder, peer *Peer) error { - // Decode the block body retrieval message - var query GetBlockBodiesPacket - if err := msg.Decode(&query); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - response := answerGetBlockBodiesQuery(backend, query, peer) - return peer.SendBlockBodiesRLP(response) -} - func handleGetBlockBodies66(backend Backend, msg Decoder, peer *Peer) error { // Decode the block body retrieval message var query GetBlockBodiesPacket66 @@ -174,16 +153,6 @@ func answerGetBlockBodiesQuery(backend Backend, query GetBlockBodiesPacket, peer return bodies } -func handleGetNodeData(backend Backend, msg Decoder, peer *Peer) error { - // Decode the trie node data retrieval message - var query GetNodeDataPacket - if err := msg.Decode(&query); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - response := answerGetNodeDataQuery(backend, query, peer) - return peer.SendNodeData(response) -} - func handleGetNodeData66(backend Backend, msg Decoder, peer *Peer) error { // Decode the trie node data retrieval message var query GetNodeDataPacket66 @@ -223,16 +192,6 @@ func answerGetNodeDataQuery(backend Backend, query GetNodeDataPacket, peer *Peer return nodes } -func handleGetReceipts(backend Backend, msg Decoder, peer *Peer) error { - // Decode the block receipts retrieval message - var query GetReceiptsPacket - if err := msg.Decode(&query); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - response := answerGetReceiptsQuery(backend, query, peer) - return peer.SendReceiptsRLP(response) -} - func handleGetReceipts66(backend Backend, msg Decoder, peer *Peer) error { // Decode the block receipts retrieval message var query GetReceiptsPacket66 @@ -312,15 +271,6 @@ func handleNewBlock(backend Backend, msg Decoder, peer *Peer) error { return backend.Handle(peer, ann) } -func handleBlockHeaders(backend Backend, msg Decoder, peer *Peer) error { - // A batch of headers arrived to one of our previous requests - res := new(BlockHeadersPacket) - if err := msg.Decode(res); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - return backend.Handle(peer, res) -} - func handleBlockHeaders66(backend Backend, msg Decoder, peer *Peer) error { // A batch of headers arrived to one of our previous requests res := new(BlockHeadersPacket66) @@ -332,15 +282,6 @@ func handleBlockHeaders66(backend Backend, msg Decoder, peer *Peer) error { return backend.Handle(peer, &res.BlockHeadersPacket) } -func handleBlockBodies(backend Backend, msg Decoder, peer *Peer) error { - // A batch of block bodies arrived to one of our previous requests - res := new(BlockBodiesPacket) - if err := msg.Decode(res); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - return backend.Handle(peer, res) -} - func handleBlockBodies66(backend Backend, msg Decoder, peer *Peer) error { // A batch of block bodies arrived to one of our previous requests res := new(BlockBodiesPacket66) @@ -352,15 +293,6 @@ func handleBlockBodies66(backend Backend, msg Decoder, peer *Peer) error { return backend.Handle(peer, &res.BlockBodiesPacket) } -func handleNodeData(backend Backend, msg Decoder, peer *Peer) error { - // A batch of node state data arrived to one of our previous requests - res := new(NodeDataPacket) - if err := msg.Decode(res); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - return backend.Handle(peer, res) -} - func handleNodeData66(backend Backend, msg Decoder, peer *Peer) error { // A batch of node state data arrived to one of our previous requests res := new(NodeDataPacket66) @@ -372,15 +304,6 @@ func handleNodeData66(backend Backend, msg Decoder, peer *Peer) error { return backend.Handle(peer, &res.NodeDataPacket) } -func handleReceipts(backend Backend, msg Decoder, peer *Peer) error { - // A batch of receipts arrived to one of our previous requests - res := new(ReceiptsPacket) - if err := msg.Decode(res); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - return backend.Handle(peer, res) -} - func handleReceipts66(backend Backend, msg Decoder, peer *Peer) error { // A batch of receipts arrived to one of our previous requests res := new(ReceiptsPacket66) @@ -409,16 +332,6 @@ func handleNewPooledTransactionHashes(backend Backend, msg Decoder, peer *Peer) return backend.Handle(peer, ann) } -func handleGetPooledTransactions(backend Backend, msg Decoder, peer *Peer) error { - // Decode the pooled transactions retrieval message - var query GetPooledTransactionsPacket - if err := msg.Decode(&query); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - hashes, txs := answerGetPooledTransactions(backend, query, peer) - return peer.SendPooledTransactionsRLP(hashes, txs) -} - func handleGetPooledTransactions66(backend Backend, msg Decoder, peer *Peer) error { // Decode the pooled transactions retrieval message var query GetPooledTransactionsPacket66 @@ -477,26 +390,6 @@ func handleTransactions(backend Backend, msg Decoder, peer *Peer) error { return backend.Handle(peer, &txs) } -func handlePooledTransactions(backend Backend, msg Decoder, peer *Peer) error { - // Transactions arrived, make sure we have a valid and fresh chain to handle them - if !backend.AcceptTxs() { - return nil - } - // Transactions can be processed, parse all of them and deliver to the pool - var txs PooledTransactionsPacket - if err := msg.Decode(&txs); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - for i, tx := range txs { - // Validate and mark the remote transaction - if tx == nil { - return fmt.Errorf("%w: transaction %d is nil", errDecode, i) - } - peer.markTransaction(tx.Hash()) - } - return backend.Handle(peer, &txs) -} - func handlePooledTransactions66(backend Backend, msg Decoder, peer *Peer) error { // Transactions arrived, make sure we have a valid and fresh chain to handle them if !backend.AcceptTxs() { diff --git a/eth/protocols/eth/handshake_test.go b/eth/protocols/eth/handshake_test.go index 3bebda2dc..05d473e05 100644 --- a/eth/protocols/eth/handshake_test.go +++ b/eth/protocols/eth/handshake_test.go @@ -27,7 +27,6 @@ import ( ) // Tests that handshake failures are detected and reported correctly. -func TestHandshake65(t *testing.T) { testHandshake(t, ETH65) } func TestHandshake66(t *testing.T) { testHandshake(t, ETH66) } func testHandshake(t *testing.T, protocol uint) { diff --git a/eth/protocols/eth/peer.go b/eth/protocols/eth/peer.go index e619c183b..98273ccfc 100644 --- a/eth/protocols/eth/peer.go +++ b/eth/protocols/eth/peer.go @@ -108,9 +108,8 @@ func NewPeer(version uint, p *p2p.Peer, rw p2p.MsgReadWriter, txpool TxPool) *Pe // Start up all the broadcasters go peer.broadcastBlocks() go peer.broadcastTransactions() - if version >= ETH65 { - go peer.announceTransactions() - } + go peer.announceTransactions() + return peer } @@ -252,22 +251,6 @@ func (p *Peer) AsyncSendPooledTransactionHashes(hashes []common.Hash) { } } -// SendPooledTransactionsRLP sends requested transactions to the peer and adds the -// hashes in its transaction hash set for future reference. -// -// Note, the method assumes the hashes are correct and correspond to the list of -// transactions being sent. -func (p *Peer) SendPooledTransactionsRLP(hashes []common.Hash, txs []rlp.RawValue) error { - // Mark all the transactions as known, but ensure we don't overflow our limits - for p.knownTxs.Cardinality() > max(0, maxKnownTxs-len(hashes)) { - p.knownTxs.Pop() - } - for _, hash := range hashes { - p.knownTxs.Add(hash) - } - return p2p.Send(p.rw, PooledTransactionsMsg, txs) // Not packed into PooledTransactionsPacket to avoid RLP decoding -} - // ReplyPooledTransactionsRLP is the eth/66 version of SendPooledTransactionsRLP. func (p *Peer) ReplyPooledTransactionsRLP(id uint64, hashes []common.Hash, txs []rlp.RawValue) error { // Mark all the transactions as known, but ensure we don't overflow our limits @@ -346,11 +329,6 @@ func (p *Peer) AsyncSendNewBlock(block *types.Block, td *big.Int) { } } -// SendBlockHeaders sends a batch of block headers to the remote peer. -func (p *Peer) SendBlockHeaders(headers []*types.Header) error { - return p2p.Send(p.rw, BlockHeadersMsg, BlockHeadersPacket(headers)) -} - // ReplyBlockHeaders is the eth/66 version of SendBlockHeaders. func (p *Peer) ReplyBlockHeaders(id uint64, headers []*types.Header) error { return p2p.Send(p.rw, BlockHeadersMsg, BlockHeadersPacket66{ @@ -359,12 +337,6 @@ func (p *Peer) ReplyBlockHeaders(id uint64, headers []*types.Header) error { }) } -// SendBlockBodiesRLP sends a batch of block contents to the remote peer from -// an already RLP encoded format. -func (p *Peer) SendBlockBodiesRLP(bodies []rlp.RawValue) error { - return p2p.Send(p.rw, BlockBodiesMsg, bodies) // Not packed into BlockBodiesPacket to avoid RLP decoding -} - // ReplyBlockBodiesRLP is the eth/66 version of SendBlockBodiesRLP. func (p *Peer) ReplyBlockBodiesRLP(id uint64, bodies []rlp.RawValue) error { // Not packed into BlockBodiesPacket to avoid RLP decoding @@ -374,12 +346,6 @@ func (p *Peer) ReplyBlockBodiesRLP(id uint64, bodies []rlp.RawValue) error { }) } -// SendNodeDataRLP sends a batch of arbitrary internal data, corresponding to the -// hashes requested. -func (p *Peer) SendNodeData(data [][]byte) error { - return p2p.Send(p.rw, NodeDataMsg, NodeDataPacket(data)) -} - // ReplyNodeData is the eth/66 response to GetNodeData. func (p *Peer) ReplyNodeData(id uint64, data [][]byte) error { return p2p.Send(p.rw, NodeDataMsg, NodeDataPacket66{ @@ -388,12 +354,6 @@ func (p *Peer) ReplyNodeData(id uint64, data [][]byte) error { }) } -// SendReceiptsRLP sends a batch of transaction receipts, corresponding to the -// ones requested from an already RLP encoded format. -func (p *Peer) SendReceiptsRLP(receipts []rlp.RawValue) error { - return p2p.Send(p.rw, ReceiptsMsg, receipts) // Not packed into ReceiptsPacket to avoid RLP decoding -} - // ReplyReceiptsRLP is the eth/66 response to GetReceipts. func (p *Peer) ReplyReceiptsRLP(id uint64, receipts []rlp.RawValue) error { return p2p.Send(p.rw, ReceiptsMsg, ReceiptsRLPPacket66{ @@ -406,138 +366,102 @@ func (p *Peer) ReplyReceiptsRLP(id uint64, receipts []rlp.RawValue) error { // single header. It is used solely by the fetcher. func (p *Peer) RequestOneHeader(hash common.Hash) error { p.Log().Debug("Fetching single header", "hash", hash) - query := GetBlockHeadersPacket{ - Origin: HashOrNumber{Hash: hash}, - Amount: uint64(1), - Skip: uint64(0), - Reverse: false, - } - if p.Version() >= ETH66 { - id := rand.Uint64() + id := rand.Uint64() - requestTracker.Track(p.id, p.version, GetBlockHeadersMsg, BlockHeadersMsg, id) - return p2p.Send(p.rw, GetBlockHeadersMsg, &GetBlockHeadersPacket66{ - RequestId: id, - GetBlockHeadersPacket: &query, - }) - } - return p2p.Send(p.rw, GetBlockHeadersMsg, &query) + requestTracker.Track(p.id, p.version, GetBlockHeadersMsg, BlockHeadersMsg, id) + return p2p.Send(p.rw, GetBlockHeadersMsg, &GetBlockHeadersPacket66{ + RequestId: id, + GetBlockHeadersPacket: &GetBlockHeadersPacket{ + Origin: HashOrNumber{Hash: hash}, + Amount: uint64(1), + Skip: uint64(0), + Reverse: false, + }, + }) } // RequestHeadersByHash fetches a batch of blocks' headers corresponding to the // specified header query, based on the hash of an origin block. func (p *Peer) RequestHeadersByHash(origin common.Hash, amount int, skip int, reverse bool) error { p.Log().Debug("Fetching batch of headers", "count", amount, "fromhash", origin, "skip", skip, "reverse", reverse) - query := GetBlockHeadersPacket{ - Origin: HashOrNumber{Hash: origin}, - Amount: uint64(amount), - Skip: uint64(skip), - Reverse: reverse, - } - if p.Version() >= ETH66 { - id := rand.Uint64() + id := rand.Uint64() - requestTracker.Track(p.id, p.version, GetBlockHeadersMsg, BlockHeadersMsg, id) - return p2p.Send(p.rw, GetBlockHeadersMsg, &GetBlockHeadersPacket66{ - RequestId: id, - GetBlockHeadersPacket: &query, - }) - } - return p2p.Send(p.rw, GetBlockHeadersMsg, &query) + requestTracker.Track(p.id, p.version, GetBlockHeadersMsg, BlockHeadersMsg, id) + return p2p.Send(p.rw, GetBlockHeadersMsg, &GetBlockHeadersPacket66{ + RequestId: id, + GetBlockHeadersPacket: &GetBlockHeadersPacket{ + Origin: HashOrNumber{Hash: origin}, + Amount: uint64(amount), + Skip: uint64(skip), + Reverse: reverse, + }, + }) } // RequestHeadersByNumber fetches a batch of blocks' headers corresponding to the // specified header query, based on the number of an origin block. func (p *Peer) RequestHeadersByNumber(origin uint64, amount int, skip int, reverse bool) error { p.Log().Debug("Fetching batch of headers", "count", amount, "fromnum", origin, "skip", skip, "reverse", reverse) - query := GetBlockHeadersPacket{ - Origin: HashOrNumber{Number: origin}, - Amount: uint64(amount), - Skip: uint64(skip), - Reverse: reverse, - } - if p.Version() >= ETH66 { - id := rand.Uint64() + id := rand.Uint64() - requestTracker.Track(p.id, p.version, GetBlockHeadersMsg, BlockHeadersMsg, id) - return p2p.Send(p.rw, GetBlockHeadersMsg, &GetBlockHeadersPacket66{ - RequestId: id, - GetBlockHeadersPacket: &query, - }) - } - return p2p.Send(p.rw, GetBlockHeadersMsg, &query) -} - -// ExpectRequestHeadersByNumber is a testing method to mirror the recipient side -// of the RequestHeadersByNumber operation. -func (p *Peer) ExpectRequestHeadersByNumber(origin uint64, amount int, skip int, reverse bool) error { - req := &GetBlockHeadersPacket{ - Origin: HashOrNumber{Number: origin}, - Amount: uint64(amount), - Skip: uint64(skip), - Reverse: reverse, - } - return p2p.ExpectMsg(p.rw, GetBlockHeadersMsg, req) + requestTracker.Track(p.id, p.version, GetBlockHeadersMsg, BlockHeadersMsg, id) + return p2p.Send(p.rw, GetBlockHeadersMsg, &GetBlockHeadersPacket66{ + RequestId: id, + GetBlockHeadersPacket: &GetBlockHeadersPacket{ + Origin: HashOrNumber{Number: origin}, + Amount: uint64(amount), + Skip: uint64(skip), + Reverse: reverse, + }, + }) } // RequestBodies fetches a batch of blocks' bodies corresponding to the hashes // specified. func (p *Peer) RequestBodies(hashes []common.Hash) error { p.Log().Debug("Fetching batch of block bodies", "count", len(hashes)) - if p.Version() >= ETH66 { - id := rand.Uint64() + id := rand.Uint64() - requestTracker.Track(p.id, p.version, GetBlockBodiesMsg, BlockBodiesMsg, id) - return p2p.Send(p.rw, GetBlockBodiesMsg, &GetBlockBodiesPacket66{ - RequestId: id, - GetBlockBodiesPacket: hashes, - }) - } - return p2p.Send(p.rw, GetBlockBodiesMsg, GetBlockBodiesPacket(hashes)) + requestTracker.Track(p.id, p.version, GetBlockBodiesMsg, BlockBodiesMsg, id) + return p2p.Send(p.rw, GetBlockBodiesMsg, &GetBlockBodiesPacket66{ + RequestId: id, + GetBlockBodiesPacket: hashes, + }) } // RequestNodeData fetches a batch of arbitrary data from a node's known state // data, corresponding to the specified hashes. func (p *Peer) RequestNodeData(hashes []common.Hash) error { p.Log().Debug("Fetching batch of state data", "count", len(hashes)) - if p.Version() >= ETH66 { - id := rand.Uint64() + id := rand.Uint64() - requestTracker.Track(p.id, p.version, GetNodeDataMsg, NodeDataMsg, id) - return p2p.Send(p.rw, GetNodeDataMsg, &GetNodeDataPacket66{ - RequestId: id, - GetNodeDataPacket: hashes, - }) - } - return p2p.Send(p.rw, GetNodeDataMsg, GetNodeDataPacket(hashes)) + requestTracker.Track(p.id, p.version, GetNodeDataMsg, NodeDataMsg, id) + return p2p.Send(p.rw, GetNodeDataMsg, &GetNodeDataPacket66{ + RequestId: id, + GetNodeDataPacket: hashes, + }) } // RequestReceipts fetches a batch of transaction receipts from a remote node. func (p *Peer) RequestReceipts(hashes []common.Hash) error { p.Log().Debug("Fetching batch of receipts", "count", len(hashes)) - if p.Version() >= ETH66 { - id := rand.Uint64() + id := rand.Uint64() - requestTracker.Track(p.id, p.version, GetReceiptsMsg, ReceiptsMsg, id) - return p2p.Send(p.rw, GetReceiptsMsg, &GetReceiptsPacket66{ - RequestId: id, - GetReceiptsPacket: hashes, - }) - } - return p2p.Send(p.rw, GetReceiptsMsg, GetReceiptsPacket(hashes)) + requestTracker.Track(p.id, p.version, GetReceiptsMsg, ReceiptsMsg, id) + return p2p.Send(p.rw, GetReceiptsMsg, &GetReceiptsPacket66{ + RequestId: id, + GetReceiptsPacket: hashes, + }) } // RequestTxs fetches a batch of transactions from a remote node. func (p *Peer) RequestTxs(hashes []common.Hash) error { p.Log().Debug("Fetching batch of transactions", "count", len(hashes)) - if p.Version() >= ETH66 { - id := rand.Uint64() + id := rand.Uint64() - requestTracker.Track(p.id, p.version, GetPooledTransactionsMsg, PooledTransactionsMsg, id) - return p2p.Send(p.rw, GetPooledTransactionsMsg, &GetPooledTransactionsPacket66{ - RequestId: id, - GetPooledTransactionsPacket: hashes, - }) - } - return p2p.Send(p.rw, GetPooledTransactionsMsg, GetPooledTransactionsPacket(hashes)) + requestTracker.Track(p.id, p.version, GetPooledTransactionsMsg, PooledTransactionsMsg, id) + return p2p.Send(p.rw, GetPooledTransactionsMsg, &GetPooledTransactionsPacket66{ + RequestId: id, + GetPooledTransactionsPacket: hashes, + }) } diff --git a/eth/protocols/eth/protocol.go b/eth/protocols/eth/protocol.go index de1b0ed1e..3c3da30fa 100644 --- a/eth/protocols/eth/protocol.go +++ b/eth/protocols/eth/protocol.go @@ -30,7 +30,6 @@ import ( // Constants to match up protocol versions and messages const ( - ETH65 = 65 ETH66 = 66 ) @@ -40,31 +39,28 @@ const ProtocolName = "eth" // ProtocolVersions are the supported versions of the `eth` protocol (first // is primary). -var ProtocolVersions = []uint{ETH66, ETH65} +var ProtocolVersions = []uint{ETH66} // protocolLengths are the number of implemented message corresponding to // different protocol versions. -var protocolLengths = map[uint]uint64{ETH66: 17, ETH65: 17} +var protocolLengths = map[uint]uint64{ETH66: 17} // maxMessageSize is the maximum cap on the size of a protocol message. const maxMessageSize = 10 * 1024 * 1024 const ( - // Protocol messages in eth/64 - StatusMsg = 0x00 - NewBlockHashesMsg = 0x01 - TransactionsMsg = 0x02 - GetBlockHeadersMsg = 0x03 - BlockHeadersMsg = 0x04 - GetBlockBodiesMsg = 0x05 - BlockBodiesMsg = 0x06 - NewBlockMsg = 0x07 - GetNodeDataMsg = 0x0d - NodeDataMsg = 0x0e - GetReceiptsMsg = 0x0f - ReceiptsMsg = 0x10 - - // Protocol messages overloaded in eth/65 + StatusMsg = 0x00 + NewBlockHashesMsg = 0x01 + TransactionsMsg = 0x02 + GetBlockHeadersMsg = 0x03 + BlockHeadersMsg = 0x04 + GetBlockBodiesMsg = 0x05 + BlockBodiesMsg = 0x06 + NewBlockMsg = 0x07 + GetNodeDataMsg = 0x0d + NodeDataMsg = 0x0e + GetReceiptsMsg = 0x0f + ReceiptsMsg = 0x10 NewPooledTransactionHashesMsg = 0x08 GetPooledTransactionsMsg = 0x09 PooledTransactionsMsg = 0x0a @@ -128,7 +124,7 @@ type GetBlockHeadersPacket struct { Reverse bool // Query direction (false = rising towards latest, true = falling towards genesis) } -// GetBlockHeadersPacket represents a block header query over eth/66 +// GetBlockHeadersPacket66 represents a block header query over eth/66 type GetBlockHeadersPacket66 struct { RequestId uint64 *GetBlockHeadersPacket diff --git a/eth/sync.go b/eth/sync.go index ab114b59f..27941158f 100644 --- a/eth/sync.go +++ b/eth/sync.go @@ -18,7 +18,6 @@ package eth import ( "math/big" - "math/rand" "sync/atomic" "time" @@ -28,23 +27,13 @@ import ( "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/p2p/enode" ) const ( forceSyncCycle = 10 * time.Second // Time interval to force syncs, even if few peers are available defaultMinSyncPeers = 5 // Amount of peers desired to start syncing - - // This is the target size for the packs of transactions sent by txsyncLoop64. - // A pack can get larger than this if a single transactions exceeds this size. - txsyncPackSize = 100 * 1024 ) -type txsync struct { - p *eth.Peer - txs []*types.Transaction -} - // syncTransactions starts sending all currently pending transactions to the given peer. func (h *handler) syncTransactions(p *eth.Peer) { // Assemble the set of transaction to broadcast or announce to the remote @@ -64,94 +53,11 @@ func (h *handler) syncTransactions(p *eth.Peer) { // The eth/65 protocol introduces proper transaction announcements, so instead // of dripping transactions across multiple peers, just send the entire list as // an announcement and let the remote side decide what they need (likely nothing). - if p.Version() >= eth.ETH65 { - hashes := make([]common.Hash, len(txs)) - for i, tx := range txs { - hashes[i] = tx.Hash() - } - p.AsyncSendPooledTransactionHashes(hashes) - return - } - // Out of luck, peer is running legacy protocols, drop the txs over - select { - case h.txsyncCh <- &txsync{p: p, txs: txs}: - case <-h.quitSync: - } -} - -// txsyncLoop64 takes care of the initial transaction sync for each new -// connection. When a new peer appears, we relay all currently pending -// transactions. In order to minimise egress bandwidth usage, we send -// the transactions in small packs to one peer at a time. -func (h *handler) txsyncLoop64() { - defer h.wg.Done() - - var ( - pending = make(map[enode.ID]*txsync) - sending = false // whether a send is active - pack = new(txsync) // the pack that is being sent - done = make(chan error, 1) // result of the send - ) - - // send starts a sending a pack of transactions from the sync. - send := func(s *txsync) { - if s.p.Version() >= eth.ETH65 { - panic("initial transaction syncer running on eth/65+") - } - // Fill pack with transactions up to the target size. - size := common.StorageSize(0) - pack.p = s.p - pack.txs = pack.txs[:0] - for i := 0; i < len(s.txs) && size < txsyncPackSize; i++ { - pack.txs = append(pack.txs, s.txs[i]) - size += s.txs[i].Size() - } - // Remove the transactions that will be sent. - s.txs = s.txs[:copy(s.txs, s.txs[len(pack.txs):])] - if len(s.txs) == 0 { - delete(pending, s.p.Peer.ID()) - } - // Send the pack in the background. - s.p.Log().Trace("Sending batch of transactions", "count", len(pack.txs), "bytes", size) - sending = true - go func() { done <- pack.p.SendTransactions(pack.txs) }() - } - // pick chooses the next pending sync. - pick := func() *txsync { - if len(pending) == 0 { - return nil - } - n := rand.Intn(len(pending)) + 1 - for _, s := range pending { - if n--; n == 0 { - return s - } - } - return nil - } - - for { - select { - case s := <-h.txsyncCh: - pending[s.p.Peer.ID()] = s - if !sending { - send(s) - } - case err := <-done: - sending = false - // Stop tracking peers that cause send failures. - if err != nil { - pack.p.Log().Debug("Transaction send failed", "err", err) - delete(pending, pack.p.Peer.ID()) - } - // Schedule the next send. - if s := pick(); s != nil { - send(s) - } - case <-h.quitSync: - return - } + hashes := make([]common.Hash, len(txs)) + for i, tx := range txs { + hashes[i] = tx.Hash() } + p.AsyncSendPooledTransactionHashes(hashes) } // chainSyncer coordinates blockchain sync components. diff --git a/eth/sync_test.go b/eth/sync_test.go index a0c6f8602..e96b9ee81 100644 --- a/eth/sync_test.go +++ b/eth/sync_test.go @@ -28,7 +28,6 @@ import ( ) // Tests that fast sync is disabled after a successful sync cycle. -func TestFastSyncDisabling65(t *testing.T) { testFastSyncDisabling(t, eth.ETH65) } func TestFastSyncDisabling66(t *testing.T) { testFastSyncDisabling(t, eth.ETH66) } // Tests that fast sync gets disabled as soon as a real block is successfully diff --git a/les/client_handler.go b/les/client_handler.go index e95996c51..b903b1106 100644 --- a/les/client_handler.go +++ b/les/client_handler.go @@ -472,7 +472,7 @@ func (d *downloaderPeerNotify) registerPeer(p *serverPeer) { handler: h, peer: p, } - h.downloader.RegisterLightPeer(p.id, eth.ETH65, pc) + h.downloader.RegisterLightPeer(p.id, eth.ETH66, pc) } func (d *downloaderPeerNotify) unregisterPeer(p *serverPeer) { From b69f5ca7d47e47335b9d19075245f9ee9fa9bd31 Mon Sep 17 00:00:00 2001 From: Ivan Bogatyy Date: Tue, 24 Aug 2021 21:54:00 +0300 Subject: [PATCH 12/73] cmd/utils: fix typo in variable name (#23451) --- cmd/geth/main.go | 2 +- cmd/geth/snapshot.go | 2 +- cmd/geth/usage.go | 2 +- cmd/utils/flags.go | 6 +++--- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index da62b8d4d..cfc1a62b6 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -124,7 +124,7 @@ var ( utils.MinerEtherbaseFlag, utils.MinerExtraDataFlag, utils.MinerRecommitIntervalFlag, - utils.MinerNoVerfiyFlag, + utils.MinerNoVerifyFlag, utils.NATFlag, utils.NoDiscoverFlag, utils.DiscoveryV5Flag, diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index 35d027fb1..9976db49f 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -232,7 +232,7 @@ func verifyState(ctx *cli.Context) error { } } if err := snaptree.Verify(root); err != nil { - log.Error("Failed to verfiy state", "root", root, "err", err) + log.Error("Failed to verify state", "root", root, "err", err) return err } log.Info("Verified the state", "root", root) diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 708edcc79..f39e4f402 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -185,7 +185,7 @@ var AppHelpFlagGroups = []flags.FlagGroup{ utils.MinerEtherbaseFlag, utils.MinerExtraDataFlag, utils.MinerRecommitIntervalFlag, - utils.MinerNoVerfiyFlag, + utils.MinerNoVerifyFlag, }, }, { diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 8a53f525f..2953a0032 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -460,7 +460,7 @@ var ( Usage: "Time interval to recreate the block being mined", Value: ethconfig.Defaults.Miner.Recommit, } - MinerNoVerfiyFlag = cli.BoolFlag{ + MinerNoVerifyFlag = cli.BoolFlag{ Name: "miner.noverify", Usage: "Disable remote sealing verification", } @@ -1398,8 +1398,8 @@ func setMiner(ctx *cli.Context, cfg *miner.Config) { if ctx.GlobalIsSet(MinerRecommitIntervalFlag.Name) { cfg.Recommit = ctx.GlobalDuration(MinerRecommitIntervalFlag.Name) } - if ctx.GlobalIsSet(MinerNoVerfiyFlag.Name) { - cfg.Noverify = ctx.GlobalBool(MinerNoVerfiyFlag.Name) + if ctx.GlobalIsSet(MinerNoVerifyFlag.Name) { + cfg.Noverify = ctx.GlobalBool(MinerNoVerifyFlag.Name) } if ctx.GlobalIsSet(LegacyMinerGasTargetFlag.Name) { log.Warn("The generic --miner.gastarget flag is deprecated and will be removed in the future!") From a789dcc9780e3d2364a7c6879af26aa0ed256fca Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 24 Aug 2021 20:54:55 +0200 Subject: [PATCH 13/73] metrics: fix compilation for GOOS=js (#23449) --- metrics/cpu_disabled.go | 2 +- metrics/cpu_enabled.go | 2 +- metrics/{cpu_windows.go => cputime_nop.go} | 2 ++ metrics/{cpu_syscall.go => cputime_unix.go} | 2 +- metrics/runtime_cgo.go | 3 +-- metrics/runtime_no_cgo.go | 2 +- 6 files changed, 7 insertions(+), 6 deletions(-) rename metrics/{cpu_windows.go => cputime_nop.go} (97%) rename metrics/{cpu_syscall.go => cputime_unix.go} (98%) diff --git a/metrics/cpu_disabled.go b/metrics/cpu_disabled.go index 6c3428993..b0b483ee4 100644 --- a/metrics/cpu_disabled.go +++ b/metrics/cpu_disabled.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -// +build ios +// +build ios js package metrics diff --git a/metrics/cpu_enabled.go b/metrics/cpu_enabled.go index 02192928b..edb824d9a 100644 --- a/metrics/cpu_enabled.go +++ b/metrics/cpu_enabled.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -// +build !ios +// +build !ios,!js package metrics diff --git a/metrics/cpu_windows.go b/metrics/cputime_nop.go similarity index 97% rename from metrics/cpu_windows.go rename to metrics/cputime_nop.go index fb29a52a8..1bf553727 100644 --- a/metrics/cpu_windows.go +++ b/metrics/cputime_nop.go @@ -14,6 +14,8 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +// +build windows js + package metrics // getProcessCPUTime returns 0 on Windows as there is no system call to resolve diff --git a/metrics/cpu_syscall.go b/metrics/cputime_unix.go similarity index 98% rename from metrics/cpu_syscall.go rename to metrics/cputime_unix.go index 50e04ef1d..55871e668 100644 --- a/metrics/cpu_syscall.go +++ b/metrics/cputime_unix.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -// +build !windows +// +build !windows,!js package metrics diff --git a/metrics/runtime_cgo.go b/metrics/runtime_cgo.go index e3391f4e8..11722f76c 100644 --- a/metrics/runtime_cgo.go +++ b/metrics/runtime_cgo.go @@ -1,5 +1,4 @@ -// +build cgo -// +build !appengine +// +build cgo,!appengine,!js package metrics diff --git a/metrics/runtime_no_cgo.go b/metrics/runtime_no_cgo.go index 616a3b475..e760af554 100644 --- a/metrics/runtime_no_cgo.go +++ b/metrics/runtime_no_cgo.go @@ -1,4 +1,4 @@ -// +build !cgo appengine +// +build !cgo appengine js package metrics From a5a52371789f2e2a3144c8a842c4238ba92cf301 Mon Sep 17 00:00:00 2001 From: gary rong Date: Wed, 25 Aug 2021 03:00:42 +0800 Subject: [PATCH 14/73] core, light, tests, trie: add state metrics (#23433) --- core/state/database.go | 2 +- core/state/metrics.go | 28 ++++++++++++ core/state/snapshot/generate.go | 2 +- core/state/snapshot/generate_test.go | 16 +++---- core/state/state_object.go | 16 ++++--- core/state/statedb.go | 26 ++++++++++- light/postprocess.go | 4 +- light/trie.go | 4 +- tests/fuzzers/stacktrie/trie_fuzzer.go | 2 +- tests/fuzzers/trie/trie-fuzzer.go | 4 +- trie/committer.go | 63 ++++++++++++++------------ trie/iterator_test.go | 2 +- trie/secure_trie.go | 2 +- trie/trie.go | 13 +++--- trie/trie_test.go | 22 ++++----- 15 files changed, 131 insertions(+), 75 deletions(-) create mode 100644 core/state/metrics.go diff --git a/core/state/database.go b/core/state/database.go index 1a06e3340..e25d059c0 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -86,7 +86,7 @@ type Trie interface { // Commit writes all nodes to the trie's memory database, tracking the internal // and external (for account tries) references. - Commit(onleaf trie.LeafCallback) (common.Hash, error) + Commit(onleaf trie.LeafCallback) (common.Hash, int, error) // NodeIterator returns an iterator that returns nodes of the trie. Iteration // starts at the key after the given start key. diff --git a/core/state/metrics.go b/core/state/metrics.go new file mode 100644 index 000000000..7b40ff37a --- /dev/null +++ b/core/state/metrics.go @@ -0,0 +1,28 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package state + +import "github.com/ethereum/go-ethereum/metrics" + +var ( + accountUpdatedMeter = metrics.NewRegisteredMeter("state/update/account", nil) + storageUpdatedMeter = metrics.NewRegisteredMeter("state/update/storage", nil) + accountDeletedMeter = metrics.NewRegisteredMeter("state/delete/account", nil) + storageDeletedMeter = metrics.NewRegisteredMeter("state/delete/storage", nil) + accountCommittedMeter = metrics.NewRegisteredMeter("state/commit/account", nil) + storageCommittedMeter = metrics.NewRegisteredMeter("state/commit/storage", nil) +) diff --git a/core/state/snapshot/generate.go b/core/state/snapshot/generate.go index 7e29e51b2..3e11b4ac6 100644 --- a/core/state/snapshot/generate.go +++ b/core/state/snapshot/generate.go @@ -436,7 +436,7 @@ func (dl *diskLayer) generateRange(root common.Hash, prefix []byte, kind string, for i, key := range result.keys { snapTrie.Update(key, result.vals[i]) } - root, _ := snapTrie.Commit(nil) + root, _, _ := snapTrie.Commit(nil) snapTrieDb.Commit(root, false, nil) } tr := result.tr diff --git a/core/state/snapshot/generate_test.go b/core/state/snapshot/generate_test.go index a92517b31..582da6a2e 100644 --- a/core/state/snapshot/generate_test.go +++ b/core/state/snapshot/generate_test.go @@ -60,7 +60,7 @@ func TestGeneration(t *testing.T) { acc = &Account{Balance: big.NewInt(3), Root: stTrie.Hash().Bytes(), CodeHash: emptyCode.Bytes()} val, _ = rlp.EncodeToBytes(acc) accTrie.Update([]byte("acc-3"), val) // 0x50815097425d000edfc8b3a4a13e175fc2bdcfee8bdfbf2d1ff61041d3c235b2 - root, _ := accTrie.Commit(nil) // Root: 0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd + root, _, _ := accTrie.Commit(nil) // Root: 0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd triedb.Commit(root, false, nil) if have, want := root, common.HexToHash("0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd"); have != want { @@ -128,7 +128,7 @@ func TestGenerateExistentState(t *testing.T) { rawdb.WriteStorageSnapshot(diskdb, hashData([]byte("acc-3")), hashData([]byte("key-2")), []byte("val-2")) rawdb.WriteStorageSnapshot(diskdb, hashData([]byte("acc-3")), hashData([]byte("key-3")), []byte("val-3")) - root, _ := accTrie.Commit(nil) // Root: 0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd + root, _, _ := accTrie.Commit(nil) // Root: 0xe3712f1a226f3782caca78ca770ccc19ee000552813a9f59d479f8611db9b1fd triedb.Commit(root, false, nil) snap := generateSnapshot(diskdb, triedb, 16, root) @@ -215,12 +215,12 @@ func (t *testHelper) makeStorageTrie(keys []string, vals []string) []byte { for i, k := range keys { stTrie.Update([]byte(k), []byte(vals[i])) } - root, _ := stTrie.Commit(nil) + root, _, _ := stTrie.Commit(nil) return root.Bytes() } func (t *testHelper) Generate() (common.Hash, *diskLayer) { - root, _ := t.accTrie.Commit(nil) + root, _, _ := t.accTrie.Commit(nil) t.triedb.Commit(root, false, nil) snap := generateSnapshot(t.diskdb, t.triedb, 16, root) return root, snap @@ -575,7 +575,7 @@ func TestGenerateWithExtraAccounts(t *testing.T) { rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("b-key-2")), []byte("b-val-2")) rawdb.WriteStorageSnapshot(diskdb, key, hashData([]byte("b-key-3")), []byte("b-val-3")) } - root, _ := accTrie.Commit(nil) + root, _, _ := accTrie.Commit(nil) t.Logf("root: %x", root) triedb.Commit(root, false, nil) // To verify the test: If we now inspect the snap db, there should exist extraneous storage items @@ -637,7 +637,7 @@ func TestGenerateWithManyExtraAccounts(t *testing.T) { rawdb.WriteAccountSnapshot(diskdb, key, val) } } - root, _ := accTrie.Commit(nil) + root, _, _ := accTrie.Commit(nil) t.Logf("root: %x", root) triedb.Commit(root, false, nil) @@ -690,7 +690,7 @@ func TestGenerateWithExtraBeforeAndAfter(t *testing.T) { rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x07"), val) } - root, _ := accTrie.Commit(nil) + root, _, _ := accTrie.Commit(nil) t.Logf("root: %x", root) triedb.Commit(root, false, nil) @@ -734,7 +734,7 @@ func TestGenerateWithMalformedSnapdata(t *testing.T) { rawdb.WriteAccountSnapshot(diskdb, common.HexToHash("0x05"), junk) } - root, _ := accTrie.Commit(nil) + root, _, _ := accTrie.Commit(nil) t.Logf("root: %x", root) triedb.Commit(root, false, nil) diff --git a/core/state/state_object.go b/core/state/state_object.go index 38621ffb6..7b366d64c 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -329,7 +329,7 @@ func (s *stateObject) finalise(prefetch bool) { // It will return nil if the trie has not been loaded and no changes have been made func (s *stateObject) updateTrie(db Database) Trie { // Make sure all dirty slots are finalized into the pending storage area - s.finalise(false) // Don't prefetch any more, pull directly if need be + s.finalise(false) // Don't prefetch anymore, pull directly if need be if len(s.pendingStorage) == 0 { return s.trie } @@ -354,10 +354,12 @@ func (s *stateObject) updateTrie(db Database) Trie { var v []byte if (value == common.Hash{}) { s.setError(tr.TryDelete(key[:])) + s.db.StorageDeleted += 1 } else { // Encoding []byte cannot fail, ok to ignore the error. v, _ = rlp.EncodeToBytes(common.TrimLeftZeroes(value[:])) s.setError(tr.TryUpdate(key[:], v)) + s.db.StorageUpdated += 1 } // If state snapshotting is active, cache the data til commit if s.db.snap != nil { @@ -368,7 +370,7 @@ func (s *stateObject) updateTrie(db Database) Trie { s.db.snapStorage[s.addrHash] = storage } } - storage[crypto.HashData(hasher, key[:])] = v // v will be nil if value is 0x00 + storage[crypto.HashData(hasher, key[:])] = v // v will be nil if it's deleted } usedStorage = append(usedStorage, common.CopyBytes(key[:])) // Copy needed for closure } @@ -396,23 +398,23 @@ func (s *stateObject) updateRoot(db Database) { // CommitTrie the storage trie of the object to db. // This updates the trie root. -func (s *stateObject) CommitTrie(db Database) error { +func (s *stateObject) CommitTrie(db Database) (int, error) { // If nothing changed, don't bother with hashing anything if s.updateTrie(db) == nil { - return nil + return 0, nil } if s.dbErr != nil { - return s.dbErr + return 0, s.dbErr } // Track the amount of time wasted on committing the storage trie if metrics.EnabledExpensive { defer func(start time.Time) { s.db.StorageCommits += time.Since(start) }(time.Now()) } - root, err := s.trie.Commit(nil) + root, committed, err := s.trie.Commit(nil) if err == nil { s.data.Root = root } - return err + return committed, err } // AddBalance adds amount to s's balance. diff --git a/core/state/statedb.go b/core/state/statedb.go index 61f0f5b9f..a6d47179b 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -117,6 +117,11 @@ type StateDB struct { SnapshotAccountReads time.Duration SnapshotStorageReads time.Duration SnapshotCommits time.Duration + + AccountUpdated int + StorageUpdated int + AccountDeleted int + StorageDeleted int } // New creates a new state from a given trie. @@ -860,8 +865,10 @@ func (s *StateDB) IntermediateRoot(deleteEmptyObjects bool) common.Hash { for addr := range s.stateObjectsPending { if obj := s.stateObjects[addr]; obj.deleted { s.deleteStateObject(obj) + s.AccountDeleted += 1 } else { s.updateStateObject(obj) + s.AccountUpdated += 1 } usedAddrs = append(usedAddrs, common.CopyBytes(addr[:])) // Copy needed for closure } @@ -903,6 +910,7 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { s.IntermediateRoot(deleteEmptyObjects) // Commit objects to the trie, measuring the elapsed time + var storageCommitted int codeWriter := s.db.TrieDB().DiskDB().NewBatch() for addr := range s.stateObjectsDirty { if obj := s.stateObjects[addr]; !obj.deleted { @@ -912,9 +920,11 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { obj.dirtyCode = false } // Write any storage changes in the state object to its storage trie - if err := obj.CommitTrie(s.db); err != nil { + committed, err := obj.CommitTrie(s.db) + if err != nil { return common.Hash{}, err } + storageCommitted += committed } } if len(s.stateObjectsDirty) > 0 { @@ -933,7 +943,7 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { // The onleaf func is called _serially_, so we can reuse the same account // for unmarshalling every time. var account Account - root, err := s.trie.Commit(func(_ [][]byte, _ []byte, leaf []byte, parent common.Hash) error { + root, accountCommitted, err := s.trie.Commit(func(_ [][]byte, _ []byte, leaf []byte, parent common.Hash) error { if err := rlp.DecodeBytes(leaf, &account); err != nil { return nil } @@ -942,8 +952,20 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { } return nil }) + if err != nil { + return common.Hash{}, err + } if metrics.EnabledExpensive { s.AccountCommits += time.Since(start) + + accountUpdatedMeter.Mark(int64(s.AccountUpdated)) + storageUpdatedMeter.Mark(int64(s.StorageUpdated)) + accountDeletedMeter.Mark(int64(s.AccountDeleted)) + storageDeletedMeter.Mark(int64(s.StorageDeleted)) + accountCommittedMeter.Mark(int64(accountCommitted)) + storageCommittedMeter.Mark(int64(storageCommitted)) + s.AccountUpdated, s.AccountDeleted = 0, 0 + s.StorageUpdated, s.StorageDeleted = 0, 0 } // If snapshotting is enabled, update the snapshot tree with this new version if s.snap != nil { diff --git a/light/postprocess.go b/light/postprocess.go index 891c8a586..ce38d091e 100644 --- a/light/postprocess.go +++ b/light/postprocess.go @@ -217,7 +217,7 @@ func (c *ChtIndexerBackend) Process(ctx context.Context, header *types.Header) e // Commit implements core.ChainIndexerBackend func (c *ChtIndexerBackend) Commit() error { - root, err := c.trie.Commit(nil) + root, _, err := c.trie.Commit(nil) if err != nil { return err } @@ -454,7 +454,7 @@ func (b *BloomTrieIndexerBackend) Commit() error { b.trie.Delete(encKey[:]) } } - root, err := b.trie.Commit(nil) + root, _, err := b.trie.Commit(nil) if err != nil { return err } diff --git a/light/trie.go b/light/trie.go index 0516b9448..39e928bbe 100644 --- a/light/trie.go +++ b/light/trie.go @@ -125,9 +125,9 @@ func (t *odrTrie) TryDelete(key []byte) error { }) } -func (t *odrTrie) Commit(onleaf trie.LeafCallback) (common.Hash, error) { +func (t *odrTrie) Commit(onleaf trie.LeafCallback) (common.Hash, int, error) { if t.trie == nil { - return t.id.Root, nil + return t.id.Root, 0, nil } return t.trie.Commit(onleaf) } diff --git a/tests/fuzzers/stacktrie/trie_fuzzer.go b/tests/fuzzers/stacktrie/trie_fuzzer.go index 5cea7769c..e73ef4851 100644 --- a/tests/fuzzers/stacktrie/trie_fuzzer.go +++ b/tests/fuzzers/stacktrie/trie_fuzzer.go @@ -173,7 +173,7 @@ func (f *fuzzer) fuzz() int { return 0 } // Flush trie -> database - rootA, err := trieA.Commit(nil) + rootA, _, err := trieA.Commit(nil) if err != nil { panic(err) } diff --git a/tests/fuzzers/trie/trie-fuzzer.go b/tests/fuzzers/trie/trie-fuzzer.go index d0b614c9a..e993af47c 100644 --- a/tests/fuzzers/trie/trie-fuzzer.go +++ b/tests/fuzzers/trie/trie-fuzzer.go @@ -162,11 +162,11 @@ func runRandTest(rt randTest) error { rt[i].err = fmt.Errorf("mismatch for key 0x%x, got 0x%x want 0x%x", step.key, v, want) } case opCommit: - _, rt[i].err = tr.Commit(nil) + _, _, rt[i].err = tr.Commit(nil) case opHash: tr.Hash() case opReset: - hash, err := tr.Commit(nil) + hash, _, err := tr.Commit(nil) if err != nil { return err } diff --git a/trie/committer.go b/trie/committer.go index ce4065f5f..0721990a2 100644 --- a/trie/committer.go +++ b/trie/committer.go @@ -72,24 +72,24 @@ func returnCommitterToPool(h *committer) { committerPool.Put(h) } -// commit collapses a node down into a hash node and inserts it into the database -func (c *committer) Commit(n node, db *Database) (hashNode, error) { +// Commit collapses a node down into a hash node and inserts it into the database +func (c *committer) Commit(n node, db *Database) (hashNode, int, error) { if db == nil { - return nil, errors.New("no db provided") + return nil, 0, errors.New("no db provided") } - h, err := c.commit(n, db) + h, committed, err := c.commit(n, db) if err != nil { - return nil, err + return nil, 0, err } - return h.(hashNode), nil + return h.(hashNode), committed, nil } // commit collapses a node down into a hash node and inserts it into the database -func (c *committer) commit(n node, db *Database) (node, error) { +func (c *committer) commit(n node, db *Database) (node, int, error) { // if this path is clean, use available cached data hash, dirty := n.cache() if hash != nil && !dirty { - return hash, nil + return hash, 0, nil } // Commit children, then parent, and remove remove the dirty flag. switch cn := n.(type) { @@ -97,37 +97,38 @@ func (c *committer) commit(n node, db *Database) (node, error) { // Commit child collapsed := cn.copy() - // If the child is fullnode, recursively commit. - // Otherwise it can only be hashNode or valueNode. + // If the child is fullNode, recursively commit, + // otherwise it can only be hashNode or valueNode. + var childCommitted int if _, ok := cn.Val.(*fullNode); ok { - childV, err := c.commit(cn.Val, db) + childV, committed, err := c.commit(cn.Val, db) if err != nil { - return nil, err + return nil, 0, err } - collapsed.Val = childV + collapsed.Val, childCommitted = childV, committed } // The key needs to be copied, since we're delivering it to database collapsed.Key = hexToCompact(cn.Key) hashedNode := c.store(collapsed, db) if hn, ok := hashedNode.(hashNode); ok { - return hn, nil + return hn, childCommitted + 1, nil } - return collapsed, nil + return collapsed, childCommitted, nil case *fullNode: - hashedKids, err := c.commitChildren(cn, db) + hashedKids, childCommitted, err := c.commitChildren(cn, db) if err != nil { - return nil, err + return nil, 0, err } collapsed := cn.copy() collapsed.Children = hashedKids hashedNode := c.store(collapsed, db) if hn, ok := hashedNode.(hashNode); ok { - return hn, nil + return hn, childCommitted + 1, nil } - return collapsed, nil + return collapsed, childCommitted, nil case hashNode: - return cn, nil + return cn, 0, nil default: // nil, valuenode shouldn't be committed panic(fmt.Sprintf("%T: invalid node: %v", n, n)) @@ -135,8 +136,11 @@ func (c *committer) commit(n node, db *Database) (node, error) { } // commitChildren commits the children of the given fullnode -func (c *committer) commitChildren(n *fullNode, db *Database) ([17]node, error) { - var children [17]node +func (c *committer) commitChildren(n *fullNode, db *Database) ([17]node, int, error) { + var ( + committed int + children [17]node + ) for i := 0; i < 16; i++ { child := n.Children[i] if child == nil { @@ -144,25 +148,26 @@ func (c *committer) commitChildren(n *fullNode, db *Database) ([17]node, error) } // If it's the hashed child, save the hash value directly. // Note: it's impossible that the child in range [0, 15] - // is a valuenode. + // is a valueNode. if hn, ok := child.(hashNode); ok { children[i] = hn continue } // Commit the child recursively and store the "hashed" value. // Note the returned node can be some embedded nodes, so it's - // possible the type is not hashnode. - hashed, err := c.commit(child, db) + // possible the type is not hashNode. + hashed, childCommitted, err := c.commit(child, db) if err != nil { - return children, err + return children, 0, err } children[i] = hashed + committed += childCommitted } // For the 17th child, it's possible the type is valuenode. if n.Children[16] != nil { children[16] = n.Children[16] } - return children, nil + return children, committed, nil } // store hashes the node n and if we have a storage layer specified, it writes @@ -176,7 +181,7 @@ func (c *committer) store(n node, db *Database) node { ) if hash == nil { // This was not generated - must be a small node stored in the parent. - // In theory we should apply the leafCall here if it's not nil(embedded + // In theory, we should apply the leafCall here if it's not nil(embedded // node usually contains value). But small value(less than 32bytes) is // not our target. return n @@ -224,7 +229,7 @@ func (c *committer) commitLoop(db *Database) { } case *fullNode: // For children in range [0, 15], it's impossible - // to contain valuenode. Only check the 17th child. + // to contain valueNode. Only check the 17th child. if n.Children[16] != nil { c.onleaf(nil, nil, n.Children[16].(valueNode), hash) } diff --git a/trie/iterator_test.go b/trie/iterator_test.go index 2518f7bac..95cafdd3b 100644 --- a/trie/iterator_test.go +++ b/trie/iterator_test.go @@ -393,7 +393,7 @@ func testIteratorContinueAfterSeekError(t *testing.T, memonly bool) { for _, val := range testdata1 { ctr.Update([]byte(val.k), []byte(val.v)) } - root, _ := ctr.Commit(nil) + root, _, _ := ctr.Commit(nil) if !memonly { triedb.Commit(root, true, nil) } diff --git a/trie/secure_trie.go b/trie/secure_trie.go index e38471c1b..4d0050bc5 100644 --- a/trie/secure_trie.go +++ b/trie/secure_trie.go @@ -144,7 +144,7 @@ func (t *SecureTrie) GetKey(shaKey []byte) []byte { // // Committing flushes nodes from memory. Subsequent Get calls will load nodes // from the database. -func (t *SecureTrie) Commit(onleaf LeafCallback) (root common.Hash, err error) { +func (t *SecureTrie) Commit(onleaf LeafCallback) (common.Hash, int, error) { // Write all the pre-images to the actual disk database if len(t.getSecKeyCache()) > 0 { if t.trie.db.preimages != nil { // Ugly direct check but avoids the below write lock diff --git a/trie/trie.go b/trie/trie.go index e492a532c..7ea7efa83 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -514,12 +514,12 @@ func (t *Trie) Hash() common.Hash { // Commit writes all nodes to the trie's memory database, tracking the internal // and external (for account tries) references. -func (t *Trie) Commit(onleaf LeafCallback) (root common.Hash, err error) { +func (t *Trie) Commit(onleaf LeafCallback) (common.Hash, int, error) { if t.db == nil { panic("commit called on trie with nil database") } if t.root == nil { - return emptyRoot, nil + return emptyRoot, 0, nil } // Derive the hash for all dirty nodes first. We hold the assumption // in the following procedure that all nodes are hashed. @@ -531,7 +531,7 @@ func (t *Trie) Commit(onleaf LeafCallback) (root common.Hash, err error) { // up goroutines. This can happen e.g. if we load a trie for reading storage // values, but don't write to it. if _, dirty := t.root.cache(); !dirty { - return rootHash, nil + return rootHash, 0, nil } var wg sync.WaitGroup if onleaf != nil { @@ -543,8 +543,7 @@ func (t *Trie) Commit(onleaf LeafCallback) (root common.Hash, err error) { h.commitLoop(t.db) }() } - var newRoot hashNode - newRoot, err = h.Commit(t.root, t.db) + newRoot, committed, err := h.Commit(t.root, t.db) if onleaf != nil { // The leafch is created in newCommitter if there was an onleaf callback // provided. The commitLoop only _reads_ from it, and the commit @@ -554,10 +553,10 @@ func (t *Trie) Commit(onleaf LeafCallback) (root common.Hash, err error) { wg.Wait() } if err != nil { - return common.Hash{}, err + return common.Hash{}, 0, err } t.root = newRoot - return rootHash, nil + return rootHash, committed, nil } // hashRoot calculates the root hash of the given trie diff --git a/trie/trie_test.go b/trie/trie_test.go index 492b423c2..ae1871e42 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -90,7 +90,7 @@ func testMissingNode(t *testing.T, memonly bool) { trie, _ := New(common.Hash{}, triedb) updateString(trie, "120000", "qwerqwerqwerqwerqwerqwerqwerqwer") updateString(trie, "123456", "asdfasdfasdfasdfasdfasdfasdfasdf") - root, _ := trie.Commit(nil) + root, _, _ := trie.Commit(nil) if !memonly { triedb.Commit(root, true, nil) } @@ -172,7 +172,7 @@ func TestInsert(t *testing.T) { updateString(trie, "A", "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa") exp = common.HexToHash("d23786fb4a010da3ce639d66d5e904a11dbc02746d1ce25029e53290cabf28ab") - root, err := trie.Commit(nil) + root, _, err := trie.Commit(nil) if err != nil { t.Fatalf("commit error: %v", err) } @@ -270,7 +270,7 @@ func TestReplication(t *testing.T) { for _, val := range vals { updateString(trie, val.k, val.v) } - exp, err := trie.Commit(nil) + exp, _, err := trie.Commit(nil) if err != nil { t.Fatalf("commit error: %v", err) } @@ -285,7 +285,7 @@ func TestReplication(t *testing.T) { t.Errorf("trie2 doesn't have %q => %q", kv.k, kv.v) } } - hash, err := trie2.Commit(nil) + hash, _, err := trie2.Commit(nil) if err != nil { t.Fatalf("commit error: %v", err) } @@ -429,11 +429,11 @@ func runRandTest(rt randTest) bool { rt[i].err = fmt.Errorf("mismatch for key 0x%x, got 0x%x want 0x%x", step.key, v, want) } case opCommit: - _, rt[i].err = tr.Commit(nil) + _, _, rt[i].err = tr.Commit(nil) case opHash: tr.Hash() case opReset: - hash, err := tr.Commit(nil) + hash, _, err := tr.Commit(nil) if err != nil { rt[i].err = err return false @@ -633,7 +633,7 @@ func TestCommitAfterHash(t *testing.T) { if exp != root { t.Errorf("got %x, exp %x", root, exp) } - root, _ = trie.Commit(nil) + root, _, _ = trie.Commit(nil) if exp != root { t.Errorf("got %x, exp %x", root, exp) } @@ -740,7 +740,7 @@ func TestCommitSequence(t *testing.T) { trie.Update(crypto.Keccak256(addresses[i][:]), accounts[i]) } // Flush trie -> database - root, _ := trie.Commit(nil) + root, _, _ := trie.Commit(nil) // Flush memdb -> disk (sponge) db.Commit(root, false, func(c common.Hash) { // And spongify the callback-order @@ -792,7 +792,7 @@ func TestCommitSequenceRandomBlobs(t *testing.T) { trie.Update(key, val) } // Flush trie -> database - root, _ := trie.Commit(nil) + root, _, _ := trie.Commit(nil) // Flush memdb -> disk (sponge) db.Commit(root, false, func(c common.Hash) { // And spongify the callback-order @@ -834,7 +834,7 @@ func TestCommitSequenceStackTrie(t *testing.T) { stTrie.TryUpdate(key, val) } // Flush trie -> database - root, _ := trie.Commit(nil) + root, _, _ := trie.Commit(nil) // Flush memdb -> disk (sponge) db.Commit(root, false, nil) // And flush stacktrie -> disk @@ -879,7 +879,7 @@ func TestCommitSequenceSmallRoot(t *testing.T) { trie.TryUpdate(key, []byte{0x1}) stTrie.TryUpdate(key, []byte{0x1}) // Flush trie -> database - root, _ := trie.Commit(nil) + root, _, _ := trie.Commit(nil) // Flush memdb -> disk (sponge) db.Commit(root, false, nil) // And flush stacktrie -> disk From fe2f153b556a0f1225c571fd5df7967a33244bf1 Mon Sep 17 00:00:00 2001 From: ucwong Date: Wed, 25 Aug 2021 03:02:46 +0800 Subject: [PATCH 15/73] go.mod: update goleveldb (#23417) --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index ee9ffa045..ce19afc85 100644 --- a/go.mod +++ b/go.mod @@ -30,7 +30,7 @@ require ( github.com/go-sourcemap/sourcemap v2.1.2+incompatible // indirect github.com/go-stack/stack v1.8.0 github.com/golang/protobuf v1.4.3 - github.com/golang/snappy v0.0.3 + github.com/golang/snappy v0.0.4 github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa github.com/google/uuid v1.1.5 github.com/gorilla/websocket v1.4.2 @@ -59,7 +59,7 @@ require ( github.com/shirou/gopsutil v3.21.4-0.20210419000835-c7a38de76ee5+incompatible github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 github.com/stretchr/testify v1.7.0 - github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 + github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 github.com/tklauser/go-sysconf v0.3.5 // indirect github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 diff --git a/go.sum b/go.sum index d6e9639d2..3d8a04236 100644 --- a/go.sum +++ b/go.sum @@ -185,9 +185,9 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= +github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -389,8 +389,8 @@ github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81P github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 h1:xQdMZ1WLrgkkvOZ/LDQxjVxMLdby7osSh4ZEVa5sIjs= -github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954/go.mod h1:u2MKkTVTVJWe5D1rCvame8WqhBd88EuIwODJZ1VHCPM= +github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= +github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tklauser/go-sysconf v0.3.5 h1:uu3Xl4nkLzQfXNsWn15rPc/HQCJKObbt1dKJeWp3vU4= github.com/tklauser/go-sysconf v0.3.5/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI= From 83ad92c42191a47b3ded38a9fd22fd0bed2f05f6 Mon Sep 17 00:00:00 2001 From: gary rong Date: Wed, 25 Aug 2021 17:56:25 +0800 Subject: [PATCH 16/73] les: fix data races in tests (#23457) --- les/client_handler.go | 11 ++++++---- les/fetcher.go | 8 ------- les/fetcher_test.go | 49 +++++++++---------------------------------- les/odr_test.go | 6 +++--- les/server_handler.go | 3 +-- les/sync_test.go | 8 +++---- les/test_helper.go | 12 +++++------ les/ulc_test.go | 27 +++++++++++++++--------- 8 files changed, 48 insertions(+), 76 deletions(-) diff --git a/les/client_handler.go b/les/client_handler.go index b903b1106..4a550b207 100644 --- a/les/client_handler.go +++ b/les/client_handler.go @@ -100,11 +100,11 @@ func (h *clientHandler) runPeer(version uint, p *p2p.Peer, rw p2p.MsgReadWriter) defer peer.close() h.wg.Add(1) defer h.wg.Done() - err := h.handle(peer) + err := h.handle(peer, false) return err } -func (h *clientHandler) handle(p *serverPeer) error { +func (h *clientHandler) handle(p *serverPeer, noInitAnnounce bool) error { if h.backend.peers.len() >= h.backend.config.LightPeers && !p.Peer.Info().Network.Trusted { return p2p.DiscTooManyPeers } @@ -143,8 +143,11 @@ func (h *clientHandler) handle(p *serverPeer) error { connectionTimer.Update(time.Duration(mclock.Now() - connectedAt)) serverConnectionGauge.Update(int64(h.backend.peers.len())) }() - h.fetcher.announce(p, &announceData{Hash: p.headInfo.Hash, Number: p.headInfo.Number, Td: p.headInfo.Td}) - + // It's mainly used in testing which requires discarding initial + // signal to prevent syncing. + if !noInitAnnounce { + h.fetcher.announce(p, &announceData{Hash: p.headInfo.Hash, Number: p.headInfo.Number, Td: p.headInfo.Td}) + } // Mark the peer starts to be served. atomic.StoreUint32(&p.serving, 1) defer atomic.StoreUint32(&p.serving, 0) diff --git a/les/fetcher.go b/les/fetcher.go index a6d1c93c4..5eea99674 100644 --- a/les/fetcher.go +++ b/les/fetcher.go @@ -153,9 +153,7 @@ type lightFetcher struct { synchronise func(peer *serverPeer) // Test fields or hooks - noAnnounce bool newHeadHook func(*types.Header) - newAnnounce func(*serverPeer, *announceData) } // newLightFetcher creates a light fetcher instance. @@ -474,12 +472,6 @@ func (f *lightFetcher) mainloop() { // announce processes a new announcement message received from a peer. func (f *lightFetcher) announce(p *serverPeer, head *announceData) { - if f.newAnnounce != nil { - f.newAnnounce(p, head) - } - if f.noAnnounce { - return - } select { case f.announceCh <- &announce{peerid: p.ID(), trust: p.trusted, data: head}: case <-f.closeCh: diff --git a/les/fetcher_test.go b/les/fetcher_test.go index d3a74d25c..ef700651e 100644 --- a/les/fetcher_test.go +++ b/les/fetcher_test.go @@ -74,14 +74,12 @@ func testSequentialAnnouncements(t *testing.T, protocol int) { s, c, teardown := newClientServerEnv(t, netconfig) defer teardown() - // Create connected peer pair. - c.handler.fetcher.noAnnounce = true // Ignore the first announce from peer which can trigger a resync. - p1, _, err := newTestPeerPair("peer", protocol, s.handler, c.handler) + // Create connected peer pair, the initial signal from LES server + // is discarded to prevent syncing. + p1, _, err := newTestPeerPair("peer", protocol, s.handler, c.handler, true) if err != nil { t.Fatalf("Failed to create peer pair %v", err) } - c.handler.fetcher.noAnnounce = false - importCh := make(chan interface{}) c.handler.fetcher.newHeadHook = func(header *types.Header) { importCh <- header @@ -114,14 +112,12 @@ func testGappedAnnouncements(t *testing.T, protocol int) { s, c, teardown := newClientServerEnv(t, netconfig) defer teardown() - // Create connected peer pair. - c.handler.fetcher.noAnnounce = true // Ignore the first announce from peer which can trigger a resync. - peer, _, err := newTestPeerPair("peer", protocol, s.handler, c.handler) + // Create connected peer pair, the initial signal from LES server + // is discarded to prevent syncing. + peer, _, err := newTestPeerPair("peer", protocol, s.handler, c.handler, true) if err != nil { t.Fatalf("Failed to create peer pair %v", err) } - c.handler.fetcher.noAnnounce = false - done := make(chan *types.Header, 1) c.handler.fetcher.newHeadHook = func(header *types.Header) { done <- header } @@ -141,29 +137,11 @@ func testGappedAnnouncements(t *testing.T, protocol int) { verifyChainHeight(t, c.handler.fetcher, 4) // Send a reorged announcement - var newAnno = make(chan struct{}, 1) - c.handler.fetcher.noAnnounce = true - c.handler.fetcher.newAnnounce = func(*serverPeer, *announceData) { - newAnno <- struct{}{} - } blocks, _ := core.GenerateChain(rawdb.ReadChainConfig(s.db, s.backend.Blockchain().Genesis().Hash()), s.backend.Blockchain().GetBlockByNumber(3), ethash.NewFaker(), s.db, 2, func(i int, gen *core.BlockGen) { gen.OffsetTime(-9) // higher block difficulty }) s.backend.Blockchain().InsertChain(blocks) - <-newAnno - c.handler.fetcher.noAnnounce = false - c.handler.fetcher.newAnnounce = nil - - latest = blocks[len(blocks)-1].Header() - hash, number = latest.Hash(), latest.Number.Uint64() - td = rawdb.ReadTd(s.db, hash, number) - - announce = announceData{hash, number, td, 1, nil} - if peer.cpeer.announceType == announceTypeSigned { - announce.sign(s.handler.server.privateKey) - } - peer.cpeer.sendAnnounce(announce) <-done // Wait syncing verifyChainHeight(t, c.handler.fetcher, 5) @@ -206,20 +184,15 @@ func testTrustedAnnouncement(t *testing.T, protocol int) { teardowns[i]() } }() - - c.handler.fetcher.noAnnounce = true // Ignore the first announce from peer which can trigger a resync. - // Connect all server instances. for i := 0; i < len(servers); i++ { - sp, cp, err := connect(servers[i].handler, nodes[i].ID(), c.handler, protocol) + sp, cp, err := connect(servers[i].handler, nodes[i].ID(), c.handler, protocol, true) if err != nil { t.Fatalf("connect server and client failed, err %s", err) } cpeers = append(cpeers, cp) speers = append(speers, sp) } - c.handler.fetcher.noAnnounce = false - newHead := make(chan *types.Header, 1) c.handler.fetcher.newHeadHook = func(header *types.Header) { newHead <- header } @@ -262,14 +235,12 @@ func testInvalidAnnounces(t *testing.T, protocol int) { s, c, teardown := newClientServerEnv(t, netconfig) defer teardown() - // Create connected peer pair. - c.handler.fetcher.noAnnounce = true // Ignore the first announce from peer which can trigger a resync. - peer, _, err := newTestPeerPair("peer", lpv3, s.handler, c.handler) + // Create connected peer pair, the initial signal from LES server + // is discarded to prevent syncing. + peer, _, err := newTestPeerPair("peer", lpv3, s.handler, c.handler, true) if err != nil { t.Fatalf("Failed to create peer pair %v", err) } - c.handler.fetcher.noAnnounce = false - done := make(chan *types.Header, 1) c.handler.fetcher.newHeadHook = func(header *types.Header) { done <- header } diff --git a/les/odr_test.go b/les/odr_test.go index ea88495d1..ad77abf5b 100644 --- a/les/odr_test.go +++ b/les/odr_test.go @@ -401,9 +401,9 @@ func testGetTxStatusFromUnindexedPeers(t *testing.T, protocol int) { closeFns = append(closeFns, closePeer) // Create a one-time routine for serving message - go func(i int, peer *testPeer) { - serveMsg(peer, testspec.txLookups[i]) - }(i, peer) + go func(i int, peer *testPeer, lookup uint64) { + serveMsg(peer, lookup) + }(i, peer, testspec.txLookups[i]) } // Send out the GetTxStatus requests, compare the result with diff --git a/les/server_handler.go b/les/server_handler.go index 80fcf1c44..fa20fd7b3 100644 --- a/les/server_handler.go +++ b/les/server_handler.go @@ -28,7 +28,6 @@ import ( "github.com/ethereum/go-ethereum/core/forkid" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" - "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/les/flowcontrol" "github.com/ethereum/go-ethereum/light" @@ -408,7 +407,7 @@ func (h *serverHandler) broadcastLoop() { defer headSub.Unsubscribe() var ( - lastHead *types.Header + lastHead = h.blockchain.CurrentHeader() lastTd = common.Big0 ) for { diff --git a/les/sync_test.go b/les/sync_test.go index d3bb90df0..3fc2a9c15 100644 --- a/les/sync_test.go +++ b/les/sync_test.go @@ -116,7 +116,7 @@ func testCheckpointSyncing(t *testing.T, protocol int, syncMode int) { } // Create connected peer pair. - peer1, peer2, err := newTestPeerPair("peer", protocol, server.handler, client.handler) + peer1, peer2, err := newTestPeerPair("peer", protocol, server.handler, client.handler, false) if err != nil { t.Fatalf("Failed to connect testing peers %v", err) } @@ -218,7 +218,7 @@ func testMissOracleBackend(t *testing.T, hasCheckpoint bool, protocol int) { } } // Create connected peer pair. - if _, _, err := newTestPeerPair("peer", 2, server.handler, client.handler); err != nil { + if _, _, err := newTestPeerPair("peer", 2, server.handler, client.handler, false); err != nil { t.Fatalf("Failed to connect testing peers %v", err) } select { @@ -291,7 +291,7 @@ func testSyncFromConfiguredCheckpoint(t *testing.T, protocol int) { } } // Create connected peer pair. - if _, _, err := newTestPeerPair("peer", 2, server.handler, client.handler); err != nil { + if _, _, err := newTestPeerPair("peer", 2, server.handler, client.handler, false); err != nil { t.Fatalf("Failed to connect testing peers %v", err) } @@ -364,7 +364,7 @@ func testSyncAll(t *testing.T, protocol int) { } } // Create connected peer pair. - if _, _, err := newTestPeerPair("peer", 2, server.handler, client.handler); err != nil { + if _, _, err := newTestPeerPair("peer", 2, server.handler, client.handler, false); err != nil { t.Fatalf("Failed to connect testing peers %v", err) } diff --git a/les/test_helper.go b/les/test_helper.go index 9ff2583b9..21d0f191c 100644 --- a/les/test_helper.go +++ b/les/test_helper.go @@ -398,7 +398,7 @@ func (p *testPeer) close() { p.app.Close() } -func newTestPeerPair(name string, version int, server *serverHandler, client *clientHandler) (*testPeer, *testPeer, error) { +func newTestPeerPair(name string, version int, server *serverHandler, client *clientHandler, noInitAnnounce bool) (*testPeer, *testPeer, error) { // Create a message pipe to communicate through app, net := p2p.MsgPipe() @@ -423,16 +423,16 @@ func newTestPeerPair(name string, version int, server *serverHandler, client *cl select { case <-client.closeCh: errc2 <- p2p.DiscQuitting - case errc2 <- client.handle(peer2): + case errc2 <- client.handle(peer2, noInitAnnounce): } }() // Ensure the connection is established or exits when any error occurs for { select { case err := <-errc1: - return nil, nil, fmt.Errorf("Failed to establish protocol connection %v", err) + return nil, nil, fmt.Errorf("failed to establish protocol connection %v", err) case err := <-errc2: - return nil, nil, fmt.Errorf("Failed to establish protocol connection %v", err) + return nil, nil, fmt.Errorf("failed to establish protocol connection %v", err) default: } if atomic.LoadUint32(&peer1.serving) == 1 && atomic.LoadUint32(&peer2.serving) == 1 { @@ -473,7 +473,7 @@ func (client *testClient) newRawPeer(t *testing.T, name string, version int, rec select { case <-client.handler.closeCh: errCh <- p2p.DiscQuitting - case errCh <- client.handler.handle(peer): + case errCh <- client.handler.handle(peer, false): } }() tp := &testPeer{ @@ -623,7 +623,7 @@ func newClientServerEnv(t *testing.T, config testnetConfig) (*testServer, *testC if config.connect { done := make(chan struct{}) client.syncEnd = func(_ *types.Header) { close(done) } - cpeer, speer, err = newTestPeerPair("peer", config.protocol, server, client) + cpeer, speer, err = newTestPeerPair("peer", config.protocol, server, client, false) if err != nil { t.Fatalf("Failed to connect testing peers %v", err) } diff --git a/les/ulc_test.go b/les/ulc_test.go index d7308fa59..ecef58d97 100644 --- a/les/ulc_test.go +++ b/les/ulc_test.go @@ -20,6 +20,7 @@ import ( "crypto/rand" "fmt" "net" + "sync/atomic" "testing" "time" @@ -65,7 +66,7 @@ func testULCAnnounceThreshold(t *testing.T, protocol int) { // Connect all servers. for i := 0; i < len(servers); i++ { - connect(servers[i].handler, nodes[i].ID(), c.handler, protocol) + connect(servers[i].handler, nodes[i].ID(), c.handler, protocol, false) } for i := 0; i < len(servers); i++ { for j := 0; j < testcase.height[i]; j++ { @@ -86,7 +87,7 @@ func testULCAnnounceThreshold(t *testing.T, protocol int) { } } -func connect(server *serverHandler, serverId enode.ID, client *clientHandler, protocol int) (*serverPeer, *clientPeer, error) { +func connect(server *serverHandler, serverId enode.ID, client *clientHandler, protocol int, noInitAnnounce bool) (*serverPeer, *clientPeer, error) { // Create a message pipe to communicate through app, net := p2p.MsgPipe() @@ -110,16 +111,22 @@ func connect(server *serverHandler, serverId enode.ID, client *clientHandler, pr select { case <-client.closeCh: errc1 <- p2p.DiscQuitting - case errc1 <- client.handle(peer1): + case errc1 <- client.handle(peer1, noInitAnnounce): } }() - - select { - case <-time.After(time.Millisecond * 100): - case err := <-errc1: - return nil, nil, fmt.Errorf("peerLight handshake error: %v", err) - case err := <-errc2: - return nil, nil, fmt.Errorf("peerFull handshake error: %v", err) + // Ensure the connection is established or exits when any error occurs + for { + select { + case err := <-errc1: + return nil, nil, fmt.Errorf("failed to establish protocol connection %v", err) + case err := <-errc2: + return nil, nil, fmt.Errorf("failed to establish protocol connection %v", err) + default: + } + if atomic.LoadUint32(&peer1.serving) == 1 && atomic.LoadUint32(&peer2.serving) == 1 { + break + } + time.Sleep(50 * time.Millisecond) } return peer1, peer2, nil } From 45f34430fd5fc5a93b666eba90f2c76bd3f770f3 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 25 Aug 2021 13:21:09 +0200 Subject: [PATCH 17/73] build: upgrade to golangci-lint v1.42.0 (#23461) The new linter version is built with go 1.17 and thus includes the go vet check for mismatched +build and go:build lines. Fortunately, no new warnings are reported with this update. --- build/checksums.txt | 37 ++++++++++++++++++++----------------- build/ci.go | 2 +- 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/build/checksums.txt b/build/checksums.txt index e667b30ce..6df097b88 100644 --- a/build/checksums.txt +++ b/build/checksums.txt @@ -14,20 +14,23 @@ ccdd2b76de1941b60734408fda0d750aaa69330d8a07430eed4c56bdb3502f6f go1.16.4.freeb 80cfac566e344096a8df8f37bbd21f89e76a6fbe601406565d71a87a665fc125 go1.16.4.linux-ppc64le.tar.gz d6431881b3573dc29ecc24fbeab5e5ec25d8c9273aa543769c86a1a3bbac1ddf go1.16.4.linux-s390x.tar.gz -7e9a47ab540aa3e8472fbf8120d28bed3b9d9cf625b955818e8bc69628d7187c golangci-lint-1.39.0-darwin-amd64.tar.gz -574daa2c9c299b01672a6daeb1873b5f12e413cdb6dc0e30f2ff163956778064 golangci-lint-1.39.0-darwin-arm64.tar.gz -6225f7014987324ab78e9b511f294e3f25be013728283c33918c67c8576d543e golangci-lint-1.39.0-freebsd-386.tar.gz -6b3e76e1e5eaf0159411c8e2727f8d533989d3bb19f10e9caa6e0b9619ee267d golangci-lint-1.39.0-freebsd-amd64.tar.gz -a301cacfff87ed9b00313d95278533c25a4527a06b040a17d969b4b7e1b8a90d golangci-lint-1.39.0-freebsd-armv7.tar.gz -25bfd96a29c3112f508d5e4fc860dbad7afce657233c343acfa20715717d51e7 golangci-lint-1.39.0-freebsd-armv6.tar.gz -9687e4ff15545cfc722b0e46107a94195166a505023b48a316579af25ad09505 golangci-lint-1.39.0-linux-armv7.tar.gz -a7fa7ab2bfc99cbe5e5bcbf5684f5a997f920afbbe2f253d2feb1001d5e3c8b3 golangci-lint-1.39.0-linux-armv6.tar.gz -c8f9634115beddb4ed9129c1f7ecd4c97c99d07aeef33e3707234097eeb51b7b golangci-lint-1.39.0-linux-mips64le.tar.gz -d1234c213b74751f1af413302dde0e9a6d4d29aecef034af7abb07dc1b6e887f golangci-lint-1.39.0-linux-arm64.tar.gz -df25d9267168323b163147acb823ab0215a8a3bb6898a4a9320afdfedde66817 golangci-lint-1.39.0-linux-386.tar.gz -1767e75fba357b7651b1a796d38453558f371c60af805505ec99e166908c04b5 golangci-lint-1.39.0-linux-ppc64le.tar.gz -25fd75bf3186b3d930ecae10185689968fd18fd8fa6f9f555d6beb04348c20f6 golangci-lint-1.39.0-linux-s390x.tar.gz -3a73aa7468087caa62673c8adea99b4e4dff846dc72707222db85f8679b40cbf golangci-lint-1.39.0-linux-amd64.tar.gz -578caceccf81739bda67dbfec52816709d03608c6878888ecdc0e186a094a41b golangci-lint-1.39.0-linux-mips64.tar.gz -494b66ba0e32c8ddf6c4f6b1d05729b110900f6017eda943057e43598c17d7a8 golangci-lint-1.39.0-windows-386.zip -52ec2e13a3cbb47147244dff8cfc35103563deb76e0459133058086fc35fb2c7 golangci-lint-1.39.0-windows-amd64.zip +d4bd25b9814eeaa2134197dd2c7671bb791eae786d42010d9d788af20dee4bfa golangci-lint-1.42.0-darwin-amd64.tar.gz +e56859c04a2ad5390c6a497b1acb1cc9329ecb1010260c6faae9b5a4c35b35ea golangci-lint-1.42.0-darwin-arm64.tar.gz +14d912a3fa856830339472fc4dc341933adf15f37bdb7130bbbfcf960ecf4809 golangci-lint-1.42.0-freebsd-386.tar.gz +337257fccc9baeb5ee1cd7e70c153e9d9f59d3afde46d631659500048afbdf80 golangci-lint-1.42.0-freebsd-amd64.tar.gz +6debcc266b629359fdd8eef4f4abb05a621604079d27016265afb5b4593b0eff golangci-lint-1.42.0-freebsd-armv6.tar.gz +878f0e190169db2ce9dde8cefbd99adc4fe28b90b68686bbfcfcc2085e6d693e golangci-lint-1.42.0-freebsd-armv7.tar.gz +42c78e31faf62b225363eff1b1d2aa74f9dbcb75686c8914aa3e90d6af65cece golangci-lint-1.42.0-linux-386.tar.gz +6937f62f8e2329e94822dc11c10b871ace5557ae1fcc4ee2f9980cd6aecbc159 golangci-lint-1.42.0-linux-amd64.tar.gz +2cf8d23d96cd854a537b355dab2962b960b88a06b615232599f066afd233f246 golangci-lint-1.42.0-linux-arm64.tar.gz +08b003d1ed61367473886defc957af5301066e62338e5d96a319c34dadc4c1d1 golangci-lint-1.42.0-linux-armv6.tar.gz +c7c00ec4845e806a1f32685f5b150219e180bd6d6a9d584be8d27f0c41d7a1bf golangci-lint-1.42.0-linux-armv7.tar.gz +3650fcf29eb3d8ee326d77791a896b15259eb2d5bf77437dc72e7efe5af6bd40 golangci-lint-1.42.0-linux-mips64.tar.gz +f51ae003fdbca4fef78ba73e2eb736a939c8eaa178cd452234213b489da5a420 golangci-lint-1.42.0-linux-mips64le.tar.gz +1b0bb7b8b22cc4ea7da44fd5ad5faaf6111d0677e01cc6f961b62a96537de2c6 golangci-lint-1.42.0-linux-ppc64le.tar.gz +8cb56927eb75e572450efbe0ff0f9cf3f56dc9faa81d9e8d30d6559fc1d06e6d golangci-lint-1.42.0-linux-riscv64.tar.gz +5ac41cd31825a176b21505a371a7b307cd9cdf17df0f35bbb3bf1466f9356ccc golangci-lint-1.42.0-linux-s390x.tar.gz +e1cebd2af621ac4b64c20937df92c3819264f2174c92f51e196db1e64ae097e0 golangci-lint-1.42.0-windows-386.zip +7e70fcde8e87a17cae0455df07d257ebc86669f3968d568e12727fa24bbe9883 golangci-lint-1.42.0-windows-amd64.zip +59da7ce1bda432616bfc28ae663e52c3675adee8d9bf5959fafd657c159576ab golangci-lint-1.42.0-windows-armv6.zip +65f62dda937bfcede0326ac77abe947ce1548931e6e13298ca036cb31f224db5 golangci-lint-1.42.0-windows-armv7.zip diff --git a/build/ci.go b/build/ci.go index d7d2ce72e..540cddd53 100644 --- a/build/ci.go +++ b/build/ci.go @@ -330,7 +330,7 @@ func doLint(cmdline []string) { // downloadLinter downloads and unpacks golangci-lint. func downloadLinter(cachedir string) string { - const version = "1.39.0" + const version = "1.42.0" csdb := build.MustLoadChecksums("build/checksums.txt") base := fmt.Sprintf("golangci-lint-%s-%s-%s", version, runtime.GOOS, runtime.GOARCH) From efee85378ef426ab1dffa2e9045a60ea411415e1 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 25 Aug 2021 14:22:33 +0200 Subject: [PATCH 18/73] build, .travis.yml: upgrade to Go 1.17 (#23464) --- .travis.yml | 24 ++++++++++++------------ build/checksums.txt | 27 ++++++++++++++------------- build/ci.go | 2 +- 3 files changed, 27 insertions(+), 26 deletions(-) diff --git a/.travis.yml b/.travis.yml index a7db711cc..f1018cc11 100644 --- a/.travis.yml +++ b/.travis.yml @@ -5,7 +5,7 @@ jobs: allow_failures: - stage: build os: osx - go: 1.15.x + go: 1.17.x env: - azure-osx - azure-ios @@ -16,7 +16,7 @@ jobs: - stage: lint os: linux dist: bionic - go: 1.16.x + go: 1.17.x env: - lint git: @@ -31,7 +31,7 @@ jobs: os: linux arch: amd64 dist: bionic - go: 1.16.x + go: 1.17.x env: - docker services: @@ -48,7 +48,7 @@ jobs: os: linux arch: arm64 dist: bionic - go: 1.16.x + go: 1.17.x env: - docker services: @@ -65,7 +65,7 @@ jobs: if: type = push os: linux dist: bionic - go: 1.16.x + go: 1.17.x env: - ubuntu-ppa - GO111MODULE=on @@ -90,7 +90,7 @@ jobs: os: linux dist: bionic sudo: required - go: 1.16.x + go: 1.17.x env: - azure-linux - GO111MODULE=on @@ -127,7 +127,7 @@ jobs: dist: bionic services: - docker - go: 1.16.x + go: 1.17.x env: - azure-linux-mips - GO111MODULE=on @@ -192,7 +192,7 @@ jobs: - stage: build if: type = push os: osx - go: 1.16.x + go: 1.17.x env: - azure-osx - azure-ios @@ -224,7 +224,7 @@ jobs: os: linux arch: amd64 dist: bionic - go: 1.16.x + go: 1.17.x env: - GO111MODULE=on script: @@ -235,7 +235,7 @@ jobs: os: linux arch: arm64 dist: bionic - go: 1.16.x + go: 1.17.x env: - GO111MODULE=on script: @@ -244,7 +244,7 @@ jobs: - stage: build os: linux dist: bionic - go: 1.15.x + go: 1.16.x env: - GO111MODULE=on script: @@ -255,7 +255,7 @@ jobs: if: type = cron os: linux dist: bionic - go: 1.16.x + go: 1.17.x env: - azure-purge - GO111MODULE=on diff --git a/build/checksums.txt b/build/checksums.txt index 6df097b88..686e1604b 100644 --- a/build/checksums.txt +++ b/build/checksums.txt @@ -1,18 +1,19 @@ # This file contains sha256 checksums of optional build dependencies. -ae4f6b6e2a1677d31817984655a762074b5356da50fb58722b99104870d43503 go1.16.4.src.tar.gz -18fe94775763db3878717393b6d41371b0b45206055e49b3838328120c977d13 go1.16.4.darwin-amd64.tar.gz -cb6b972cc42e669f3585c648198cd5b6f6d7a0811d413ad64b50c02ba06ccc3a go1.16.4.darwin-arm64.tar.gz -cd1b146ef6e9006f27dd99e9687773e7fef30e8c985b7d41bff33e955a3bb53a go1.16.4.linux-386.tar.gz -7154e88f5a8047aad4b80ebace58a059e36e7e2e4eb3b383127a28c711b4ff59 go1.16.4.linux-amd64.tar.gz -8b18eb05ddda2652d69ab1b1dd1f40dd731799f43c6a58b512ad01ae5b5bba21 go1.16.4.linux-arm64.tar.gz -a53391a800ddec749ee90d38992babb27b95cfb864027350c737b9aa8e069494 go1.16.4.linux-armv6l.tar.gz -e75c0b114a09eb5499874162b208931dc260de0fedaeedac8621bf263c974605 go1.16.4.windows-386.zip -d40139b7ade8a3008e3240a6f86fe8f899a9c465c917e11dac8758af216f5eb0 go1.16.4.windows-amd64.zip -7cf2bc8a175d6d656861165bfc554f92dc78d2abf5afe5631db3579555d97409 go1.16.4.freebsd-386.tar.gz -ccdd2b76de1941b60734408fda0d750aaa69330d8a07430eed4c56bdb3502f6f go1.16.4.freebsd-amd64.tar.gz -80cfac566e344096a8df8f37bbd21f89e76a6fbe601406565d71a87a665fc125 go1.16.4.linux-ppc64le.tar.gz -d6431881b3573dc29ecc24fbeab5e5ec25d8c9273aa543769c86a1a3bbac1ddf go1.16.4.linux-s390x.tar.gz +3a70e5055509f347c0fb831ca07a2bf3b531068f349b14a3c652e9b5b67beb5d go1.17.src.tar.gz +355bd544ce08d7d484d9d7de05a71b5c6f5bc10aa4b316688c2192aeb3dacfd1 go1.17.darwin-amd64.tar.gz +da4e3e3c194bf9eed081de8842a157120ef44a7a8d7c820201adae7b0e28b20b go1.17.darwin-arm64.tar.gz +6819a7a11b8351d5d5768f2fff666abde97577602394f132cb7f85b3a7151f05 go1.17.freebsd-386.tar.gz +15c184c83d99441d719da201b26256455eee85a808747c404b4183e9aa6c64b4 go1.17.freebsd-amd64.tar.gz +c19e3227a6ac6329db91d1af77bbf239ccd760a259c16e6b9c932d527ff14848 go1.17.linux-386.tar.gz +6bf89fc4f5ad763871cf7eac80a2d594492de7a818303283f1366a7f6a30372d go1.17.linux-amd64.tar.gz +01a9af009ada22122d3fcb9816049c1d21842524b38ef5d5a0e2ee4b26d7c3e7 go1.17.linux-arm64.tar.gz +ae89d33f4e4acc222bdb04331933d5ece4ae71039812f6ccd7493cb3e8ddfb4e go1.17.linux-armv6l.tar.gz +ee84350114d532bf15f096198c675aafae9ff091dc4cc69eb49e1817ff94dbd7 go1.17.linux-ppc64le.tar.gz +a50aaecf054f393575f969a9105d5c6864dd91afc5287d772449033fbafcf7e3 go1.17.linux-s390x.tar.gz +c5afdd2ea4969f2b44637e913b04f7c15265d7beb60924a28063722670a52feb go1.17.windows-386.zip +2a18bd65583e221be8b9b7c2fbe3696c40f6e27c2df689bbdcc939d49651d151 go1.17.windows-amd64.zip +5256f92f643d9022394ddc84de5c74fe8660c2151daaa199b12e60e542d694ae go1.17.windows-arm64.zip d4bd25b9814eeaa2134197dd2c7671bb791eae786d42010d9d788af20dee4bfa golangci-lint-1.42.0-darwin-amd64.tar.gz e56859c04a2ad5390c6a497b1acb1cc9329ecb1010260c6faae9b5a4c35b35ea golangci-lint-1.42.0-darwin-arm64.tar.gz diff --git a/build/ci.go b/build/ci.go index 540cddd53..3de135120 100644 --- a/build/ci.go +++ b/build/ci.go @@ -153,7 +153,7 @@ var ( // This is the version of go that will be downloaded by // // go run ci.go install -dlgo - dlgoVersion = "1.16.4" + dlgoVersion = "1.17" ) var GOBIN, _ = filepath.Abs(filepath.Join("build", "bin")) From 84c51bc5ec9038db6ab28195384a6bd720a28af9 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Wed, 25 Aug 2021 15:04:26 +0200 Subject: [PATCH 19/73] cmd/evm: add difficulty calculation to t8n tool (#23353) This PR adds functionality to the evm t8n to calculate ethash difficulty. If the caller does not provide a currentDifficulty, but instead provides the parentTimestamp (well, semi-optional, will default to 0 if not given), and parentDifficulty, we can calculate it for him. The caller can also provide a parentUncleHash. In most, but not all cases, the parent uncle hash also affects the formula. If no such hash is provided (or, if the empty all-zero hash is provided), it's assumed that there were no uncles. --- cmd/evm/internal/t8ntool/execution.go | 70 ++++++++++++++++++-------- cmd/evm/internal/t8ntool/gen_stenv.go | 55 +++++++++++++------- cmd/evm/internal/t8ntool/transition.go | 14 ++++++ cmd/evm/testdata/14/alloc.json | 12 +++++ cmd/evm/testdata/14/env.json | 9 ++++ cmd/evm/testdata/14/env.uncles.json | 10 ++++ cmd/evm/testdata/14/readme.md | 41 +++++++++++++++ cmd/evm/testdata/14/txs.json | 1 + 8 files changed, 172 insertions(+), 40 deletions(-) create mode 100644 cmd/evm/testdata/14/alloc.json create mode 100644 cmd/evm/testdata/14/env.json create mode 100644 cmd/evm/testdata/14/env.uncles.json create mode 100644 cmd/evm/testdata/14/readme.md create mode 100644 cmd/evm/testdata/14/txs.json diff --git a/cmd/evm/internal/t8ntool/execution.go b/cmd/evm/internal/t8ntool/execution.go index 1ab2f001e..fae65767b 100644 --- a/cmd/evm/internal/t8ntool/execution.go +++ b/cmd/evm/internal/t8ntool/execution.go @@ -23,6 +23,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/math" + "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/consensus/misc" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" @@ -46,13 +47,14 @@ type Prestate struct { // ExecutionResult contains the execution status after running a state test, any // error that might have occurred and a dump of the final state if requested. type ExecutionResult struct { - StateRoot common.Hash `json:"stateRoot"` - TxRoot common.Hash `json:"txRoot"` - ReceiptRoot common.Hash `json:"receiptRoot"` - LogsHash common.Hash `json:"logsHash"` - Bloom types.Bloom `json:"logsBloom" gencodec:"required"` - Receipts types.Receipts `json:"receipts"` - Rejected []*rejectedTx `json:"rejected,omitempty"` + StateRoot common.Hash `json:"stateRoot"` + TxRoot common.Hash `json:"txRoot"` + ReceiptRoot common.Hash `json:"receiptRoot"` + LogsHash common.Hash `json:"logsHash"` + Bloom types.Bloom `json:"logsBloom" gencodec:"required"` + Receipts types.Receipts `json:"receipts"` + Rejected []*rejectedTx `json:"rejected,omitempty"` + Difficulty *math.HexOrDecimal256 `json:"currentDifficulty" gencodec:"required"` } type ommer struct { @@ -62,23 +64,28 @@ type ommer struct { //go:generate gencodec -type stEnv -field-override stEnvMarshaling -out gen_stenv.go type stEnv struct { - Coinbase common.Address `json:"currentCoinbase" gencodec:"required"` - Difficulty *big.Int `json:"currentDifficulty" gencodec:"required"` - GasLimit uint64 `json:"currentGasLimit" gencodec:"required"` - Number uint64 `json:"currentNumber" gencodec:"required"` - Timestamp uint64 `json:"currentTimestamp" gencodec:"required"` - BlockHashes map[math.HexOrDecimal64]common.Hash `json:"blockHashes,omitempty"` - Ommers []ommer `json:"ommers,omitempty"` - BaseFee *big.Int `json:"currentBaseFee,omitempty"` + Coinbase common.Address `json:"currentCoinbase" gencodec:"required"` + Difficulty *big.Int `json:"currentDifficulty"` + ParentDifficulty *big.Int `json:"parentDifficulty"` + GasLimit uint64 `json:"currentGasLimit" gencodec:"required"` + Number uint64 `json:"currentNumber" gencodec:"required"` + Timestamp uint64 `json:"currentTimestamp" gencodec:"required"` + ParentTimestamp uint64 `json:"parentTimestamp,omitempty"` + BlockHashes map[math.HexOrDecimal64]common.Hash `json:"blockHashes,omitempty"` + Ommers []ommer `json:"ommers,omitempty"` + BaseFee *big.Int `json:"currentBaseFee,omitempty"` + ParentUncleHash common.Hash `json:"parentUncleHash"` } type stEnvMarshaling struct { - Coinbase common.UnprefixedAddress - Difficulty *math.HexOrDecimal256 - GasLimit math.HexOrDecimal64 - Number math.HexOrDecimal64 - Timestamp math.HexOrDecimal64 - BaseFee *math.HexOrDecimal256 + Coinbase common.UnprefixedAddress + Difficulty *math.HexOrDecimal256 + ParentDifficulty *math.HexOrDecimal256 + GasLimit math.HexOrDecimal64 + Number math.HexOrDecimal64 + Timestamp math.HexOrDecimal64 + ParentTimestamp math.HexOrDecimal64 + BaseFee *math.HexOrDecimal256 } type rejectedTx struct { @@ -247,6 +254,7 @@ func (pre *Prestate) Apply(vmConfig vm.Config, chainConfig *params.ChainConfig, LogsHash: rlpHash(statedb.Logs()), Receipts: receipts, Rejected: rejectedTxs, + Difficulty: (*math.HexOrDecimal256)(vmContext.Difficulty), } return statedb, execRs, nil } @@ -274,3 +282,23 @@ func rlpHash(x interface{}) (h common.Hash) { hw.Sum(h[:0]) return h } + +// calcDifficulty is based on ethash.CalcDifficulty. This method is used in case +// the caller does not provide an explicit difficulty, but instead provides only +// parent timestamp + difficulty. +// Note: this method only works for ethash engine. +func calcDifficulty(config *params.ChainConfig, number, currentTime, parentTime uint64, + parentDifficulty *big.Int, parentUncleHash common.Hash) *big.Int { + uncleHash := parentUncleHash + if uncleHash == (common.Hash{}) { + uncleHash = types.EmptyUncleHash + } + parent := &types.Header{ + ParentHash: common.Hash{}, + UncleHash: uncleHash, + Difficulty: parentDifficulty, + Number: new(big.Int).SetUint64(number - 1), + Time: parentTime, + } + return ethash.CalcDifficulty(config, currentTime, parent) +} diff --git a/cmd/evm/internal/t8ntool/gen_stenv.go b/cmd/evm/internal/t8ntool/gen_stenv.go index c7f079c02..1bb3c6a46 100644 --- a/cmd/evm/internal/t8ntool/gen_stenv.go +++ b/cmd/evm/internal/t8ntool/gen_stenv.go @@ -16,38 +16,47 @@ var _ = (*stEnvMarshaling)(nil) // MarshalJSON marshals as JSON. func (s stEnv) MarshalJSON() ([]byte, error) { type stEnv struct { - Coinbase common.UnprefixedAddress `json:"currentCoinbase" gencodec:"required"` - Difficulty *math.HexOrDecimal256 `json:"currentDifficulty" gencodec:"required"` - GasLimit math.HexOrDecimal64 `json:"currentGasLimit" gencodec:"required"` - Number math.HexOrDecimal64 `json:"currentNumber" gencodec:"required"` - Timestamp math.HexOrDecimal64 `json:"currentTimestamp" gencodec:"required"` - BlockHashes map[math.HexOrDecimal64]common.Hash `json:"blockHashes,omitempty"` - Ommers []ommer `json:"ommers,omitempty"` - BaseFee *math.HexOrDecimal256 `json:"currentBaseFee,omitempty"` + Coinbase common.UnprefixedAddress `json:"currentCoinbase" gencodec:"required"` + Difficulty *math.HexOrDecimal256 `json:"currentDifficulty"` + ParentDifficulty *math.HexOrDecimal256 `json:"parentDifficulty"` + GasLimit math.HexOrDecimal64 `json:"currentGasLimit" gencodec:"required"` + Number math.HexOrDecimal64 `json:"currentNumber" gencodec:"required"` + Timestamp math.HexOrDecimal64 `json:"currentTimestamp" gencodec:"required"` + ParentTimestamp math.HexOrDecimal64 `json:"parentTimestamp,omitempty"` + BlockHashes map[math.HexOrDecimal64]common.Hash `json:"blockHashes,omitempty"` + Ommers []ommer `json:"ommers,omitempty"` + BaseFee *math.HexOrDecimal256 `json:"currentBaseFee,omitempty"` + ParentUncleHash common.Hash `json:"parentUncleHash"` } var enc stEnv enc.Coinbase = common.UnprefixedAddress(s.Coinbase) enc.Difficulty = (*math.HexOrDecimal256)(s.Difficulty) + enc.ParentDifficulty = (*math.HexOrDecimal256)(s.ParentDifficulty) enc.GasLimit = math.HexOrDecimal64(s.GasLimit) enc.Number = math.HexOrDecimal64(s.Number) enc.Timestamp = math.HexOrDecimal64(s.Timestamp) + enc.ParentTimestamp = math.HexOrDecimal64(s.ParentTimestamp) enc.BlockHashes = s.BlockHashes enc.Ommers = s.Ommers enc.BaseFee = (*math.HexOrDecimal256)(s.BaseFee) + enc.ParentUncleHash = s.ParentUncleHash return json.Marshal(&enc) } // UnmarshalJSON unmarshals from JSON. func (s *stEnv) UnmarshalJSON(input []byte) error { type stEnv struct { - Coinbase *common.UnprefixedAddress `json:"currentCoinbase" gencodec:"required"` - Difficulty *math.HexOrDecimal256 `json:"currentDifficulty" gencodec:"required"` - GasLimit *math.HexOrDecimal64 `json:"currentGasLimit" gencodec:"required"` - Number *math.HexOrDecimal64 `json:"currentNumber" gencodec:"required"` - Timestamp *math.HexOrDecimal64 `json:"currentTimestamp" gencodec:"required"` - BlockHashes map[math.HexOrDecimal64]common.Hash `json:"blockHashes,omitempty"` - Ommers []ommer `json:"ommers,omitempty"` - BaseFee *math.HexOrDecimal256 `json:"currentBaseFee,omitempty"` + Coinbase *common.UnprefixedAddress `json:"currentCoinbase" gencodec:"required"` + Difficulty *math.HexOrDecimal256 `json:"currentDifficulty"` + ParentDifficulty *math.HexOrDecimal256 `json:"parentDifficulty"` + GasLimit *math.HexOrDecimal64 `json:"currentGasLimit" gencodec:"required"` + Number *math.HexOrDecimal64 `json:"currentNumber" gencodec:"required"` + Timestamp *math.HexOrDecimal64 `json:"currentTimestamp" gencodec:"required"` + ParentTimestamp *math.HexOrDecimal64 `json:"parentTimestamp,omitempty"` + BlockHashes map[math.HexOrDecimal64]common.Hash `json:"blockHashes,omitempty"` + Ommers []ommer `json:"ommers,omitempty"` + BaseFee *math.HexOrDecimal256 `json:"currentBaseFee,omitempty"` + ParentUncleHash *common.Hash `json:"parentUncleHash"` } var dec stEnv if err := json.Unmarshal(input, &dec); err != nil { @@ -57,10 +66,12 @@ func (s *stEnv) UnmarshalJSON(input []byte) error { return errors.New("missing required field 'currentCoinbase' for stEnv") } s.Coinbase = common.Address(*dec.Coinbase) - if dec.Difficulty == nil { - return errors.New("missing required field 'currentDifficulty' for stEnv") + if dec.Difficulty != nil { + s.Difficulty = (*big.Int)(dec.Difficulty) + } + if dec.ParentDifficulty != nil { + s.ParentDifficulty = (*big.Int)(dec.ParentDifficulty) } - s.Difficulty = (*big.Int)(dec.Difficulty) if dec.GasLimit == nil { return errors.New("missing required field 'currentGasLimit' for stEnv") } @@ -73,6 +84,9 @@ func (s *stEnv) UnmarshalJSON(input []byte) error { return errors.New("missing required field 'currentTimestamp' for stEnv") } s.Timestamp = uint64(*dec.Timestamp) + if dec.ParentTimestamp != nil { + s.ParentTimestamp = uint64(*dec.ParentTimestamp) + } if dec.BlockHashes != nil { s.BlockHashes = dec.BlockHashes } @@ -82,5 +96,8 @@ func (s *stEnv) UnmarshalJSON(input []byte) error { if dec.BaseFee != nil { s.BaseFee = (*big.Int)(dec.BaseFee) } + if dec.ParentUncleHash != nil { + s.ParentUncleHash = *dec.ParentUncleHash + } return nil } diff --git a/cmd/evm/internal/t8ntool/transition.go b/cmd/evm/internal/t8ntool/transition.go index 8334aa01d..9fdbbe29c 100644 --- a/cmd/evm/internal/t8ntool/transition.go +++ b/cmd/evm/internal/t8ntool/transition.go @@ -252,6 +252,20 @@ func Main(ctx *cli.Context) error { return NewError(ErrorVMConfig, errors.New("EIP-1559 config but missing 'currentBaseFee' in env section")) } } + if env := prestate.Env; env.Difficulty == nil { + // If difficulty was not provided by caller, we need to calculate it. + switch { + case env.ParentDifficulty == nil: + return NewError(ErrorVMConfig, errors.New("currentDifficulty was not provided, and cannot be calculated due to missing parentDifficulty")) + case env.Number == 0: + return NewError(ErrorVMConfig, errors.New("currentDifficulty needs to be provided for block number 0")) + case env.Timestamp <= env.ParentTimestamp: + return NewError(ErrorVMConfig, fmt.Errorf("currentDifficulty cannot be calculated -- currentTime (%d) needs to be after parent time (%d)", + env.Timestamp, env.ParentTimestamp)) + } + prestate.Env.Difficulty = calcDifficulty(chainConfig, env.Number, env.Timestamp, + env.ParentTimestamp, env.ParentDifficulty, env.ParentUncleHash) + } // Run the test and aggregate the result s, result, err := prestate.Apply(vmConfig, chainConfig, txs, ctx.Int64(RewardFlag.Name), getTracer) if err != nil { diff --git a/cmd/evm/testdata/14/alloc.json b/cmd/evm/testdata/14/alloc.json new file mode 100644 index 000000000..cef1a25ff --- /dev/null +++ b/cmd/evm/testdata/14/alloc.json @@ -0,0 +1,12 @@ +{ + "a94f5374fce5edbc8e2a8697c15331677e6ebf0b": { + "balance": "0x5ffd4878be161d74", + "code": "0x", + "nonce": "0xac", + "storage": {} + }, + "0x8a8eafb1cf62bfbeb1741769dae1a9dd47996192":{ + "balance": "0xfeedbead", + "nonce" : "0x00" + } +} \ No newline at end of file diff --git a/cmd/evm/testdata/14/env.json b/cmd/evm/testdata/14/env.json new file mode 100644 index 000000000..6b7457ffd --- /dev/null +++ b/cmd/evm/testdata/14/env.json @@ -0,0 +1,9 @@ +{ + "currentCoinbase": "0xc94f5374fce5edbc8e2a8697c15331677e6ebf0b", + "currentGasLimit": "0x750a163df65e8a", + "currentBaseFee": "0x500", + "currentNumber": "12800000", + "currentTimestamp": "10015", + "parentTimestamp" : "99999", + "parentDifficulty" : "0x2000000000000" +} diff --git a/cmd/evm/testdata/14/env.uncles.json b/cmd/evm/testdata/14/env.uncles.json new file mode 100644 index 000000000..9c2149b39 --- /dev/null +++ b/cmd/evm/testdata/14/env.uncles.json @@ -0,0 +1,10 @@ +{ + "currentCoinbase": "0xc94f5374fce5edbc8e2a8697c15331677e6ebf0b", + "currentGasLimit": "0x750a163df65e8a", + "currentBaseFee": "0x500", + "currentNumber": "12800000", + "currentTimestamp": "10035", + "parentTimestamp" : "99999", + "parentDifficulty" : "0x2000000000000", + "parentUncleHash" : "0x000000000000000000000000000000000000000000000000000000000000beef" +} diff --git a/cmd/evm/testdata/14/readme.md b/cmd/evm/testdata/14/readme.md new file mode 100644 index 000000000..ce31abb80 --- /dev/null +++ b/cmd/evm/testdata/14/readme.md @@ -0,0 +1,41 @@ +## Difficulty calculation + +This test shows how the `evm t8n` can be used to calculate the (ethash) difficulty, if none is provided by the caller. + +Calculating it (with an empty set of txs) using `London` rules (and no provided unclehash for the parent block): +``` +[user@work evm]$ ./evm t8n --input.alloc=./testdata/14/alloc.json --input.txs=./testdata/14/txs.json --input.env=./testdata/14/env.json --output.result=stdout --state.fork=London +INFO [08-08|17:35:46.876] Trie dumping started root=6f0588..7f4bdc +INFO [08-08|17:35:46.876] Trie dumping complete accounts=2 elapsed="89.313µs" +INFO [08-08|17:35:46.877] Wrote file file=alloc.json +{ + "result": { + "stateRoot": "0x6f058887ca01549716789c380ede95aecc510e6d1fdc4dbf67d053c7c07f4bdc", + "txRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "receiptRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "logsHash": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "receipts": [], + "currentDifficulty": 3311729559732224 + } +} +``` +Same thing, but this time providing a non-empty (and non-`emptyKeccak`) unclehash, which leads to a slightly different result: +``` +[user@work evm]$ ./evm t8n --input.alloc=./testdata/14/alloc.json --input.txs=./testdata/14/txs.json --input.env=./testdata/14/env.uncles.json --output.result=stdout --state.fork=London +INFO [08-08|17:35:49.232] Trie dumping started root=6f0588..7f4bdc +INFO [08-08|17:35:49.232] Trie dumping complete accounts=2 elapsed="83.069µs" +INFO [08-08|17:35:49.233] Wrote file file=alloc.json +{ + "result": { + "stateRoot": "0x6f058887ca01549716789c380ede95aecc510e6d1fdc4dbf67d053c7c07f4bdc", + "txRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "receiptRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "logsHash": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "receipts": [], + "currentDifficulty": 3311179803918336 + } +} +``` + diff --git a/cmd/evm/testdata/14/txs.json b/cmd/evm/testdata/14/txs.json new file mode 100644 index 000000000..fe51488c7 --- /dev/null +++ b/cmd/evm/testdata/14/txs.json @@ -0,0 +1 @@ +[] From 3784e15106c8502b0a612d932be2e995fd182ea6 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Wed, 25 Aug 2021 15:07:58 +0200 Subject: [PATCH 20/73] tests: avoid performance tests during CI runs (#23304) --- tests/state_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/state_test.go b/tests/state_test.go index c2ca0e8d6..a7d9653a3 100644 --- a/tests/state_test.go +++ b/tests/state_test.go @@ -42,6 +42,7 @@ func TestState(t *testing.T) { // Very time consuming st.skipLoad(`^stTimeConsuming/`) + st.skipLoad(`.*vmPerformance/loop.*`) // Uses 1GB RAM per tested fork st.skipLoad(`^stStaticCall/static_Call1MB`) From d16214228f40079af8db6752ca5f51b2022878c2 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 25 Aug 2021 15:18:31 +0200 Subject: [PATCH 21/73] Dockerfile: upgrade to Go 1.17 (#23465) --- Dockerfile | 2 +- Dockerfile.alltools | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index e76c5765b..7badbc132 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ ARG VERSION="" ARG BUILDNUM="" # Build Geth in a stock Go builder container -FROM golang:1.16-alpine as builder +FROM golang:1.17-alpine as builder RUN apk add --no-cache gcc musl-dev linux-headers git diff --git a/Dockerfile.alltools b/Dockerfile.alltools index 71f63b7a4..3ae5377e4 100644 --- a/Dockerfile.alltools +++ b/Dockerfile.alltools @@ -4,7 +4,7 @@ ARG VERSION="" ARG BUILDNUM="" # Build Geth in a stock Go builder container -FROM golang:1.16-alpine as builder +FROM golang:1.17-alpine as builder RUN apk add --no-cache gcc musl-dev linux-headers git From 887902ea4d7ee77118ce803e05085bd9055aa46d Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Wed, 25 Aug 2021 17:33:09 +0200 Subject: [PATCH 22/73] crypto/cloudflare/bn256: fix in-place addition and unmarshalling (#23419) --- crypto/bn256/cloudflare/bn256_test.go | 13 +++++++++++++ crypto/bn256/cloudflare/curve.go | 6 +++--- crypto/bn256/cloudflare/gfp.go | 1 + crypto/bn256/cloudflare/twist.go | 6 +++--- 4 files changed, 20 insertions(+), 6 deletions(-) diff --git a/crypto/bn256/cloudflare/bn256_test.go b/crypto/bn256/cloudflare/bn256_test.go index 0c8016d86..481e2f78c 100644 --- a/crypto/bn256/cloudflare/bn256_test.go +++ b/crypto/bn256/cloudflare/bn256_test.go @@ -92,6 +92,19 @@ func TestTripartiteDiffieHellman(t *testing.T) { } } +func TestG2SelfAddition(t *testing.T) { + s, _ := rand.Int(rand.Reader, Order) + p := new(G2).ScalarBaseMult(s) + + if !p.p.IsOnCurve() { + t.Fatal("p isn't on curve") + } + m := p.Add(p, p).Marshal() + if _, err := p.Unmarshal(m); err != nil { + t.Fatalf("p.Add(p, p) ∉ Gâ‚‚: %v", err) + } +} + func BenchmarkG1(b *testing.B) { x, _ := rand.Int(rand.Reader, Order) b.ResetTimer() diff --git a/crypto/bn256/cloudflare/curve.go b/crypto/bn256/cloudflare/curve.go index 18e9b38f3..16f0489e3 100644 --- a/crypto/bn256/cloudflare/curve.go +++ b/crypto/bn256/cloudflare/curve.go @@ -171,15 +171,15 @@ func (c *curvePoint) Double(a *curvePoint) { gfpAdd(t, d, d) gfpSub(&c.x, f, t) + gfpMul(&c.z, &a.y, &a.z) + gfpAdd(&c.z, &c.z, &c.z) + gfpAdd(t, C, C) gfpAdd(t2, t, t) gfpAdd(t, t2, t2) gfpSub(&c.y, d, &c.x) gfpMul(t2, e, &c.y) gfpSub(&c.y, t2, t) - - gfpMul(t, &a.y, &a.z) - gfpAdd(&c.z, t, t) } func (c *curvePoint) Mul(a *curvePoint, scalar *big.Int) { diff --git a/crypto/bn256/cloudflare/gfp.go b/crypto/bn256/cloudflare/gfp.go index e8e84e7b3..b15e1697e 100644 --- a/crypto/bn256/cloudflare/gfp.go +++ b/crypto/bn256/cloudflare/gfp.go @@ -61,6 +61,7 @@ func (e *gfP) Marshal(out []byte) { func (e *gfP) Unmarshal(in []byte) error { // Unmarshal the bytes into little endian form for w := uint(0); w < 4; w++ { + e[3-w] = 0 for b := uint(0); b < 8; b++ { e[3-w] += uint64(in[8*w+b]) << (56 - 8*b) } diff --git a/crypto/bn256/cloudflare/twist.go b/crypto/bn256/cloudflare/twist.go index 0c2f80d4e..2c7a69a4d 100644 --- a/crypto/bn256/cloudflare/twist.go +++ b/crypto/bn256/cloudflare/twist.go @@ -150,15 +150,15 @@ func (c *twistPoint) Double(a *twistPoint) { t.Add(d, d) c.x.Sub(f, t) + c.z.Mul(&a.y, &a.z) + c.z.Add(&c.z, &c.z) + t.Add(C, C) t2.Add(t, t) t.Add(t2, t2) c.y.Sub(d, &c.x) t2.Mul(e, &c.y) c.y.Sub(t2, t) - - t.Mul(&a.y, &a.z) - c.z.Add(t, t) } func (c *twistPoint) Mul(a *twistPoint, scalar *big.Int) { From 8a134014b4b370b4a3632e32a2fc8e84ee2b6947 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 25 Aug 2021 18:46:29 +0200 Subject: [PATCH 23/73] all: add go:build lines (#23468) Generated by go1.17 fmt ./... --- accounts/keystore/watch.go | 1 + accounts/keystore/watch_fallback.go | 1 + cmd/utils/diskusage.go | 1 + cmd/utils/diskusage_openbsd.go | 1 + common/fdlimit/fdlimit_bsd.go | 1 + common/fdlimit/fdlimit_unix.go | 1 + core/mkalloc.go | 1 + crypto/blake2b/blake2bAVX2_amd64.go | 1 + crypto/blake2b/blake2b_amd64.go | 1 + crypto/blake2b/blake2b_f_fuzz.go | 1 + crypto/blake2b/blake2b_ref.go | 1 + crypto/blake2b/register.go | 1 + crypto/bls12381/arithmetic_decl.go | 1 + crypto/bls12381/arithmetic_fallback.go | 1 + crypto/bls12381/arithmetic_x86_adx.go | 1 + crypto/bls12381/arithmetic_x86_noadx.go | 1 + crypto/bn256/bn256_fast.go | 1 + crypto/bn256/bn256_slow.go | 1 + crypto/bn256/cloudflare/gfp_decl.go | 1 + crypto/bn256/cloudflare/gfp_generic.go | 1 + crypto/secp256k1/dummy.go | 1 + crypto/secp256k1/panic_cb.go | 4 ++-- crypto/secp256k1/scalar_mult_cgo.go | 4 ++-- crypto/secp256k1/scalar_mult_nocgo.go | 1 + crypto/secp256k1/secp256.go | 4 ++-- crypto/signature_cgo.go | 1 + crypto/signature_nocgo.go | 1 + crypto/signify/signify_fuzz.go | 1 + ethdb/leveldb/leveldb.go | 1 + internal/debug/loudpanic.go | 1 + internal/debug/loudpanic_fallback.go | 1 + internal/debug/trace.go | 3 ++- internal/debug/trace_fallback.go | 3 ++- log/handler_go13.go | 1 + log/handler_go14.go | 1 + log/syslog.go | 1 + metrics/cpu_disabled.go | 1 + metrics/cpu_enabled.go | 1 + metrics/cputime_nop.go | 1 + metrics/cputime_unix.go | 1 + metrics/disk_nop.go | 1 + metrics/runtime_cgo.go | 1 + metrics/runtime_gccpufraction.go | 1 + metrics/runtime_no_cgo.go | 1 + metrics/runtime_no_gccpufraction.go | 1 + metrics/syslog.go | 1 + mobile/geth_android.go | 1 + mobile/geth_ios.go | 1 + mobile/geth_other.go | 1 + p2p/netutil/toobig_notwindows.go | 3 ++- p2p/netutil/toobig_windows.go | 3 ++- rlp/safe.go | 1 + rlp/unsafe.go | 1 + rpc/constants_unix.go | 1 + rpc/constants_unix_nocgo.go | 1 + rpc/ipc_js.go | 1 + rpc/ipc_unix.go | 1 + rpc/ipc_windows.go | 1 + tests/fuzzers/bls12381/bls12381_fuzz.go | 1 + 59 files changed, 66 insertions(+), 10 deletions(-) diff --git a/accounts/keystore/watch.go b/accounts/keystore/watch.go index d6ef53327..ad176040d 100644 --- a/accounts/keystore/watch.go +++ b/accounts/keystore/watch.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build (darwin && !ios && cgo) || freebsd || (linux && !arm64) || netbsd || solaris // +build darwin,!ios,cgo freebsd linux,!arm64 netbsd solaris package keystore diff --git a/accounts/keystore/watch_fallback.go b/accounts/keystore/watch_fallback.go index de0e87f8a..e40eca42f 100644 --- a/accounts/keystore/watch_fallback.go +++ b/accounts/keystore/watch_fallback.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build (darwin && !cgo) || ios || (linux && arm64) || windows || (!darwin && !freebsd && !linux && !netbsd && !solaris) // +build darwin,!cgo ios linux,arm64 windows !darwin,!freebsd,!linux,!netbsd,!solaris // This is the fallback implementation of directory watching. diff --git a/cmd/utils/diskusage.go b/cmd/utils/diskusage.go index c3d51765f..09844652e 100644 --- a/cmd/utils/diskusage.go +++ b/cmd/utils/diskusage.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build !windows && !openbsd // +build !windows,!openbsd package utils diff --git a/cmd/utils/diskusage_openbsd.go b/cmd/utils/diskusage_openbsd.go index 54f759d29..52502d0cf 100644 --- a/cmd/utils/diskusage_openbsd.go +++ b/cmd/utils/diskusage_openbsd.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build openbsd // +build openbsd package utils diff --git a/common/fdlimit/fdlimit_bsd.go b/common/fdlimit/fdlimit_bsd.go index 86181337a..a3a6902c0 100644 --- a/common/fdlimit/fdlimit_bsd.go +++ b/common/fdlimit/fdlimit_bsd.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build freebsd || dragonfly // +build freebsd dragonfly package fdlimit diff --git a/common/fdlimit/fdlimit_unix.go b/common/fdlimit/fdlimit_unix.go index e5a575f7a..a1f388ebb 100644 --- a/common/fdlimit/fdlimit_unix.go +++ b/common/fdlimit/fdlimit_unix.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build linux || netbsd || openbsd || solaris // +build linux netbsd openbsd solaris package fdlimit diff --git a/core/mkalloc.go b/core/mkalloc.go index 5118a4fcb..df167d708 100644 --- a/core/mkalloc.go +++ b/core/mkalloc.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build none // +build none /* diff --git a/crypto/blake2b/blake2bAVX2_amd64.go b/crypto/blake2b/blake2bAVX2_amd64.go index 0d52b1869..3a85d0e73 100644 --- a/crypto/blake2b/blake2bAVX2_amd64.go +++ b/crypto/blake2b/blake2bAVX2_amd64.go @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build go1.7 && amd64 && !gccgo && !appengine // +build go1.7,amd64,!gccgo,!appengine package blake2b diff --git a/crypto/blake2b/blake2b_amd64.go b/crypto/blake2b/blake2b_amd64.go index 4dbe90da8..a318b2b61 100644 --- a/crypto/blake2b/blake2b_amd64.go +++ b/crypto/blake2b/blake2b_amd64.go @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build !go1.7 && amd64 && !gccgo && !appengine // +build !go1.7,amd64,!gccgo,!appengine package blake2b diff --git a/crypto/blake2b/blake2b_f_fuzz.go b/crypto/blake2b/blake2b_f_fuzz.go index ab7334280..b2f405707 100644 --- a/crypto/blake2b/blake2b_f_fuzz.go +++ b/crypto/blake2b/blake2b_f_fuzz.go @@ -1,3 +1,4 @@ +//go:build gofuzz // +build gofuzz package blake2b diff --git a/crypto/blake2b/blake2b_ref.go b/crypto/blake2b/blake2b_ref.go index 9d0ade473..095c71a64 100644 --- a/crypto/blake2b/blake2b_ref.go +++ b/crypto/blake2b/blake2b_ref.go @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build !amd64 || appengine || gccgo // +build !amd64 appengine gccgo package blake2b diff --git a/crypto/blake2b/register.go b/crypto/blake2b/register.go index efd689af4..9d8633963 100644 --- a/crypto/blake2b/register.go +++ b/crypto/blake2b/register.go @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build go1.9 // +build go1.9 package blake2b diff --git a/crypto/bls12381/arithmetic_decl.go b/crypto/bls12381/arithmetic_decl.go index ec0b21e80..f6d232d65 100644 --- a/crypto/bls12381/arithmetic_decl.go +++ b/crypto/bls12381/arithmetic_decl.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build (amd64 && blsasm) || (amd64 && blsadx) // +build amd64,blsasm amd64,blsadx package bls12381 diff --git a/crypto/bls12381/arithmetic_fallback.go b/crypto/bls12381/arithmetic_fallback.go index 91cabf4f3..c09ae0d91 100644 --- a/crypto/bls12381/arithmetic_fallback.go +++ b/crypto/bls12381/arithmetic_fallback.go @@ -31,6 +31,7 @@ // Package bls (generated by goff) contains field arithmetics operations +//go:build !amd64 || (!blsasm && !blsadx) // +build !amd64 !blsasm,!blsadx package bls12381 diff --git a/crypto/bls12381/arithmetic_x86_adx.go b/crypto/bls12381/arithmetic_x86_adx.go index 9c30741e6..a40c7384e 100644 --- a/crypto/bls12381/arithmetic_x86_adx.go +++ b/crypto/bls12381/arithmetic_x86_adx.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build amd64 && blsadx // +build amd64,blsadx package bls12381 diff --git a/crypto/bls12381/arithmetic_x86_noadx.go b/crypto/bls12381/arithmetic_x86_noadx.go index eaac4b45d..679b30ec8 100644 --- a/crypto/bls12381/arithmetic_x86_noadx.go +++ b/crypto/bls12381/arithmetic_x86_noadx.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build amd64 && blsasm // +build amd64,blsasm package bls12381 diff --git a/crypto/bn256/bn256_fast.go b/crypto/bn256/bn256_fast.go index 14b596539..e3c9b6051 100644 --- a/crypto/bn256/bn256_fast.go +++ b/crypto/bn256/bn256_fast.go @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style license that can be found // in the LICENSE file. +//go:build amd64 || arm64 // +build amd64 arm64 // Package bn256 implements the Optimal Ate pairing over a 256-bit Barreto-Naehrig curve. diff --git a/crypto/bn256/bn256_slow.go b/crypto/bn256/bn256_slow.go index 49021082f..4c0c351e2 100644 --- a/crypto/bn256/bn256_slow.go +++ b/crypto/bn256/bn256_slow.go @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style license that can be found // in the LICENSE file. +//go:build !amd64 && !arm64 // +build !amd64,!arm64 // Package bn256 implements the Optimal Ate pairing over a 256-bit Barreto-Naehrig curve. diff --git a/crypto/bn256/cloudflare/gfp_decl.go b/crypto/bn256/cloudflare/gfp_decl.go index fdea5c11a..ec4018e88 100644 --- a/crypto/bn256/cloudflare/gfp_decl.go +++ b/crypto/bn256/cloudflare/gfp_decl.go @@ -1,3 +1,4 @@ +//go:build (amd64 && !generic) || (arm64 && !generic) // +build amd64,!generic arm64,!generic package bn256 diff --git a/crypto/bn256/cloudflare/gfp_generic.go b/crypto/bn256/cloudflare/gfp_generic.go index 8e6be9596..7742dda4c 100644 --- a/crypto/bn256/cloudflare/gfp_generic.go +++ b/crypto/bn256/cloudflare/gfp_generic.go @@ -1,3 +1,4 @@ +//go:build (!amd64 && !arm64) || generic // +build !amd64,!arm64 generic package bn256 diff --git a/crypto/secp256k1/dummy.go b/crypto/secp256k1/dummy.go index c0f2ee52c..65a75080f 100644 --- a/crypto/secp256k1/dummy.go +++ b/crypto/secp256k1/dummy.go @@ -1,3 +1,4 @@ +//go:build dummy // +build dummy // This file is part of a workaround for `go mod vendor` which won't vendor diff --git a/crypto/secp256k1/panic_cb.go b/crypto/secp256k1/panic_cb.go index 5da2bea37..a30b04f51 100644 --- a/crypto/secp256k1/panic_cb.go +++ b/crypto/secp256k1/panic_cb.go @@ -2,8 +2,8 @@ // Use of this source code is governed by a BSD-style license that can be found in // the LICENSE file. -// +build !gofuzz -// +build cgo +//go:build !gofuzz && cgo +// +build !gofuzz,cgo package secp256k1 diff --git a/crypto/secp256k1/scalar_mult_cgo.go b/crypto/secp256k1/scalar_mult_cgo.go index f28a1c782..8afa9d023 100644 --- a/crypto/secp256k1/scalar_mult_cgo.go +++ b/crypto/secp256k1/scalar_mult_cgo.go @@ -2,8 +2,8 @@ // Use of this source code is governed by a BSD-style license that can be found in // the LICENSE file. -// +build !gofuzz -// +build cgo +//go:build !gofuzz && cgo +// +build !gofuzz,cgo package secp256k1 diff --git a/crypto/secp256k1/scalar_mult_nocgo.go b/crypto/secp256k1/scalar_mult_nocgo.go index 55756b5be..22f53ac6a 100644 --- a/crypto/secp256k1/scalar_mult_nocgo.go +++ b/crypto/secp256k1/scalar_mult_nocgo.go @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style license that can be found in // the LICENSE file. +//go:build gofuzz || !cgo // +build gofuzz !cgo package secp256k1 diff --git a/crypto/secp256k1/secp256.go b/crypto/secp256k1/secp256.go index a1bcf7796..c9c01b320 100644 --- a/crypto/secp256k1/secp256.go +++ b/crypto/secp256k1/secp256.go @@ -2,8 +2,8 @@ // Use of this source code is governed by a BSD-style license that can be found in // the LICENSE file. -// +build !gofuzz -// +build cgo +//go:build !gofuzz && cgo +// +build !gofuzz,cgo // Package secp256k1 wraps the bitcoin secp256k1 C library. package secp256k1 diff --git a/crypto/signature_cgo.go b/crypto/signature_cgo.go index 843360298..bd72d97d3 100644 --- a/crypto/signature_cgo.go +++ b/crypto/signature_cgo.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build !nacl && !js && cgo && !gofuzz // +build !nacl,!js,cgo,!gofuzz package crypto diff --git a/crypto/signature_nocgo.go b/crypto/signature_nocgo.go index 77c8a1db0..fd1e66c7e 100644 --- a/crypto/signature_nocgo.go +++ b/crypto/signature_nocgo.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build nacl || js || !cgo || gofuzz // +build nacl js !cgo gofuzz package crypto diff --git a/crypto/signify/signify_fuzz.go b/crypto/signify/signify_fuzz.go index f9167900a..2dc9b2102 100644 --- a/crypto/signify/signify_fuzz.go +++ b/crypto/signify/signify_fuzz.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build gofuzz // +build gofuzz package signify diff --git a/ethdb/leveldb/leveldb.go b/ethdb/leveldb/leveldb.go index 5d19cc357..9ff1a2ce1 100644 --- a/ethdb/leveldb/leveldb.go +++ b/ethdb/leveldb/leveldb.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build !js // +build !js // Package leveldb implements the key-value database layer based on LevelDB. diff --git a/internal/debug/loudpanic.go b/internal/debug/loudpanic.go index 572ebcefa..86e6bc88f 100644 --- a/internal/debug/loudpanic.go +++ b/internal/debug/loudpanic.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build go1.6 // +build go1.6 package debug diff --git a/internal/debug/loudpanic_fallback.go b/internal/debug/loudpanic_fallback.go index 4ce4985da..377490e5b 100644 --- a/internal/debug/loudpanic_fallback.go +++ b/internal/debug/loudpanic_fallback.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build !go1.6 // +build !go1.6 package debug diff --git a/internal/debug/trace.go b/internal/debug/trace.go index cab5deaaf..a273e4a9d 100644 --- a/internal/debug/trace.go +++ b/internal/debug/trace.go @@ -14,7 +14,8 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -//+build go1.5 +//go:build go1.5 +// +build go1.5 package debug diff --git a/internal/debug/trace_fallback.go b/internal/debug/trace_fallback.go index 4118ff408..ec07d991e 100644 --- a/internal/debug/trace_fallback.go +++ b/internal/debug/trace_fallback.go @@ -14,7 +14,8 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -//+build !go1.5 +//go:build !go1.5 +// +build !go1.5 // no-op implementation of tracing methods for Go < 1.5. diff --git a/log/handler_go13.go b/log/handler_go13.go index 0843ed0e5..4df694deb 100644 --- a/log/handler_go13.go +++ b/log/handler_go13.go @@ -1,3 +1,4 @@ +//go:build !go1.4 // +build !go1.4 package log diff --git a/log/handler_go14.go b/log/handler_go14.go index 05dedbf2a..d0cb14aa0 100644 --- a/log/handler_go14.go +++ b/log/handler_go14.go @@ -1,3 +1,4 @@ +//go:build go1.4 // +build go1.4 package log diff --git a/log/syslog.go b/log/syslog.go index 71a17b30b..451d831b6 100644 --- a/log/syslog.go +++ b/log/syslog.go @@ -1,3 +1,4 @@ +//go:build !windows && !plan9 // +build !windows,!plan9 package log diff --git a/metrics/cpu_disabled.go b/metrics/cpu_disabled.go index b0b483ee4..025d97aeb 100644 --- a/metrics/cpu_disabled.go +++ b/metrics/cpu_disabled.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build ios || js // +build ios js package metrics diff --git a/metrics/cpu_enabled.go b/metrics/cpu_enabled.go index edb824d9a..533d40b85 100644 --- a/metrics/cpu_enabled.go +++ b/metrics/cpu_enabled.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build !ios && !js // +build !ios,!js package metrics diff --git a/metrics/cputime_nop.go b/metrics/cputime_nop.go index 1bf553727..0188735a7 100644 --- a/metrics/cputime_nop.go +++ b/metrics/cputime_nop.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build windows || js // +build windows js package metrics diff --git a/metrics/cputime_unix.go b/metrics/cputime_unix.go index 55871e668..3c56a75d0 100644 --- a/metrics/cputime_unix.go +++ b/metrics/cputime_unix.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build !windows && !js // +build !windows,!js package metrics diff --git a/metrics/disk_nop.go b/metrics/disk_nop.go index 4319f8b27..58fa4e02f 100644 --- a/metrics/disk_nop.go +++ b/metrics/disk_nop.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build !linux // +build !linux package metrics diff --git a/metrics/runtime_cgo.go b/metrics/runtime_cgo.go index 11722f76c..4307ebdba 100644 --- a/metrics/runtime_cgo.go +++ b/metrics/runtime_cgo.go @@ -1,3 +1,4 @@ +//go:build cgo && !appengine && !js // +build cgo,!appengine,!js package metrics diff --git a/metrics/runtime_gccpufraction.go b/metrics/runtime_gccpufraction.go index ca12c05ba..28cd44752 100644 --- a/metrics/runtime_gccpufraction.go +++ b/metrics/runtime_gccpufraction.go @@ -1,3 +1,4 @@ +//go:build go1.5 // +build go1.5 package metrics diff --git a/metrics/runtime_no_cgo.go b/metrics/runtime_no_cgo.go index e760af554..1799bef63 100644 --- a/metrics/runtime_no_cgo.go +++ b/metrics/runtime_no_cgo.go @@ -1,3 +1,4 @@ +//go:build !cgo || appengine || js // +build !cgo appengine js package metrics diff --git a/metrics/runtime_no_gccpufraction.go b/metrics/runtime_no_gccpufraction.go index be96aa6f1..af1a4b63c 100644 --- a/metrics/runtime_no_gccpufraction.go +++ b/metrics/runtime_no_gccpufraction.go @@ -1,3 +1,4 @@ +//go:build !go1.5 // +build !go1.5 package metrics diff --git a/metrics/syslog.go b/metrics/syslog.go index a0ed4b1b2..551a2bd0f 100644 --- a/metrics/syslog.go +++ b/metrics/syslog.go @@ -1,3 +1,4 @@ +//go:build !windows // +build !windows package metrics diff --git a/mobile/geth_android.go b/mobile/geth_android.go index 8e4ebe638..cfdf1c28c 100644 --- a/mobile/geth_android.go +++ b/mobile/geth_android.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build android // +build android package geth diff --git a/mobile/geth_ios.go b/mobile/geth_ios.go index 307cd0858..aab839727 100644 --- a/mobile/geth_ios.go +++ b/mobile/geth_ios.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build ios // +build ios package geth diff --git a/mobile/geth_other.go b/mobile/geth_other.go index 6f0c5dda6..c5cad4a7b 100644 --- a/mobile/geth_other.go +++ b/mobile/geth_other.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build !android && !ios // +build !android,!ios package geth diff --git a/p2p/netutil/toobig_notwindows.go b/p2p/netutil/toobig_notwindows.go index 47b643857..f9f936ae5 100644 --- a/p2p/netutil/toobig_notwindows.go +++ b/p2p/netutil/toobig_notwindows.go @@ -14,7 +14,8 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -//+build !windows +//go:build !windows +// +build !windows package netutil diff --git a/p2p/netutil/toobig_windows.go b/p2p/netutil/toobig_windows.go index dfbb6d44f..652903e83 100644 --- a/p2p/netutil/toobig_windows.go +++ b/p2p/netutil/toobig_windows.go @@ -14,7 +14,8 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -//+build windows +//go:build windows +// +build windows package netutil diff --git a/rlp/safe.go b/rlp/safe.go index a80380aef..6f3b151ff 100644 --- a/rlp/safe.go +++ b/rlp/safe.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build nacl || js || !cgo // +build nacl js !cgo package rlp diff --git a/rlp/unsafe.go b/rlp/unsafe.go index 94ed5405a..30bb9a659 100644 --- a/rlp/unsafe.go +++ b/rlp/unsafe.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build !nacl && !js && cgo // +build !nacl,!js,cgo package rlp diff --git a/rpc/constants_unix.go b/rpc/constants_unix.go index 2f98d6499..1f04d15d7 100644 --- a/rpc/constants_unix.go +++ b/rpc/constants_unix.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build darwin || dragonfly || freebsd || linux || nacl || netbsd || openbsd || solaris // +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris package rpc diff --git a/rpc/constants_unix_nocgo.go b/rpc/constants_unix_nocgo.go index ecb231f92..a62e4ee52 100644 --- a/rpc/constants_unix_nocgo.go +++ b/rpc/constants_unix_nocgo.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build !cgo && !windows // +build !cgo,!windows package rpc diff --git a/rpc/ipc_js.go b/rpc/ipc_js.go index 7e7554a76..453a20bc1 100644 --- a/rpc/ipc_js.go +++ b/rpc/ipc_js.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build js // +build js package rpc diff --git a/rpc/ipc_unix.go b/rpc/ipc_unix.go index f4690cc0a..249a9cf04 100644 --- a/rpc/ipc_unix.go +++ b/rpc/ipc_unix.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build darwin || dragonfly || freebsd || linux || nacl || netbsd || openbsd || solaris // +build darwin dragonfly freebsd linux nacl netbsd openbsd solaris package rpc diff --git a/rpc/ipc_windows.go b/rpc/ipc_windows.go index ca56a3ce4..adb1826f0 100644 --- a/rpc/ipc_windows.go +++ b/rpc/ipc_windows.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build windows // +build windows package rpc diff --git a/tests/fuzzers/bls12381/bls12381_fuzz.go b/tests/fuzzers/bls12381/bls12381_fuzz.go index c0f452f3e..b283ed11f 100644 --- a/tests/fuzzers/bls12381/bls12381_fuzz.go +++ b/tests/fuzzers/bls12381/bls12381_fuzz.go @@ -14,6 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . +//go:build gofuzz // +build gofuzz package bls From 32c576bd3c64bb19e0b6aa7b67342092a2022242 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 25 Aug 2021 19:01:10 +0200 Subject: [PATCH 24/73] rlp: minor optimizations for slice/array encoding (#23467) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit As per benchmark results below, these changes speed up encoding/decoding of consensus objects a bit. name old time/op new time/op delta EncodeRLP/legacy-header-8 384ns ± 1% 331ns ± 3% -13.83% (p=0.000 n=7+8) EncodeRLP/london-header-8 411ns ± 1% 359ns ± 2% -12.53% (p=0.000 n=8+8) EncodeRLP/receipt-for-storage-8 251ns ± 0% 239ns ± 0% -4.97% (p=0.000 n=8+8) EncodeRLP/receipt-full-8 319ns ± 0% 300ns ± 0% -5.89% (p=0.000 n=8+7) EncodeRLP/legacy-transaction-8 389ns ± 1% 387ns ± 1% ~ (p=0.099 n=8+8) EncodeRLP/access-transaction-8 607ns ± 0% 581ns ± 0% -4.26% (p=0.000 n=8+8) EncodeRLP/1559-transaction-8 627ns ± 0% 606ns ± 1% -3.44% (p=0.000 n=8+8) DecodeRLP/legacy-header-8 831ns ± 1% 813ns ± 1% -2.20% (p=0.000 n=8+8) DecodeRLP/london-header-8 824ns ± 0% 804ns ± 1% -2.44% (p=0.000 n=8+7) * rlp: pass length to byteArrayBytes This makes it possible to inline byteArrayBytes. For arrays, the length is known at encoder construction time, so the call to v.Len() can be avoided. * rlp: avoid IsNil for pointer encoding It's actually cheaper to use Elem first, because it performs less checks on the value. If the pointer was nil, the result of Elem is 'invalid'. * rlp: minor optimizations for slice/array encoding For empty slices/arrays, we can avoid storing a list header entry in the encoder buffer. Also avoid doing the tail check at encoding time because it is already known at encoder construction time. --- rlp/decode.go | 2 +- rlp/encode.go | 107 ++++++++++++++++++++++++++++----------------- rlp/encode_test.go | 28 ++++++++++++ rlp/safe.go | 4 +- rlp/unsafe.go | 7 ++- 5 files changed, 101 insertions(+), 47 deletions(-) diff --git a/rlp/decode.go b/rlp/decode.go index ac04d5d56..5f2e5ad5f 100644 --- a/rlp/decode.go +++ b/rlp/decode.go @@ -379,7 +379,7 @@ func decodeByteArray(s *Stream, val reflect.Value) error { if err != nil { return err } - slice := byteArrayBytes(val) + slice := byteArrayBytes(val, val.Len()) switch kind { case Byte: if len(slice) == 0 { diff --git a/rlp/encode.go b/rlp/encode.go index 334864434..1623e97a3 100644 --- a/rlp/encode.go +++ b/rlp/encode.go @@ -432,7 +432,20 @@ func makeByteArrayWriter(typ reflect.Type) writer { case 1: return writeLengthOneByteArray default: - return writeByteArray + length := typ.Len() + return func(val reflect.Value, w *encbuf) error { + if !val.CanAddr() { + // Getting the byte slice of val requires it to be addressable. Make it + // addressable by copying. + copy := reflect.New(val.Type()).Elem() + copy.Set(val) + val = copy + } + slice := byteArrayBytes(val, length) + w.encodeStringHeader(len(slice)) + w.str = append(w.str, slice...) + return nil + } } } @@ -451,21 +464,6 @@ func writeLengthOneByteArray(val reflect.Value, w *encbuf) error { return nil } -func writeByteArray(val reflect.Value, w *encbuf) error { - if !val.CanAddr() { - // Getting the byte slice of val requires it to be addressable. Make it - // addressable by copying. - copy := reflect.New(val.Type()).Elem() - copy.Set(val) - val = copy - } - - slice := byteArrayBytes(val) - w.encodeStringHeader(len(slice)) - w.str = append(w.str, slice...) - return nil -} - func writeString(val reflect.Value, w *encbuf) error { s := val.String() if len(s) == 1 && s[0] <= 0x7f { @@ -499,19 +497,39 @@ func makeSliceWriter(typ reflect.Type, ts tags) (writer, error) { if etypeinfo.writerErr != nil { return nil, etypeinfo.writerErr } - writer := func(val reflect.Value, w *encbuf) error { - if !ts.tail { - defer w.listEnd(w.list()) - } - vlen := val.Len() - for i := 0; i < vlen; i++ { - if err := etypeinfo.writer(val.Index(i), w); err != nil { - return err + + var wfn writer + if ts.tail { + // This is for struct tail slices. + // w.list is not called for them. + wfn = func(val reflect.Value, w *encbuf) error { + vlen := val.Len() + for i := 0; i < vlen; i++ { + if err := etypeinfo.writer(val.Index(i), w); err != nil { + return err + } } + return nil + } + } else { + // This is for regular slices and arrays. + wfn = func(val reflect.Value, w *encbuf) error { + vlen := val.Len() + if vlen == 0 { + w.str = append(w.str, 0xC0) + return nil + } + listOffset := w.list() + for i := 0; i < vlen; i++ { + if err := etypeinfo.writer(val.Index(i), w); err != nil { + return err + } + } + w.listEnd(listOffset) + return nil } - return nil } - return writer, nil + return wfn, nil } func makeStructWriter(typ reflect.Type) (writer, error) { @@ -562,12 +580,8 @@ func makeStructWriter(typ reflect.Type) (writer, error) { return writer, nil } -func makePtrWriter(typ reflect.Type, ts tags) (writer, error) { - etypeinfo := theTC.infoWhileGenerating(typ.Elem(), tags{}) - if etypeinfo.writerErr != nil { - return nil, etypeinfo.writerErr - } - // Determine how to encode nil pointers. +// nilEncoding returns the encoded value of a nil pointer. +func nilEncoding(typ reflect.Type, ts tags) uint8 { var nilKind Kind if ts.nilOK { nilKind = ts.nilKind // use struct tag if provided @@ -575,16 +589,29 @@ func makePtrWriter(typ reflect.Type, ts tags) (writer, error) { nilKind = defaultNilKind(typ.Elem()) } + switch nilKind { + case String: + return 0x80 + case List: + return 0xC0 + default: + panic(fmt.Errorf("rlp: invalid nil kind %d", nilKind)) + } +} + +func makePtrWriter(typ reflect.Type, ts tags) (writer, error) { + etypeinfo := theTC.infoWhileGenerating(typ.Elem(), tags{}) + if etypeinfo.writerErr != nil { + return nil, etypeinfo.writerErr + } + nilEncoding := nilEncoding(typ, ts) + writer := func(val reflect.Value, w *encbuf) error { - if val.IsNil() { - if nilKind == String { - w.str = append(w.str, 0x80) - } else { - w.listEnd(w.list()) - } - return nil + if ev := val.Elem(); ev.IsValid() { + return etypeinfo.writer(ev, w) } - return etypeinfo.writer(val.Elem(), w) + w.str = append(w.str, nilEncoding) + return nil } return writer, nil } diff --git a/rlp/encode_test.go b/rlp/encode_test.go index 25d4aac26..a63743440 100644 --- a/rlp/encode_test.go +++ b/rlp/encode_test.go @@ -540,3 +540,31 @@ func BenchmarkEncodeByteArrayStruct(b *testing.B) { } } } + +type structSliceElem struct { + X uint64 + Y uint64 + Z uint64 +} + +type structPtrSlice []*structSliceElem + +func BenchmarkEncodeStructPtrSlice(b *testing.B) { + var out bytes.Buffer + var value = structPtrSlice{ + &structSliceElem{1, 1, 1}, + &structSliceElem{2, 2, 2}, + &structSliceElem{3, 3, 3}, + &structSliceElem{5, 5, 5}, + &structSliceElem{6, 6, 6}, + &structSliceElem{7, 7, 7}, + } + + b.ReportAllocs() + for i := 0; i < b.N; i++ { + out.Reset() + if err := Encode(&out, &value); err != nil { + b.Fatal(err) + } + } +} diff --git a/rlp/safe.go b/rlp/safe.go index 6f3b151ff..3c910337b 100644 --- a/rlp/safe.go +++ b/rlp/safe.go @@ -22,6 +22,6 @@ package rlp import "reflect" // byteArrayBytes returns a slice of the byte array v. -func byteArrayBytes(v reflect.Value) []byte { - return v.Slice(0, v.Len()).Bytes() +func byteArrayBytes(v reflect.Value, length int) []byte { + return v.Slice(0, length).Bytes() } diff --git a/rlp/unsafe.go b/rlp/unsafe.go index 30bb9a659..2152ba35f 100644 --- a/rlp/unsafe.go +++ b/rlp/unsafe.go @@ -25,12 +25,11 @@ import ( ) // byteArrayBytes returns a slice of the byte array v. -func byteArrayBytes(v reflect.Value) []byte { - len := v.Len() +func byteArrayBytes(v reflect.Value, length int) []byte { var s []byte hdr := (*reflect.SliceHeader)(unsafe.Pointer(&s)) hdr.Data = v.UnsafeAddr() - hdr.Cap = len - hdr.Len = len + hdr.Cap = length + hdr.Len = length return s } From 154b525ce8908dfa8e4fa19b9da7aee78ecba156 Mon Sep 17 00:00:00 2001 From: baptiste-b-pegasys <85155432+baptiste-b-pegasys@users.noreply.github.com> Date: Wed, 25 Aug 2021 19:06:42 +0200 Subject: [PATCH 25/73] cmd/devp2p/internal/ethtest: remove WriteTo method (#23379) WriteTo is unused, and go vet warns about it because it doesn't match the usual signature of the io.WriterTo method. --- cmd/devp2p/internal/ethtest/chain.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/cmd/devp2p/internal/ethtest/chain.go b/cmd/devp2p/internal/ethtest/chain.go index 34a20c515..7dcb412b5 100644 --- a/cmd/devp2p/internal/ethtest/chain.go +++ b/cmd/devp2p/internal/ethtest/chain.go @@ -39,16 +39,6 @@ type Chain struct { chainConfig *params.ChainConfig } -func (c *Chain) WriteTo(writer io.Writer) error { - for _, block := range c.blocks { - if err := rlp.Encode(writer, block); err != nil { - return err - } - } - - return nil -} - // Len returns the length of the chain. func (c *Chain) Len() int { return len(c.blocks) From 7c4cad064cf7a9f48f9d69c6a4f459ebe8bb19ab Mon Sep 17 00:00:00 2001 From: Dmitry Zenovich Date: Wed, 25 Aug 2021 20:30:29 +0300 Subject: [PATCH 26/73] rpc: add BlockNumber.MarshalText (#23324) Currently rpc.BlockNumber is marshalled to JSON as a numeric value, which is wrong because BlockNumber.UnmarshalJSON() wants it to either be hex-encoded or string "earliest"/"latest"/"pending". As a result, the call chain rpc.BlockNumberOrHashWithNumber(123) -> json.Marshal() -> json.Unmarshal() fails with error "cannot unmarshal object into Go value of type string". --- rpc/types.go | 16 ++++++++++++++++ rpc/types_test.go | 31 +++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+) diff --git a/rpc/types.go b/rpc/types.go index ad068defa..d9c2317a7 100644 --- a/rpc/types.go +++ b/rpc/types.go @@ -98,6 +98,22 @@ func (bn *BlockNumber) UnmarshalJSON(data []byte) error { return nil } +// MarshalText implements encoding.TextMarshaler. It marshals: +// - "latest", "earliest" or "pending" as strings +// - other numbers as hex +func (bn BlockNumber) MarshalText() ([]byte, error) { + switch bn { + case EarliestBlockNumber: + return []byte("earliest"), nil + case LatestBlockNumber: + return []byte("latest"), nil + case PendingBlockNumber: + return []byte("pending"), nil + default: + return hexutil.Uint64(bn).MarshalText() + } +} + func (bn BlockNumber) Int64() int64 { return (int64)(bn) } diff --git a/rpc/types_test.go b/rpc/types_test.go index 89b0c9171..f110dee7c 100644 --- a/rpc/types_test.go +++ b/rpc/types_test.go @@ -18,6 +18,7 @@ package rpc import ( "encoding/json" + "reflect" "testing" "github.com/ethereum/go-ethereum/common" @@ -122,3 +123,33 @@ func TestBlockNumberOrHash_UnmarshalJSON(t *testing.T) { } } } + +func TestBlockNumberOrHash_WithNumber_MarshalAndUnmarshal(t *testing.T) { + tests := []struct { + name string + number int64 + }{ + {"max", math.MaxInt64}, + {"pending", int64(PendingBlockNumber)}, + {"latest", int64(LatestBlockNumber)}, + {"earliest", int64(EarliestBlockNumber)}, + } + for _, test := range tests { + test := test + t.Run(test.name, func(t *testing.T) { + bnh := BlockNumberOrHashWithNumber(BlockNumber(test.number)) + marshalled, err := json.Marshal(bnh) + if err != nil { + t.Fatal("cannot marshal:", err) + } + var unmarshalled BlockNumberOrHash + err = json.Unmarshal(marshalled, &unmarshalled) + if err != nil { + t.Fatal("cannot unmarshal:", err) + } + if !reflect.DeepEqual(bnh, unmarshalled) { + t.Fatalf("wrong result: expected %v, got %v", bnh, unmarshalled) + } + }) + } +} From d584e3986251902bce78369ab4b0f32da70bc82f Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 25 Aug 2021 21:57:31 +0200 Subject: [PATCH 27/73] build: stop publishing deb packages for Ubuntu 20.10 Groovy Gorilla (#23470) Support for this Ubuntu release has ended on July 22 2021 and Launchpad no longer accepts uploads for it. --- build/ci.go | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/build/ci.go b/build/ci.go index 3de135120..6f1e975a5 100644 --- a/build/ci.go +++ b/build/ci.go @@ -129,19 +129,13 @@ var ( // Distros for which packages are created. // Note: vivid is unsupported because there is no golang-1.6 package for it. - // Note: wily is unsupported because it was officially deprecated on Launchpad. - // Note: yakkety is unsupported because it was officially deprecated on Launchpad. - // Note: zesty is unsupported because it was officially deprecated on Launchpad. - // Note: artful is unsupported because it was officially deprecated on Launchpad. - // Note: cosmic is unsupported because it was officially deprecated on Launchpad. - // Note: disco is unsupported because it was officially deprecated on Launchpad. - // Note: eoan is unsupported because it was officially deprecated on Launchpad. + // Note: the following Ubuntu releases have been officially deprecated on Launchpad: + // wily, yakkety, zesty, artful, cosmic, disco, eoan, groovy debDistroGoBoots = map[string]string{ "trusty": "golang-1.11", "xenial": "golang-go", "bionic": "golang-go", "focal": "golang-go", - "groovy": "golang-go", "hirsute": "golang-go", } From 108eec3feeca2e05d3daf1a198b8286623f56755 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi <1591639+s1na@users.noreply.github.com> Date: Wed, 25 Aug 2021 22:34:22 +0200 Subject: [PATCH 28/73] node: remove dependency on wallet backend packages (#23019) * accounts: new AddBackends method in manager * node,cmd/geth: mv accman backend init to cmd/geth * node,cmd/geth: mv scrypt config downstreawm from node * accounts: use static buffer size for accman sub chan minor fix * accounts,cmd/geth: update accman backends through its event loop * accounts,node: add comments * accounts: un-export newBackendEvent * accounts: use chan instead of wg in newBlockEvent * node: rename isKeyDirEphem * accounts,cmd: AddBackends->AddBackend * accounts: fix potential blocking when adding backend --- accounts/manager.go | 64 ++++++++++++++++++++++++++------- cmd/geth/accountcmd.go | 9 +++-- cmd/geth/config.go | 68 +++++++++++++++++++++++++++++++++++ node/config.go | 82 ++++++++---------------------------------- node/node.go | 23 +++++++----- 5 files changed, 156 insertions(+), 90 deletions(-) diff --git a/accounts/manager.go b/accounts/manager.go index acf41ed8e..1e111d194 100644 --- a/accounts/manager.go +++ b/accounts/manager.go @@ -25,6 +25,10 @@ import ( "github.com/ethereum/go-ethereum/event" ) +// managerSubBufferSize determines how many incoming wallet events +// the manager will buffer in its channel. +const managerSubBufferSize = 50 + // Config contains the settings of the global account manager. // // TODO(rjl493456442, karalabe, holiman): Get rid of this when account management @@ -33,18 +37,27 @@ type Config struct { InsecureUnlockAllowed bool // Whether account unlocking in insecure environment is allowed } +// newBackendEvent lets the manager know it should +// track the given backend for wallet updates. +type newBackendEvent struct { + backend Backend + processed chan struct{} // Informs event emitter that backend has been integrated +} + // Manager is an overarching account manager that can communicate with various // backends for signing transactions. type Manager struct { - config *Config // Global account manager configurations - backends map[reflect.Type][]Backend // Index of backends currently registered - updaters []event.Subscription // Wallet update subscriptions for all backends - updates chan WalletEvent // Subscription sink for backend wallet changes - wallets []Wallet // Cache of all wallets from all registered backends + config *Config // Global account manager configurations + backends map[reflect.Type][]Backend // Index of backends currently registered + updaters []event.Subscription // Wallet update subscriptions for all backends + updates chan WalletEvent // Subscription sink for backend wallet changes + newBackends chan newBackendEvent // Incoming backends to be tracked by the manager + wallets []Wallet // Cache of all wallets from all registered backends feed event.Feed // Wallet feed notifying of arrivals/departures quit chan chan error + term chan struct{} // Channel is closed upon termination of the update loop lock sync.RWMutex } @@ -57,7 +70,7 @@ func NewManager(config *Config, backends ...Backend) *Manager { wallets = merge(wallets, backend.Wallets()...) } // Subscribe to wallet notifications from all backends - updates := make(chan WalletEvent, 4*len(backends)) + updates := make(chan WalletEvent, managerSubBufferSize) subs := make([]event.Subscription, len(backends)) for i, backend := range backends { @@ -65,12 +78,14 @@ func NewManager(config *Config, backends ...Backend) *Manager { } // Assemble the account manager and return am := &Manager{ - config: config, - backends: make(map[reflect.Type][]Backend), - updaters: subs, - updates: updates, - wallets: wallets, - quit: make(chan chan error), + config: config, + backends: make(map[reflect.Type][]Backend), + updaters: subs, + updates: updates, + newBackends: make(chan newBackendEvent), + wallets: wallets, + quit: make(chan chan error), + term: make(chan struct{}), } for _, backend := range backends { kind := reflect.TypeOf(backend) @@ -93,6 +108,14 @@ func (am *Manager) Config() *Config { return am.config } +// AddBackend starts the tracking of an additional backend for wallet updates. +// cmd/geth assumes once this func returns the backends have been already integrated. +func (am *Manager) AddBackend(backend Backend) { + done := make(chan struct{}) + am.newBackends <- newBackendEvent{backend, done} + <-done +} + // update is the wallet event loop listening for notifications from the backends // and updating the cache of wallets. func (am *Manager) update() { @@ -122,10 +145,22 @@ func (am *Manager) update() { // Notify any listeners of the event am.feed.Send(event) - + case event := <-am.newBackends: + am.lock.Lock() + // Update caches + backend := event.backend + am.wallets = merge(am.wallets, backend.Wallets()...) + am.updaters = append(am.updaters, backend.Subscribe(am.updates)) + kind := reflect.TypeOf(backend) + am.backends[kind] = append(am.backends[kind], backend) + am.lock.Unlock() + close(event.processed) case errc := <-am.quit: // Manager terminating, return errc <- nil + // Signals event emitters the loop is not receiving values + // to prevent them from getting stuck. + close(am.term) return } } @@ -133,6 +168,9 @@ func (am *Manager) update() { // Backends retrieves the backend(s) with the given type from the account manager. func (am *Manager) Backends(kind reflect.Type) []Backend { + am.lock.RLock() + defer am.lock.RUnlock() + return am.backends[kind] } diff --git a/cmd/geth/accountcmd.go b/cmd/geth/accountcmd.go index 6d45c8876..e33b9eb0f 100644 --- a/cmd/geth/accountcmd.go +++ b/cmd/geth/accountcmd.go @@ -268,11 +268,16 @@ func accountCreate(ctx *cli.Context) error { } } utils.SetNodeConfig(ctx, &cfg.Node) - scryptN, scryptP, keydir, err := cfg.Node.AccountConfig() - + keydir, err := cfg.Node.KeyDirConfig() if err != nil { utils.Fatalf("Failed to read configuration: %v", err) } + scryptN := keystore.StandardScryptN + scryptP := keystore.StandardScryptP + if cfg.Node.UseLightweightKDF { + scryptN = keystore.LightScryptN + scryptP = keystore.LightScryptP + } password := utils.GetPassPhraseWithList("Your new account is locked with a password. Please give a password. Do not forget this password.", true, 0, utils.MakePasswordList(ctx)) diff --git a/cmd/geth/config.go b/cmd/geth/config.go index 604f907b7..c97a64f17 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -27,6 +27,10 @@ import ( "gopkg.in/urfave/cli.v1" + "github.com/ethereum/go-ethereum/accounts/external" + "github.com/ethereum/go-ethereum/accounts/keystore" + "github.com/ethereum/go-ethereum/accounts/scwallet" + "github.com/ethereum/go-ethereum/accounts/usbwallet" "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/eth/catalyst" "github.com/ethereum/go-ethereum/eth/ethconfig" @@ -135,6 +139,11 @@ func makeConfigNode(ctx *cli.Context) (*node.Node, gethConfig) { if err != nil { utils.Fatalf("Failed to create the protocol stack: %v", err) } + // Node doesn't by default populate account manager backends + if err := setAccountManagerBackends(stack); err != nil { + utils.Fatalf("Failed to set account manager backends: %v", err) + } + utils.SetEthConfig(ctx, stack, &cfg.Eth) if ctx.GlobalIsSet(utils.EthStatsURLFlag.Name) { cfg.Ethstats.URL = ctx.GlobalString(utils.EthStatsURLFlag.Name) @@ -257,3 +266,62 @@ func deprecated(field string) bool { return false } } + +func setAccountManagerBackends(stack *node.Node) error { + conf := stack.Config() + am := stack.AccountManager() + keydir := stack.KeyStoreDir() + scryptN := keystore.StandardScryptN + scryptP := keystore.StandardScryptP + if conf.UseLightweightKDF { + scryptN = keystore.LightScryptN + scryptP = keystore.LightScryptP + } + + // Assemble the supported backends + if len(conf.ExternalSigner) > 0 { + log.Info("Using external signer", "url", conf.ExternalSigner) + if extapi, err := external.NewExternalBackend(conf.ExternalSigner); err == nil { + am.AddBackend(extapi) + return nil + } else { + return fmt.Errorf("error connecting to external signer: %v", err) + } + } + + // For now, we're using EITHER external signer OR local signers. + // If/when we implement some form of lockfile for USB and keystore wallets, + // we can have both, but it's very confusing for the user to see the same + // accounts in both externally and locally, plus very racey. + am.AddBackend(keystore.NewKeyStore(keydir, scryptN, scryptP)) + if conf.USB { + // Start a USB hub for Ledger hardware wallets + if ledgerhub, err := usbwallet.NewLedgerHub(); err != nil { + log.Warn(fmt.Sprintf("Failed to start Ledger hub, disabling: %v", err)) + } else { + am.AddBackend(ledgerhub) + } + // Start a USB hub for Trezor hardware wallets (HID version) + if trezorhub, err := usbwallet.NewTrezorHubWithHID(); err != nil { + log.Warn(fmt.Sprintf("Failed to start HID Trezor hub, disabling: %v", err)) + } else { + am.AddBackend(trezorhub) + } + // Start a USB hub for Trezor hardware wallets (WebUSB version) + if trezorhub, err := usbwallet.NewTrezorHubWithWebUSB(); err != nil { + log.Warn(fmt.Sprintf("Failed to start WebUSB Trezor hub, disabling: %v", err)) + } else { + am.AddBackend(trezorhub) + } + } + if len(conf.SmartCardDaemonPath) > 0 { + // Start a smart card hub + if schub, err := scwallet.NewHub(conf.SmartCardDaemonPath, scwallet.Scheme, keydir); err != nil { + log.Warn(fmt.Sprintf("Failed to start smart card hub, disabling: %v", err)) + } else { + am.AddBackend(schub) + } + } + + return nil +} diff --git a/node/config.go b/node/config.go index b7e99eb7b..26f00cd67 100644 --- a/node/config.go +++ b/node/config.go @@ -26,11 +26,6 @@ import ( "strings" "sync" - "github.com/ethereum/go-ethereum/accounts" - "github.com/ethereum/go-ethereum/accounts/external" - "github.com/ethereum/go-ethereum/accounts/keystore" - "github.com/ethereum/go-ethereum/accounts/scwallet" - "github.com/ethereum/go-ethereum/accounts/usbwallet" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" @@ -430,15 +425,8 @@ func (c *Config) parsePersistentNodes(w *bool, path string) []*enode.Node { return nodes } -// AccountConfig determines the settings for scrypt and keydirectory -func (c *Config) AccountConfig() (int, int, string, error) { - scryptN := keystore.StandardScryptN - scryptP := keystore.StandardScryptP - if c.UseLightweightKDF { - scryptN = keystore.LightScryptN - scryptP = keystore.LightScryptP - } - +// KeyDirConfig determines the settings for keydirectory +func (c *Config) KeyDirConfig() (string, error) { var ( keydir string err error @@ -455,71 +443,31 @@ func (c *Config) AccountConfig() (int, int, string, error) { case c.KeyStoreDir != "": keydir, err = filepath.Abs(c.KeyStoreDir) } - return scryptN, scryptP, keydir, err + return keydir, err } -func makeAccountManager(conf *Config) (*accounts.Manager, string, error) { - scryptN, scryptP, keydir, err := conf.AccountConfig() - var ephemeral string +// getKeyStoreDir retrieves the key directory and will create +// and ephemeral one if necessary. +func getKeyStoreDir(conf *Config) (string, bool, error) { + keydir, err := conf.KeyDirConfig() + if err != nil { + return "", false, err + } + isEphemeral := false if keydir == "" { // There is no datadir. keydir, err = ioutil.TempDir("", "go-ethereum-keystore") - ephemeral = keydir + isEphemeral = true } if err != nil { - return nil, "", err + return "", false, err } if err := os.MkdirAll(keydir, 0700); err != nil { - return nil, "", err - } - // Assemble the account manager and supported backends - var backends []accounts.Backend - if len(conf.ExternalSigner) > 0 { - log.Info("Using external signer", "url", conf.ExternalSigner) - if extapi, err := external.NewExternalBackend(conf.ExternalSigner); err == nil { - backends = append(backends, extapi) - } else { - return nil, "", fmt.Errorf("error connecting to external signer: %v", err) - } - } - if len(backends) == 0 { - // For now, we're using EITHER external signer OR local signers. - // If/when we implement some form of lockfile for USB and keystore wallets, - // we can have both, but it's very confusing for the user to see the same - // accounts in both externally and locally, plus very racey. - backends = append(backends, keystore.NewKeyStore(keydir, scryptN, scryptP)) - if conf.USB { - // Start a USB hub for Ledger hardware wallets - if ledgerhub, err := usbwallet.NewLedgerHub(); err != nil { - log.Warn(fmt.Sprintf("Failed to start Ledger hub, disabling: %v", err)) - } else { - backends = append(backends, ledgerhub) - } - // Start a USB hub for Trezor hardware wallets (HID version) - if trezorhub, err := usbwallet.NewTrezorHubWithHID(); err != nil { - log.Warn(fmt.Sprintf("Failed to start HID Trezor hub, disabling: %v", err)) - } else { - backends = append(backends, trezorhub) - } - // Start a USB hub for Trezor hardware wallets (WebUSB version) - if trezorhub, err := usbwallet.NewTrezorHubWithWebUSB(); err != nil { - log.Warn(fmt.Sprintf("Failed to start WebUSB Trezor hub, disabling: %v", err)) - } else { - backends = append(backends, trezorhub) - } - } - if len(conf.SmartCardDaemonPath) > 0 { - // Start a smart card hub - if schub, err := scwallet.NewHub(conf.SmartCardDaemonPath, scwallet.Scheme, keydir); err != nil { - log.Warn(fmt.Sprintf("Failed to start smart card hub, disabling: %v", err)) - } else { - backends = append(backends, schub) - } - } + return "", false, err } - return accounts.NewManager(&accounts.Config{InsecureUnlockAllowed: conf.InsecureUnlockAllowed}, backends...), ephemeral, nil + return keydir, isEphemeral, nil } var warnLock sync.Mutex diff --git a/node/node.go b/node/node.go index 1e65fff1c..ceab1c909 100644 --- a/node/node.go +++ b/node/node.go @@ -42,7 +42,8 @@ type Node struct { config *Config accman *accounts.Manager log log.Logger - ephemKeystore string // if non-empty, the key directory that will be removed by Stop + keyDir string // key store directory + keyDirTemp bool // If true, key directory will be removed by Stop dirLock fileutil.Releaser // prevents concurrent use of instance directory stop chan struct{} // Channel to wait for termination notifications server *p2p.Server // Currently running P2P networking layer @@ -112,14 +113,15 @@ func New(conf *Config) (*Node, error) { if err := node.openDataDir(); err != nil { return nil, err } - // Ensure that the AccountManager method works before the node has started. We rely on - // this in cmd/geth. - am, ephemeralKeystore, err := makeAccountManager(conf) + keyDir, isEphem, err := getKeyStoreDir(conf) if err != nil { return nil, err } - node.accman = am - node.ephemKeystore = ephemeralKeystore + node.keyDir = keyDir + node.keyDirTemp = isEphem + // Creates an empty AccountManager with no backends. Callers (e.g. cmd/geth) + // are required to add the backends later on. + node.accman = accounts.NewManager(&accounts.Config{InsecureUnlockAllowed: conf.InsecureUnlockAllowed}) // Initialize the p2p server. This creates the node key and discovery databases. node.server.Config.PrivateKey = node.config.NodeKey() @@ -233,8 +235,8 @@ func (n *Node) doClose(errs []error) error { if err := n.accman.Close(); err != nil { errs = append(errs, err) } - if n.ephemKeystore != "" { - if err := os.RemoveAll(n.ephemKeystore); err != nil { + if n.keyDirTemp { + if err := os.RemoveAll(n.keyDir); err != nil { errs = append(errs, err) } } @@ -514,6 +516,11 @@ func (n *Node) InstanceDir() string { return n.config.instanceDir() } +// KeyStoreDir retrieves the key directory +func (n *Node) KeyStoreDir() string { + return n.keyDir +} + // AccountManager retrieves the account manager used by the protocol stack. func (n *Node) AccountManager() *accounts.Manager { return n.accman From 62ad17fb0046243255048fbf8cb0882f48d8d850 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Thu, 26 Aug 2021 08:35:37 +0200 Subject: [PATCH 29/73] Revert "eth, internal/ethapi: make RPC block miner field show block sealer correctly (#23312)" (#23466) This reverts commit 57feabea663496109e59df669238398239438fb1. --- eth/api.go | 2 +- internal/ethapi/api.go | 14 ++++++-------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/eth/api.go b/eth/api.go index 0f57128d7..8b96d1f31 100644 --- a/eth/api.go +++ b/eth/api.go @@ -342,7 +342,7 @@ func (api *PrivateDebugAPI) GetBadBlocks(ctx context.Context) ([]*BadBlockArgs, } else { blockRlp = fmt.Sprintf("0x%x", rlpBytes) } - if blockJSON, err = ethapi.RPCMarshalBlock(block, true, true, api.eth.engine); err != nil { + if blockJSON, err = ethapi.RPCMarshalBlock(block, true, true); err != nil { blockJSON = map[string]interface{}{"error": err.Error()} } results = append(results, &BadBlockArgs{ diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 1af98e107..9a82824ad 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -32,7 +32,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/common/math" - "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus/clique" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/consensus/misc" @@ -1176,8 +1175,7 @@ func FormatLogs(logs []vm.StructLog) []StructLogRes { } // RPCMarshalHeader converts the given header to the RPC output . -func RPCMarshalHeader(head *types.Header, engine consensus.Engine) map[string]interface{} { - miner, _ := engine.Author(head) +func RPCMarshalHeader(head *types.Header) map[string]interface{} { result := map[string]interface{}{ "number": (*hexutil.Big)(head.Number), "hash": head.Hash(), @@ -1187,7 +1185,7 @@ func RPCMarshalHeader(head *types.Header, engine consensus.Engine) map[string]in "sha3Uncles": head.UncleHash, "logsBloom": head.Bloom, "stateRoot": head.Root, - "miner": miner, + "miner": head.Coinbase, "difficulty": (*hexutil.Big)(head.Difficulty), "extraData": hexutil.Bytes(head.Extra), "size": hexutil.Uint64(head.Size()), @@ -1208,8 +1206,8 @@ func RPCMarshalHeader(head *types.Header, engine consensus.Engine) map[string]in // RPCMarshalBlock converts the given block to the RPC output which depends on fullTx. If inclTx is true transactions are // returned. When fullTx is true the returned block contains full transaction details, otherwise it will only contain // transaction hashes. -func RPCMarshalBlock(block *types.Block, inclTx bool, fullTx bool, engine consensus.Engine) (map[string]interface{}, error) { - fields := RPCMarshalHeader(block.Header(), engine) +func RPCMarshalBlock(block *types.Block, inclTx bool, fullTx bool) (map[string]interface{}, error) { + fields := RPCMarshalHeader(block.Header()) fields["size"] = hexutil.Uint64(block.Size()) if inclTx { @@ -1244,7 +1242,7 @@ func RPCMarshalBlock(block *types.Block, inclTx bool, fullTx bool, engine consen // rpcMarshalHeader uses the generalized output filler, then adds the total difficulty field, which requires // a `PublicBlockchainAPI`. func (s *PublicBlockChainAPI) rpcMarshalHeader(ctx context.Context, header *types.Header) map[string]interface{} { - fields := RPCMarshalHeader(header, s.b.Engine()) + fields := RPCMarshalHeader(header) fields["totalDifficulty"] = (*hexutil.Big)(s.b.GetTd(ctx, header.Hash())) return fields } @@ -1252,7 +1250,7 @@ func (s *PublicBlockChainAPI) rpcMarshalHeader(ctx context.Context, header *type // rpcMarshalBlock uses the generalized output filler, then adds the total difficulty field, which requires // a `PublicBlockchainAPI`. func (s *PublicBlockChainAPI) rpcMarshalBlock(ctx context.Context, b *types.Block, inclTx bool, fullTx bool) (map[string]interface{}, error) { - fields, err := RPCMarshalBlock(b, inclTx, fullTx, s.b.Engine()) + fields, err := RPCMarshalBlock(b, inclTx, fullTx) if err != nil { return nil, err } From ffae2043f0482e81ac9f5459bf9fceafd74f52e9 Mon Sep 17 00:00:00 2001 From: lightclient <14004106+lightclient@users.noreply.github.com> Date: Fri, 27 Aug 2021 06:11:15 -0600 Subject: [PATCH 30/73] internal/ethapi: support both input and data for personal_sendTransaction (#23476) Currently, setDefaults overwrites the transaction input value if only input is provided. This causes personal_sendTransaction to estimate the gas based on a transaction with empty data. eth_estimateGas never calls setDefaults so it was unaffected by this. --- internal/ethapi/transaction_args.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/ethapi/transaction_args.go b/internal/ethapi/transaction_args.go index 52811b2a9..2d08d3008 100644 --- a/internal/ethapi/transaction_args.go +++ b/internal/ethapi/transaction_args.go @@ -146,6 +146,7 @@ func (args *TransactionArgs) setDefaults(ctx context.Context, b Backend) error { if args.Gas == nil { // These fields are immutable during the estimation, safe to // pass the pointer directly. + data := args.data() callArgs := TransactionArgs{ From: args.From, To: args.To, @@ -153,7 +154,7 @@ func (args *TransactionArgs) setDefaults(ctx context.Context, b Backend) error { MaxFeePerGas: args.MaxFeePerGas, MaxPriorityFeePerGas: args.MaxPriorityFeePerGas, Value: args.Value, - Data: args.Data, + Data: (*hexutil.Bytes)(&data), AccessList: args.AccessList, } pendingBlockNr := rpc.BlockNumberOrHashWithNumber(rpc.PendingBlockNumber) From 3258211f6833394f3ec064154eb214a6e0c02f6c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Mon, 30 Aug 2021 14:13:06 +0200 Subject: [PATCH 31/73] core/vm: rework jumpdest analysis benchmarks (#23499) * core/vm: rework jumpdest analysis benchmarks For BenchmarkJumpdestOpAnalysis use fixed code size of ~1.2MB and classic benchmark loop. * core/vm: clear bitvec in jumpdest analysis benchmark --- core/vm/analysis_test.go | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/core/vm/analysis_test.go b/core/vm/analysis_test.go index 585bb3097..d7f21e04a 100644 --- a/core/vm/analysis_test.go +++ b/core/vm/analysis_test.go @@ -55,9 +55,12 @@ func TestJumpDestAnalysis(t *testing.T) { } } +const analysisCodeSize = 1200 * 1024 + func BenchmarkJumpdestAnalysis_1200k(bench *testing.B) { // 1.4 ms - code := make([]byte, 1200000) + code := make([]byte, analysisCodeSize) + bench.SetBytes(analysisCodeSize) bench.ResetTimer() for i := 0; i < bench.N; i++ { codeBitmap(code) @@ -66,7 +69,8 @@ func BenchmarkJumpdestAnalysis_1200k(bench *testing.B) { } func BenchmarkJumpdestHashing_1200k(bench *testing.B) { // 4 ms - code := make([]byte, 1200000) + code := make([]byte, analysisCodeSize) + bench.SetBytes(analysisCodeSize) bench.ResetTimer() for i := 0; i < bench.N; i++ { crypto.Keccak256Hash(code) @@ -77,13 +81,19 @@ func BenchmarkJumpdestHashing_1200k(bench *testing.B) { func BenchmarkJumpdestOpAnalysis(bench *testing.B) { var op OpCode bencher := func(b *testing.B) { - code := make([]byte, 32*b.N) + code := make([]byte, analysisCodeSize) + b.SetBytes(analysisCodeSize) for i := range code { code[i] = byte(op) } bits := make(bitvec, len(code)/8+1+4) b.ResetTimer() - codeBitmapInternal(code, bits) + for i := 0; i < b.N; i++ { + for j := range bits { + bits[j] = 0 + } + codeBitmapInternal(code, bits) + } } for op = PUSH1; op <= PUSH32; op++ { bench.Run(op.String(), bencher) From f85cf722ffde1b4a309be847ac394f90e9ace8eb Mon Sep 17 00:00:00 2001 From: Fredrik Svantes Date: Tue, 31 Aug 2021 09:02:04 +0200 Subject: [PATCH 32/73] docs: add links to more audits in SECURITY.md (#23482) --- SECURITY.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/SECURITY.md b/SECURITY.md index bdce7b8d2..635c0869f 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -12,6 +12,8 @@ Audit reports are published in the `docs` folder: https://github.com/ethereum/go | ------- | ------- | ----------- | | `geth` | 20170425 | [pdf](https://github.com/ethereum/go-ethereum/blob/master/docs/audits/2017-04-25_Geth-audit_Truesec.pdf) | | `clef` | 20180914 | [pdf](https://github.com/ethereum/go-ethereum/blob/master/docs/audits/2018-09-14_Clef-audit_NCC.pdf) | +| `Discv5` | 20191015 | [pdf](https://github.com/ethereum/go-ethereum/blob/master/docs/audits/2019-10-15_Discv5_audit_LeastAuthority.pdf) | +| `Discv5` | 20200124 | [pdf](https://github.com/ethereum/go-ethereum/blob/master/docs/audits/2020-01-24_DiscV5_audit_Cure53.pdf) | ## Reporting a Vulnerability From 31be5d41d9a8463e0767ddf21f06ce0edd777e6d Mon Sep 17 00:00:00 2001 From: Ikko Ashimine Date: Tue, 31 Aug 2021 17:21:42 +0900 Subject: [PATCH 33/73] core: fix typo in iterator.go (#23502) strorage -> storage --- core/state/snapshot/iterator.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/state/snapshot/iterator.go b/core/state/snapshot/iterator.go index 1d9340bba..c1a196c7f 100644 --- a/core/state/snapshot/iterator.go +++ b/core/state/snapshot/iterator.go @@ -385,7 +385,7 @@ func (it *diskStorageIterator) Hash() common.Hash { return common.BytesToHash(it.it.Key()) // The prefix will be truncated } -// Slot returns the raw strorage slot content the iterator is currently at. +// Slot returns the raw storage slot content the iterator is currently at. func (it *diskStorageIterator) Slot() []byte { return it.it.Value() } From d019e901621e6ce4efe1acc7d4fd6c41fe1349b1 Mon Sep 17 00:00:00 2001 From: Alexander Yastrebov Date: Tue, 31 Aug 2021 10:22:20 +0200 Subject: [PATCH 34/73] common: fixes format verb (#23495) --- common/types.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/common/types.go b/common/types.go index d71535669..2205835cb 100644 --- a/common/types.go +++ b/common/types.go @@ -86,7 +86,7 @@ func (h Hash) String() string { } // Format implements fmt.Formatter. -// Hash supports the %v, %s, %v, %x, %X and %d format verbs. +// Hash supports the %v, %s, %q, %x, %X and %d format verbs. func (h Hash) Format(s fmt.State, c rune) { hexb := make([]byte, 2+len(h)*2) copy(hexb, "0x") @@ -270,7 +270,7 @@ func (a Address) hex() []byte { } // Format implements fmt.Formatter. -// Address supports the %v, %s, %v, %x, %X and %d format verbs. +// Address supports the %v, %s, %q, %x, %X and %d format verbs. func (a Address) Format(s fmt.State, c rune) { switch c { case 'v', 's': From 067084fedab1d50e224e40e6442a7740fc53611a Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Tue, 31 Aug 2021 10:33:39 +0200 Subject: [PATCH 35/73] core: fix race conditions in txpool (#23474) * core: fix race conditions in txpool * core: fixed races in the txpool * core: rebased on master * core: move reheap mutex * core: renamed mutex * core: revert Reheap changes --- core/tx_list.go | 23 ++++++++++++++--------- core/tx_pool.go | 7 ++++++- core/tx_pool_test.go | 7 +++++-- 3 files changed, 25 insertions(+), 12 deletions(-) diff --git a/core/tx_list.go b/core/tx_list.go index 607838ba3..1d5c336a1 100644 --- a/core/tx_list.go +++ b/core/tx_list.go @@ -21,6 +21,8 @@ import ( "math" "math/big" "sort" + "sync" + "sync/atomic" "time" "github.com/ethereum/go-ethereum/common" @@ -478,9 +480,10 @@ func (h *priceHeap) Pop() interface{} { // better candidates for inclusion while in other cases (at the top of the baseFee peak) // the floating heap is better. When baseFee is decreasing they behave similarly. type txPricedList struct { - all *txLookup // Pointer to the map of all transactions - urgent, floating priceHeap // Heaps of prices of all the stored **remote** transactions - stales int // Number of stale price points to (re-heap trigger) + all *txLookup // Pointer to the map of all transactions + urgent, floating priceHeap // Heaps of prices of all the stored **remote** transactions + stales int64 // Number of stale price points to (re-heap trigger) + reheapMu sync.Mutex // Mutex asserts that only one routine is reheaping the list } const ( @@ -510,8 +513,8 @@ func (l *txPricedList) Put(tx *types.Transaction, local bool) { // the heap if a large enough ratio of transactions go stale. func (l *txPricedList) Removed(count int) { // Bump the stale counter, but exit if still too low (< 25%) - l.stales += count - if l.stales <= (len(l.urgent.list)+len(l.floating.list))/4 { + stales := atomic.AddInt64(&l.stales, int64(count)) + if int(stales) <= (len(l.urgent.list)+len(l.floating.list))/4 { return } // Seems we've reached a critical number of stale transactions, reheap @@ -535,7 +538,7 @@ func (l *txPricedList) underpricedFor(h *priceHeap, tx *types.Transaction) bool for len(h.list) > 0 { head := h.list[0] if l.all.GetRemote(head.Hash()) == nil { // Removed or migrated - l.stales-- + atomic.AddInt64(&l.stales, -1) heap.Pop(h) continue } @@ -561,7 +564,7 @@ func (l *txPricedList) Discard(slots int, force bool) (types.Transactions, bool) // Discard stale transactions if found during cleanup tx := heap.Pop(&l.urgent).(*types.Transaction) if l.all.GetRemote(tx.Hash()) == nil { // Removed or migrated - l.stales-- + atomic.AddInt64(&l.stales, -1) continue } // Non stale transaction found, move to floating heap @@ -574,7 +577,7 @@ func (l *txPricedList) Discard(slots int, force bool) (types.Transactions, bool) // Discard stale transactions if found during cleanup tx := heap.Pop(&l.floating).(*types.Transaction) if l.all.GetRemote(tx.Hash()) == nil { // Removed or migrated - l.stales-- + atomic.AddInt64(&l.stales, -1) continue } // Non stale transaction found, discard it @@ -594,8 +597,10 @@ func (l *txPricedList) Discard(slots int, force bool) (types.Transactions, bool) // Reheap forcibly rebuilds the heap based on the current remote transaction set. func (l *txPricedList) Reheap() { + l.reheapMu.Lock() + defer l.reheapMu.Unlock() start := time.Now() - l.stales = 0 + atomic.StoreInt64(&l.stales, 0) l.urgent.list = make([]*types.Transaction, 0, l.all.RemoteCount()) l.all.Range(func(hash common.Hash, tx *types.Transaction, local bool) bool { l.urgent.list = append(l.urgent.list, tx) diff --git a/core/tx_pool.go b/core/tx_pool.go index ee56dae88..5c70cf170 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -22,6 +22,7 @@ import ( "math/big" "sort" "sync" + "sync/atomic" "time" "github.com/ethereum/go-ethereum/common" @@ -264,6 +265,7 @@ type TxPool struct { reorgDoneCh chan chan struct{} reorgShutdownCh chan struct{} // requests shutdown of scheduleReorgLoop wg sync.WaitGroup // tracks loop, scheduleReorgLoop + initDoneCh chan struct{} // is closed once the pool is initialized (for tests) changesSinceReorg int // A counter for how many drops we've performed in-between reorg. } @@ -294,6 +296,7 @@ func NewTxPool(config TxPoolConfig, chainconfig *params.ChainConfig, chain block queueTxEventCh: make(chan *types.Transaction), reorgDoneCh: make(chan chan struct{}), reorgShutdownCh: make(chan struct{}), + initDoneCh: make(chan struct{}), gasPrice: new(big.Int).SetUint64(config.PriceLimit), } pool.locals = newAccountSet(pool.signer) @@ -347,6 +350,8 @@ func (pool *TxPool) loop() { defer evict.Stop() defer journal.Stop() + // Notify tests that the init phase is done + close(pool.initDoneCh) for { select { // Handle ChainHeadEvent @@ -365,8 +370,8 @@ func (pool *TxPool) loop() { case <-report.C: pool.mu.RLock() pending, queued := pool.stats() - stales := pool.priced.stales pool.mu.RUnlock() + stales := int(atomic.LoadInt64(&pool.priced.stales)) if pending != prevPending || queued != prevQueued || stales != prevStales { log.Debug("Transaction pool status report", "executable", pending, "queued", queued, "stales", stales) diff --git a/core/tx_pool_test.go b/core/tx_pool_test.go index f86f64bf2..ce2fe7755 100644 --- a/core/tx_pool_test.go +++ b/core/tx_pool_test.go @@ -24,6 +24,7 @@ import ( "math/big" "math/rand" "os" + "sync/atomic" "testing" "time" @@ -64,7 +65,7 @@ type testBlockChain struct { func (bc *testBlockChain) CurrentBlock() *types.Block { return types.NewBlock(&types.Header{ - GasLimit: bc.gasLimit, + GasLimit: atomic.LoadUint64(&bc.gasLimit), }, nil, nil, nil, trie.NewStackTrie(nil)) } @@ -123,6 +124,8 @@ func setupTxPoolWithConfig(config *params.ChainConfig) (*TxPool, *ecdsa.PrivateK key, _ := crypto.GenerateKey() pool := NewTxPool(testTxPoolConfig, config, blockchain) + // wait for the pool to initialize + <-pool.initDoneCh return pool, key } @@ -625,7 +628,7 @@ func TestTransactionDropping(t *testing.T) { t.Errorf("total transaction mismatch: have %d, want %d", pool.all.Count(), 4) } // Reduce the block gas limit, check that invalidated transactions are dropped - pool.chain.(*testBlockChain).gasLimit = 100 + atomic.StoreUint64(&pool.chain.(*testBlockChain).gasLimit, 100) <-pool.requestReset(nil, nil) if _, ok := pool.pending[account].txs.items[tx0.Nonce()]; !ok { From 4d88974864c3ee84a24b1088064162b8dbab8ad5 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Thu, 2 Sep 2021 09:22:43 +0200 Subject: [PATCH 36/73] cmd/evm: add tests for evm t8n (#23507) --- cmd/evm/README.md | 2 +- cmd/evm/internal/t8ntool/transition.go | 11 +- cmd/evm/main.go | 2 +- cmd/evm/t8n_test.go | 211 ++++++++++++++++++++++++ cmd/evm/testdata/1/exp.json | 43 +++++ cmd/evm/testdata/13/exp.json | 3 + cmd/evm/testdata/13/exp2.json | 38 +++++ cmd/evm/testdata/13/signed_txs.rlp | 1 + cmd/evm/testdata/14/env.json | 2 +- cmd/evm/testdata/14/env.uncles.json | 2 +- cmd/evm/testdata/14/exp.json | 11 ++ cmd/evm/testdata/14/exp2.json | 11 ++ cmd/evm/testdata/14/readme.md | 48 +++--- cmd/evm/testdata/15/signed_txs.rlp.json | 4 + cmd/evm/testdata/3/exp.json | 37 +++++ cmd/evm/testdata/5/exp.json | 22 +++ internal/cmdtest/test_cmd.go | 7 + 17 files changed, 424 insertions(+), 31 deletions(-) create mode 100644 cmd/evm/t8n_test.go create mode 100644 cmd/evm/testdata/1/exp.json create mode 100644 cmd/evm/testdata/13/exp.json create mode 100644 cmd/evm/testdata/13/exp2.json create mode 100644 cmd/evm/testdata/13/signed_txs.rlp create mode 100644 cmd/evm/testdata/14/exp.json create mode 100644 cmd/evm/testdata/14/exp2.json create mode 100644 cmd/evm/testdata/15/signed_txs.rlp.json create mode 100644 cmd/evm/testdata/3/exp.json create mode 100644 cmd/evm/testdata/5/exp.json diff --git a/cmd/evm/README.md b/cmd/evm/README.md index d5257069f..1a029ab70 100644 --- a/cmd/evm/README.md +++ b/cmd/evm/README.md @@ -208,7 +208,7 @@ Example: ] } ``` -When applying this, using a reward of `0x08` +When applying this, using a reward of `0x80` Output: ```json { diff --git a/cmd/evm/internal/t8ntool/transition.go b/cmd/evm/internal/t8ntool/transition.go index 9fdbbe29c..545962269 100644 --- a/cmd/evm/internal/t8ntool/transition.go +++ b/cmd/evm/internal/t8ntool/transition.go @@ -65,10 +65,15 @@ func (n *NumberedError) Error() string { return fmt.Sprintf("ERROR(%d): %v", n.errorCode, n.err.Error()) } -func (n *NumberedError) Code() int { +func (n *NumberedError) ExitCode() int { return n.errorCode } +// compile-time conformance test +var ( + _ cli.ExitCoder = (*NumberedError)(nil) +) + type input struct { Alloc core.GenesisAlloc `json:"alloc,omitempty"` Env *stEnv `json:"env,omitempty"` @@ -409,7 +414,7 @@ func dispatchOutput(ctx *cli.Context, baseDir string, result *ExecutionResult, a return err } if len(stdOutObject) > 0 { - b, err := json.MarshalIndent(stdOutObject, "", " ") + b, err := json.MarshalIndent(stdOutObject, "", " ") if err != nil { return NewError(ErrorJson, fmt.Errorf("failed marshalling output: %v", err)) } @@ -417,7 +422,7 @@ func dispatchOutput(ctx *cli.Context, baseDir string, result *ExecutionResult, a os.Stdout.Write([]byte("\n")) } if len(stdErrObject) > 0 { - b, err := json.MarshalIndent(stdErrObject, "", " ") + b, err := json.MarshalIndent(stdErrObject, "", " ") if err != nil { return NewError(ErrorJson, fmt.Errorf("failed marshalling output: %v", err)) } diff --git a/cmd/evm/main.go b/cmd/evm/main.go index b9c0d17f3..c32b6c382 100644 --- a/cmd/evm/main.go +++ b/cmd/evm/main.go @@ -195,7 +195,7 @@ func main() { if err := app.Run(os.Args); err != nil { code := 1 if ec, ok := err.(*t8ntool.NumberedError); ok { - code = ec.Code() + code = ec.ExitCode() } fmt.Fprintln(os.Stderr, err) os.Exit(code) diff --git a/cmd/evm/t8n_test.go b/cmd/evm/t8n_test.go new file mode 100644 index 000000000..5af7ee19c --- /dev/null +++ b/cmd/evm/t8n_test.go @@ -0,0 +1,211 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + "reflect" + "strings" + "testing" + + "github.com/docker/docker/pkg/reexec" + "github.com/ethereum/go-ethereum/internal/cmdtest" +) + +func TestMain(m *testing.M) { + // Run the app if we've been exec'd as "ethkey-test" in runEthkey. + reexec.Register("evm-test", func() { + if err := app.Run(os.Args); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + os.Exit(0) + }) + // check if we have been reexec'd + if reexec.Init() { + return + } + os.Exit(m.Run()) +} + +type testT8n struct { + *cmdtest.TestCmd +} + +type t8nInput struct { + inAlloc string + inTxs string + inEnv string + stFork string + stReward string +} + +func (args *t8nInput) get(base string) []string { + var out []string + if opt := args.inAlloc; opt != "" { + out = append(out, "--input.alloc") + out = append(out, fmt.Sprintf("%v/%v", base, opt)) + } + if opt := args.inTxs; opt != "" { + out = append(out, "--input.txs") + out = append(out, fmt.Sprintf("%v/%v", base, opt)) + } + if opt := args.inEnv; opt != "" { + out = append(out, "--input.env") + out = append(out, fmt.Sprintf("%v/%v", base, opt)) + } + if opt := args.stFork; opt != "" { + out = append(out, "--state.fork", opt) + } + if opt := args.stReward; opt != "" { + out = append(out, "--state.reward", opt) + } + return out +} + +type t8nOutput struct { + alloc bool + result bool + body bool +} + +func (args *t8nOutput) get() (out []string) { + out = append(out, "t8n") + if args.body { + out = append(out, "--output.body", "stdout") + } else { + out = append(out, "--output.body", "") // empty means ignore + } + if args.result { + out = append(out, "--output.result", "stdout") + } else { + out = append(out, "--output.result", "") + } + if args.alloc { + out = append(out, "--output.alloc", "stdout") + } else { + out = append(out, "--output.alloc", "") + } + return out +} + +func TestT8n(t *testing.T) { + tt := new(testT8n) + tt.TestCmd = cmdtest.NewTestCmd(t, tt) + for i, tc := range []struct { + base string + input t8nInput + output t8nOutput + expExitCode int + expOut string + }{ + { // Test exit (3) on bad config + base: "./testdata/1", + input: t8nInput{ + "alloc.json", "txs.json", "env.json", "Frontier+1346", "", + }, + output: t8nOutput{alloc: true, result: true}, + expExitCode: 3, + }, + { + base: "./testdata/1", + input: t8nInput{ + "alloc.json", "txs.json", "env.json", "Byzantium", "", + }, + output: t8nOutput{alloc: true, result: true}, + expOut: "exp.json", + }, + { // blockhash test + base: "./testdata/3", + input: t8nInput{ + "alloc.json", "txs.json", "env.json", "Berlin", "", + }, + output: t8nOutput{alloc: true, result: true}, + expOut: "exp.json", + }, + { // missing blockhash test + base: "./testdata/4", + input: t8nInput{ + "alloc.json", "txs.json", "env.json", "Berlin", "", + }, + output: t8nOutput{alloc: true, result: true}, + expExitCode: 4, + }, + { // Ommer test + base: "./testdata/5", + input: t8nInput{ + "alloc.json", "txs.json", "env.json", "Byzantium", "0x80", + }, + output: t8nOutput{alloc: true, result: true}, + expOut: "exp.json", + }, + { // Sign json transactions + base: "./testdata/13", + input: t8nInput{ + "alloc.json", "txs.json", "env.json", "London", "", + }, + output: t8nOutput{body: true}, + expOut: "exp.json", + }, + { // Already signed transactions + base: "./testdata/13", + input: t8nInput{ + "alloc.json", "signed_txs.rlp", "env.json", "London", "", + }, + output: t8nOutput{result: true}, + expOut: "exp2.json", + }, + { // Difficulty calculation - no uncles + base: "./testdata/14", + input: t8nInput{ + "alloc.json", "txs.json", "env.json", "London", "", + }, + output: t8nOutput{result: true}, + expOut: "exp.json", + }, + { // Difficulty calculation - with uncles + base: "./testdata/14", + input: t8nInput{ + "alloc.json", "txs.json", "env.uncles.json", "London", "", + }, + output: t8nOutput{result: true}, + expOut: "exp2.json", + }, + } { + + args := append(tc.output.get(), tc.input.get(tc.base)...) + tt.Run("evm-test", args...) + tt.Logf("args: %v\n", strings.Join(args, " ")) + // Compare the expected output, if provided + if tc.expOut != "" { + want, err := os.ReadFile(fmt.Sprintf("%v/%v", tc.base, tc.expOut)) + if err != nil { + t.Fatalf("test %d: could not read expected output: %v", i, err) + } + have := tt.Output() + ok, err := cmpJson(have, want) + switch { + case err != nil: + t.Fatalf("test %d, json parsing failed: %v", i, err) + case !ok: + t.Fatalf("test %d: output wrong, have \n%v\nwant\n%v\n", i, string(have), string(want)) + } + } + tt.WaitExit() + if have, want := tt.ExitStatus(), tc.expExitCode; have != want { + t.Fatalf("test %d: wrong exit code, have %d, want %d", i, have, want) + } + } +} + +// cmpJson compares the JSON in two byte slices. +func cmpJson(a, b []byte) (bool, error) { + var j, j2 interface{} + if err := json.Unmarshal(a, &j); err != nil { + return false, err + } + if err := json.Unmarshal(b, &j2); err != nil { + return false, err + } + return reflect.DeepEqual(j2, j), nil +} diff --git a/cmd/evm/testdata/1/exp.json b/cmd/evm/testdata/1/exp.json new file mode 100644 index 000000000..17d2f8267 --- /dev/null +++ b/cmd/evm/testdata/1/exp.json @@ -0,0 +1,43 @@ +{ + "alloc": { + "0x8a8eafb1cf62bfbeb1741769dae1a9dd47996192": { + "balance": "0xfeed1a9d", + "nonce": "0x1" + }, + "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": { + "balance": "0x5ffd4878be161d74", + "nonce": "0xac" + }, + "0xc94f5374fce5edbc8e2a8697c15331677e6ebf0b": { + "balance": "0xa410" + } + }, + "result": { + "stateRoot": "0x84208a19bc2b46ada7445180c1db162be5b39b9abc8c0a54b05d32943eae4e13", + "txRoot": "0xc4761fd7b87ff2364c7c60b6c5c8d02e522e815328aaea3f20e3b7b7ef52c42d", + "receiptRoot": "0x056b23fbba480696b65fe5a59b8f2148a1299103c4f57df839233af2cf4ca2d2", + "logsHash": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "receipts": [ + { + "root": "0x", + "status": "0x1", + "cumulativeGasUsed": "0x5208", + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "logs": null, + "transactionHash": "0x0557bacce3375c98d806609b8d5043072f0b6a8bae45ae5a67a00d3a1a18d673", + "contractAddress": "0x0000000000000000000000000000000000000000", + "gasUsed": "0x5208", + "blockHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "transactionIndex": "0x0" + } + ], + "rejected": [ + { + "index": 1, + "error": "nonce too low: address 0x8A8eAFb1cf62BfBeb1741769DAE1a9dd47996192, tx: 0 state: 1" + } + ], + "currentDifficulty": "0x20000" + } +} \ No newline at end of file diff --git a/cmd/evm/testdata/13/exp.json b/cmd/evm/testdata/13/exp.json new file mode 100644 index 000000000..2b049dfb2 --- /dev/null +++ b/cmd/evm/testdata/13/exp.json @@ -0,0 +1,3 @@ +{ + "body": "0xf8d2b86702f864010180820fa08284d09411111111111111111111111111111111111111118080c001a0b7dfab36232379bb3d1497a4f91c1966b1f932eae3ade107bf5d723b9cb474e0a06261c359a10f2132f126d250485b90cf20f30340801244a08ef6142ab33d1904b86702f864010280820fa08284d09411111111111111111111111111111111111111118080c080a0d4ec563b6568cd42d998fc4134b36933c6568d01533b5adf08769270243c6c7fa072bf7c21eac6bbeae5143371eef26d5e279637f3bd73482b55979d76d935b1e9" +} diff --git a/cmd/evm/testdata/13/exp2.json b/cmd/evm/testdata/13/exp2.json new file mode 100644 index 000000000..01ab59e84 --- /dev/null +++ b/cmd/evm/testdata/13/exp2.json @@ -0,0 +1,38 @@ +{ + "result": { + "stateRoot": "0xe4b924a6adb5959fccf769d5b7bb2f6359e26d1e76a2443c5a91a36d826aef61", + "txRoot": "0x013509c8563d41c0ae4bf38f2d6d19fc6512a1d0d6be045079c8c9f68bf45f9d", + "receiptRoot": "0xa532a08aa9f62431d6fe5d924951b8efb86ed3c54d06fee77788c3767dd13420", + "logsHash": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "receipts": [ + { + "type": "0x2", + "root": "0x", + "status": "0x0", + "cumulativeGasUsed": "0x84d0", + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "logs": null, + "transactionHash": "0xa98a24882ea90916c6a86da650fbc6b14238e46f0af04a131ce92be897507476", + "contractAddress": "0x0000000000000000000000000000000000000000", + "gasUsed": "0x84d0", + "blockHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "transactionIndex": "0x0" + }, + { + "type": "0x2", + "root": "0x", + "status": "0x0", + "cumulativeGasUsed": "0x109a0", + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "logs": null, + "transactionHash": "0x36bad80acce7040c45fd32764b5c2b2d2e6f778669fb41791f73f546d56e739a", + "contractAddress": "0x0000000000000000000000000000000000000000", + "gasUsed": "0x84d0", + "blockHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "transactionIndex": "0x1" + } + ], + "currentDifficulty": "0x20000" + } +} diff --git a/cmd/evm/testdata/13/signed_txs.rlp b/cmd/evm/testdata/13/signed_txs.rlp new file mode 100644 index 000000000..9d1157ea4 --- /dev/null +++ b/cmd/evm/testdata/13/signed_txs.rlp @@ -0,0 +1 @@ +"0xf8d2b86702f864010180820fa08284d09411111111111111111111111111111111111111118080c001a0b7dfab36232379bb3d1497a4f91c1966b1f932eae3ade107bf5d723b9cb474e0a06261c359a10f2132f126d250485b90cf20f30340801244a08ef6142ab33d1904b86702f864010280820fa08284d09411111111111111111111111111111111111111118080c080a0d4ec563b6568cd42d998fc4134b36933c6568d01533b5adf08769270243c6c7fa072bf7c21eac6bbeae5143371eef26d5e279637f3bd73482b55979d76d935b1e9" \ No newline at end of file diff --git a/cmd/evm/testdata/14/env.json b/cmd/evm/testdata/14/env.json index 6b7457ffd..0bf1c5cf4 100644 --- a/cmd/evm/testdata/14/env.json +++ b/cmd/evm/testdata/14/env.json @@ -3,7 +3,7 @@ "currentGasLimit": "0x750a163df65e8a", "currentBaseFee": "0x500", "currentNumber": "12800000", - "currentTimestamp": "10015", + "currentTimestamp": "100015", "parentTimestamp" : "99999", "parentDifficulty" : "0x2000000000000" } diff --git a/cmd/evm/testdata/14/env.uncles.json b/cmd/evm/testdata/14/env.uncles.json index 9c2149b39..83811b95e 100644 --- a/cmd/evm/testdata/14/env.uncles.json +++ b/cmd/evm/testdata/14/env.uncles.json @@ -3,7 +3,7 @@ "currentGasLimit": "0x750a163df65e8a", "currentBaseFee": "0x500", "currentNumber": "12800000", - "currentTimestamp": "10035", + "currentTimestamp": "100035", "parentTimestamp" : "99999", "parentDifficulty" : "0x2000000000000", "parentUncleHash" : "0x000000000000000000000000000000000000000000000000000000000000beef" diff --git a/cmd/evm/testdata/14/exp.json b/cmd/evm/testdata/14/exp.json new file mode 100644 index 000000000..bbe6a1317 --- /dev/null +++ b/cmd/evm/testdata/14/exp.json @@ -0,0 +1,11 @@ +{ + "result": { + "stateRoot": "0x6f058887ca01549716789c380ede95aecc510e6d1fdc4dbf67d053c7c07f4bdc", + "txRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "receiptRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "logsHash": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "currentDifficulty": "0x2000020000000", + "receipts": [] + } +} \ No newline at end of file diff --git a/cmd/evm/testdata/14/exp2.json b/cmd/evm/testdata/14/exp2.json new file mode 100644 index 000000000..195c738d9 --- /dev/null +++ b/cmd/evm/testdata/14/exp2.json @@ -0,0 +1,11 @@ +{ + "result": { + "stateRoot": "0x6f058887ca01549716789c380ede95aecc510e6d1fdc4dbf67d053c7c07f4bdc", + "txRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "receiptRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "logsHash": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "receipts": [], + "currentDifficulty": "0x1ff8020000000" + } +} \ No newline at end of file diff --git a/cmd/evm/testdata/14/readme.md b/cmd/evm/testdata/14/readme.md index ce31abb80..9d0dc9569 100644 --- a/cmd/evm/testdata/14/readme.md +++ b/cmd/evm/testdata/14/readme.md @@ -5,37 +5,37 @@ This test shows how the `evm t8n` can be used to calculate the (ethash) difficul Calculating it (with an empty set of txs) using `London` rules (and no provided unclehash for the parent block): ``` [user@work evm]$ ./evm t8n --input.alloc=./testdata/14/alloc.json --input.txs=./testdata/14/txs.json --input.env=./testdata/14/env.json --output.result=stdout --state.fork=London -INFO [08-08|17:35:46.876] Trie dumping started root=6f0588..7f4bdc -INFO [08-08|17:35:46.876] Trie dumping complete accounts=2 elapsed="89.313µs" -INFO [08-08|17:35:46.877] Wrote file file=alloc.json +INFO [08-30|20:43:09.352] Trie dumping started root=6f0588..7f4bdc +INFO [08-30|20:43:09.352] Trie dumping complete accounts=2 elapsed="82.533µs" +INFO [08-30|20:43:09.352] Wrote file file=alloc.json { - "result": { - "stateRoot": "0x6f058887ca01549716789c380ede95aecc510e6d1fdc4dbf67d053c7c07f4bdc", - "txRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", - "receiptRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", - "logsHash": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", - "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "receipts": [], - "currentDifficulty": 3311729559732224 - } + "result": { + "stateRoot": "0x6f058887ca01549716789c380ede95aecc510e6d1fdc4dbf67d053c7c07f4bdc", + "txRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "receiptRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "logsHash": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "receipts": [], + "currentDifficulty": "0x2000020000000" + } } ``` Same thing, but this time providing a non-empty (and non-`emptyKeccak`) unclehash, which leads to a slightly different result: ``` [user@work evm]$ ./evm t8n --input.alloc=./testdata/14/alloc.json --input.txs=./testdata/14/txs.json --input.env=./testdata/14/env.uncles.json --output.result=stdout --state.fork=London -INFO [08-08|17:35:49.232] Trie dumping started root=6f0588..7f4bdc -INFO [08-08|17:35:49.232] Trie dumping complete accounts=2 elapsed="83.069µs" -INFO [08-08|17:35:49.233] Wrote file file=alloc.json +INFO [08-30|20:44:33.102] Trie dumping started root=6f0588..7f4bdc +INFO [08-30|20:44:33.102] Trie dumping complete accounts=2 elapsed="72.91µs" +INFO [08-30|20:44:33.102] Wrote file file=alloc.json { - "result": { - "stateRoot": "0x6f058887ca01549716789c380ede95aecc510e6d1fdc4dbf67d053c7c07f4bdc", - "txRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", - "receiptRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", - "logsHash": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", - "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", - "receipts": [], - "currentDifficulty": 3311179803918336 - } + "result": { + "stateRoot": "0x6f058887ca01549716789c380ede95aecc510e6d1fdc4dbf67d053c7c07f4bdc", + "txRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "receiptRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "logsHash": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "receipts": [], + "currentDifficulty": "0x1ff8020000000" + } } ``` diff --git a/cmd/evm/testdata/15/signed_txs.rlp.json b/cmd/evm/testdata/15/signed_txs.rlp.json new file mode 100644 index 000000000..187f40f24 --- /dev/null +++ b/cmd/evm/testdata/15/signed_txs.rlp.json @@ -0,0 +1,4 @@ +{ + "txsRlp" : "0xf8d2b86702f864010180820fa08284d09411111111111111111111111111111111111111118080c001a0b7dfab36232379bb3d1497a4f91c1966b1f932eae3ade107bf5d723b9cb474e0a06261c359a10f2132f126d250485b90cf20f30340801244a08ef6142ab33d1904b86702f864010280820fa08284d09411111111111111111111111111111111111111118080c080a0d4ec563b6568cd42d998fc4134b36933c6568d01533b5adf08769270243c6c7fa072bf7c21eac6bbeae5143371eef26d5e279637f3bd73482b55979d76d935b1e9" +} + diff --git a/cmd/evm/testdata/3/exp.json b/cmd/evm/testdata/3/exp.json new file mode 100644 index 000000000..ade09e9ac --- /dev/null +++ b/cmd/evm/testdata/3/exp.json @@ -0,0 +1,37 @@ +{ + "alloc": { + "0x095e7baea6a6c7c4c2dfeb977efac326af552d87": { + "code": "0x600140", + "balance": "0xde0b6b3a76586a0" + }, + "0x2adc25665018aa1fe0e6bc666dac8fc2697ff9ba": { + "balance": "0x521f" + }, + "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": { + "balance": "0xde0b6b3a7622741", + "nonce": "0x1" + } + }, + "result": { + "stateRoot": "0xb7341da3f9f762a6884eaa186c32942734c146b609efee11c4b0214c44857ea1", + "txRoot": "0x75e61774a2ff58cbe32653420256c7f44bc715715a423b0b746d5c622979af6b", + "receiptRoot": "0xd0d26df80374a327c025d405ebadc752b1bbd089d864801ae78ab704bcad8086", + "logsHash": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "receipts": [ + { + "root": "0x", + "status": "0x1", + "cumulativeGasUsed": "0x521f", + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "logs": null, + "transactionHash": "0x72fadbef39cd251a437eea619cfeda752271a5faaaa2147df012e112159ffb81", + "contractAddress": "0x0000000000000000000000000000000000000000", + "gasUsed": "0x521f", + "blockHash": "0x0000000000000000000000000000000000000000000000000000000000000000", + "transactionIndex": "0x0" + } + ], + "currentDifficulty": "0x20000" + } +} diff --git a/cmd/evm/testdata/5/exp.json b/cmd/evm/testdata/5/exp.json new file mode 100644 index 000000000..6340d4cc3 --- /dev/null +++ b/cmd/evm/testdata/5/exp.json @@ -0,0 +1,22 @@ +{ + "alloc": { + "0xaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa": { + "balance": "0x88" + }, + "0xbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb": { + "balance": "0x70" + }, + "0xcccccccccccccccccccccccccccccccccccccccc": { + "balance": "0x60" + } + }, + "result": { + "stateRoot": "0xa7312add33811645c6aa65d928a1a4f49d65d448801912c069a0aa8fe9c1f393", + "txRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "receiptRoot": "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421", + "logsHash": "0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "logsBloom": "0x00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000", + "receipts": [], + "currentDifficulty": "0x20000" + } +} diff --git a/internal/cmdtest/test_cmd.go b/internal/cmdtest/test_cmd.go index 82ad9c15b..d75db41e0 100644 --- a/internal/cmdtest/test_cmd.go +++ b/internal/cmdtest/test_cmd.go @@ -118,6 +118,13 @@ func (tt *TestCmd) Expect(tplsource string) { tt.Logf("Matched stdout text:\n%s", want) } +// Output reads all output from stdout, and returns the data. +func (tt *TestCmd) Output() []byte { + var buf []byte + tt.withKillTimeout(func() { buf, _ = io.ReadAll(tt.stdout) }) + return buf +} + func (tt *TestCmd) matchExactOutput(want []byte) error { buf := make([]byte, len(want)) n := 0 From de2c44ab5c9c5c2e76039388d632841c1c16bff3 Mon Sep 17 00:00:00 2001 From: aaronbuchwald Date: Tue, 7 Sep 2021 00:25:54 -0400 Subject: [PATCH 37/73] trie: add missing copyright header (#23530) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add missing copyright header * Update stacktrie_test.go Co-authored-by: Péter Szilágyi --- trie/stacktrie_test.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/trie/stacktrie_test.go b/trie/stacktrie_test.go index ccdf389d5..bd2574d5d 100644 --- a/trie/stacktrie_test.go +++ b/trie/stacktrie_test.go @@ -1,3 +1,19 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + package trie import ( From 79575302257b1f5c43b36826d0d5572a11e04987 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 7 Sep 2021 09:22:40 +0200 Subject: [PATCH 38/73] docs: add post-mortem (#23518) * docs: add post-mortem * Update docs/postmortems/2021-08-22-split-postmortem.md Co-authored-by: Sina Mahmoodi <1591639+s1na@users.noreply.github.com> * Update docs/postmortems/2021-08-22-split-postmortem.md Co-authored-by: Sina Mahmoodi <1591639+s1na@users.noreply.github.com> * Update docs/postmortems/2021-08-22-split-postmortem.md Co-authored-by: Sina Mahmoodi <1591639+s1na@users.noreply.github.com> * Update docs/postmortems/2021-08-22-split-postmortem.md * Update 2021-08-22-split-postmortem.md * Update docs/postmortems/2021-08-22-split-postmortem.md Co-authored-by: Sina Mahmoodi <1591639+s1na@users.noreply.github.com> Co-authored-by: Sina Mahmoodi <1591639+s1na@users.noreply.github.com> --- .../2021-08-22-split-postmortem.md | 266 ++++++++++++++++++ 1 file changed, 266 insertions(+) create mode 100644 docs/postmortems/2021-08-22-split-postmortem.md diff --git a/docs/postmortems/2021-08-22-split-postmortem.md b/docs/postmortems/2021-08-22-split-postmortem.md new file mode 100644 index 000000000..888f7d77c --- /dev/null +++ b/docs/postmortems/2021-08-22-split-postmortem.md @@ -0,0 +1,266 @@ +# Minority split 2021-08-27 post mortem + +This is a post-mortem concerning the minority split that occurred on Ethereum mainnet on block [13107518](https://etherscan.io/block/13107518), at which a minority chain split occurred. + +## Timeline + + +- 2021-08-17: Guido Vranken submitted bounty report. Investigation started, root cause identified, patch variations discussed. +- 2021-08-18: Made public announcement over twitter about upcoming security release upcoming Tuesday. Downstream projects were also notified about the upcoming patch-release. +- 2021-08-24: Released [v1.10.8](https://github.com/ethereum/go-ethereum/releases/tag/v1.10.8) containing the fix on Tuesday morning (CET). Erigon released [v2021.08.04](https://github.com/ledgerwatch/erigon/releases/tag/v2021.08.04). +- 2021-08-27: At 12:50:07 UTC, issue exploited. Analysis started roughly 30m later, + + + +## Bounty report + +### 2021-08-17 RETURNDATA corruption via datacopy + +On 2021-08-17, Guido Vranken submitted a report to bounty@ethereum.org. This co-incided with a geth-meetup in Berlin, so the geth team could fairly quickly analyse the issue. + +He submitted a proof of concept which called the `dataCopy` precompile, where the input slice and output slice were overlapping but shifted. Doing a `copy` where the `src` and `dest` overlaps is not a problem in itself, however, the `returnData`slice was _also_ using the same memory as a backing-array. + +#### Technical details + +During CALL-variants, `geth` does not copy the input. This was changed at one point, to avoid a DoS attack reported by Hubert Ritzdorf, to avoid copying data a lot on repeated `CALL`s -- essentially combating a DoS via `malloc`. Further, the datacopy precompile also does not copy the data, but just returns the same slice. This is fine so far. + +After the execution of `dataCopy`, we copy the `ret` into the designated memory area, and this is what causes a problem. Because we're copying a slice of memory over a slice of memory, and this operation modifies (shifts) the data in the source -- the `ret`. So this means we wind up with corrupted returndata. + + +``` +1. Calling datacopy + + memory: [0, 1, 2, 3, 4] + in (mem[0:4]) : [0,1,2,3] + out (mem[1:5]): [1,2,3,4] + +2. dataCopy returns + + returndata (==in, mem[0:4]): [0,1,2,3] + +3. Copy in -> out + + => memory: [0,0,1,2,3] + => returndata: [0,0,1,2] +``` + + +#### Summary + +A memory-corruption bug within the EVM can cause a consensus error, where vulnerable nodes obtain a different `stateRoot` when processing a maliciously crafted transaction. This, in turn, would lead to the chain being split: mainnet splitting in two forks. + +#### Handling + +On the evening of 17th, we discussed options how to handle it. We made a state test to reproduce the issue, and verified that neither `openethereum`, `nethermind` nor `besu` were affected by the same vulnerability, and started a full-sync with a patched version of `geth`. + +It was decided that in this specific instance, it would be possible to make a public announcement and a patch release: + +- The fix can be made pretty 'generically', e.g. always copying data on input to precompiles. +- The flaw is pretty difficult to find, given a generic fix in the call. The attacker needs to figure out that it concerns the precompiles, specifically the datcopy, and that it concerns the `RETURNDATA` buffer rather than the regular memory, and lastly the special circumstances to trigger it (overlapping but shifted input/output). + +Since we had merged the removal of `ETH65`, if the entire network were to upgrade, then nodes which have not yet implemented `ETH66` would be cut off from the network. After further discussions, we decided to: + +- Announce an upcoming security release on Tuesday (August 24th), via Twitter and official channels, plus reach out to downstream projects. +- Temporarily revert the `ETH65`-removal. +- Place the fix into the PR optimizing the jumpdest analysis [233381](https://github.com/ethereum/go-ethereum/pull/23381). +- After 4-8 weeks, release details about the vulnerability. + + +## Exploit + +At block [13107518](https://etherscan.io/block/13107518), mined at (Aug-27-2021 12:50:07 PM +UTC), a minority chain split occurred. The discord user @AlexSSD7 notified the allcoredevs-channel on the Eth R&D discord, on Aug 27 13:09 UTC. + + +At 14:09 UTC, it was confirmed that the transaction `0x1cb6fb36633d270edefc04d048145b4298e67b8aa82a9e5ec4aa1435dd770ce4` had triggered the bug, leading to a minority-split of the chain. The term 'minority split' means that the majority of miners continued to mine on the correct chain. + +At 14:17 UTC, @mhswende tweeted out about the issue [2]. + +The attack was sent from an account funded from Tornado cash. + +It was also found that the same attack had been carried out on the BSC chain at roughly the same time -- at a block mined [12 minutes earlier](https://bscscan.com/tx/0xf667f820631f6adbd04a4c92274374034a3e41fa9057dc42cb4e787535136dce), at Aug-27-2021 12:38:30 PM +UTC. + +The blocks on the 'bad' chain were investigated, and Tim Beiko reached out to those mining operators on the minority chain who could be identified via block extradata. + + +## Lessons learned + + +### Disclosure decision + +The geth-team have an official policy regarding [vulnerability disclosure](https://geth.ethereum.org/docs/vulnerabilities/vulnerabilities). + +> The primary goal for the Geth team is the health of the Ethereum network as a whole, and the decision whether or not to publish details about a serious vulnerability boils down to minimizing the risk and/or impact of discovery and exploitation. + +In this case, it was decided that public pre-announce + patch would likely lead to sufficient update-window for a critical mass of nodes/miners to upgrade in time before it could be exploited. In hindsight, this was a dangerous decision, and it's unlikely that the same decision would be reached were a similar incident to happen again. + + +### Disclosure path + +Several subprojects were informed about the upcoming security patch: + +- Polygon/Matic +- MEV +- Avalanche +- Erigon +- BSC +- EWF +- Quorum +- ETC +- xDAI + +However, some were 'lost', and only notified later + +- Optimism +- Summa +- Harmony + +Action point: create a low-volume geth-announce@ethereum.org email list where dependent projects/operators can receive public announcements. +- This has been done. If you wish to receive release- and security announcements, sign up [here](https://groups.google.com/a/ethereum.org/g/geth-announce/about) + +### Fork monitoring + +The fork monitor behaved 'ok' during the incident, but had to be restarted during the evening. + +Action point: improve the resiliency of the forkmon, which is currently not performing great when many nodes are connected. + +Action point: enable push-based alerts to be sent from the forkmon, to speed up the fork detection. + + +## Links + +- [1] https://twitter.com/go_ethereum/status/1428051458763763721 +- [2] https://twitter.com/mhswende/status/1431259601530458112 + + +## Appendix + +### Subprojects + + +The projects were sent variations of the following text: +``` +We have identified a security issue with go-ethereum, and will issue a +new release (v1.10.8) on Tuesday next week. + +At this point, we will not disclose details about the issue, but +recommend downstream/dependent projects to be ready to take actions to +upgrade to the latest go-ethereum codebase. More information about the +issue will be disclosed at a later date. + +https://twitter.com/go_ethereum/status/1428051458763763721 + +``` +### Patch + +```diff +diff --git a/core/vm/instructions.go b/core/vm/instructions.go +index f7ef2f900e..6c8c6e6e6f 100644 +--- a/core/vm/instructions.go ++++ b/core/vm/instructions.go +@@ -669,6 +669,7 @@ func opCall(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byt + } + stack.push(&temp) + if err == nil || err == ErrExecutionReverted { ++ ret = common.CopyBytes(ret) + scope.Memory.Set(retOffset.Uint64(), retSize.Uint64(), ret) + } + scope.Contract.Gas += returnGas +@@ -703,6 +704,7 @@ func opCallCode(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([ + } + stack.push(&temp) + if err == nil || err == ErrExecutionReverted { ++ ret = common.CopyBytes(ret) + scope.Memory.Set(retOffset.Uint64(), retSize.Uint64(), ret) + } + scope.Contract.Gas += returnGas +@@ -730,6 +732,7 @@ func opDelegateCall(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext + } + stack.push(&temp) + if err == nil || err == ErrExecutionReverted { ++ ret = common.CopyBytes(ret) + scope.Memory.Set(retOffset.Uint64(), retSize.Uint64(), ret) + } + scope.Contract.Gas += returnGas +@@ -757,6 +760,7 @@ func opStaticCall(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) + } + stack.push(&temp) + if err == nil || err == ErrExecutionReverted { ++ ret = common.CopyBytes(ret) + scope.Memory.Set(retOffset.Uint64(), retSize.Uint64(), ret) + } + scope.Contract.Gas += returnGas +diff --git a/core/vm/interpreter.go b/core/vm/interpreter.go +index 9cf0c4e2c1..9fb83799c9 100644 +--- a/core/vm/interpreter.go ++++ b/core/vm/interpreter.go +@@ -262,7 +262,7 @@ func (in *EVMInterpreter) Run(contract *Contract, input []byte, readOnly bool) ( + // if the operation clears the return data (e.g. it has returning data) + // set the last return to the result of the operation. + if operation.returns { +- in.returnData = common.CopyBytes(res) ++ in.returnData = res + } + + switch { +``` + +### Statetest to test for the issue + +```json +{ + "trigger-issue": { + "env": { + "currentCoinbase": "b94f5374fce5edbc8e2a8697c15331677e6ebf0b", + "currentDifficulty": "0x20000", + "currentGasLimit": "0x26e1f476fe1e22", + "currentNumber": "0x1", + "currentTimestamp": "0x3e8", + "previousHash": "0x0000000000000000000000000000000000000000000000000000000000000000" + }, + "pre": { + "0x00000000000000000000000000000000000000bb": { + "code": "0x6001600053600260015360036002536004600353600560045360066005536006600260066000600060047f7ef0367e633852132a0ebbf70eb714015dd44bc82e1e55a96ef1389c999c1bcaf13d600060003e596000208055", + "storage": {}, + "balance": "0x5", + "nonce": "0x0" + }, + "0xa94f5374fce5edbc8e2a8697c15331677e6ebf0b": { + "code": "0x", + "storage": {}, + "balance": "0xffffffff", + "nonce": "0x0" + } + }, + "transaction": { + "gasPrice": "0x1", + "nonce": "0x0", + "to": "0x00000000000000000000000000000000000000bb", + "data": [ + "0x" + ], + "gasLimit": [ + "0x7a1200" + ], + "value": [ + "0x01" + ], + "secretKey": "0x45a915e4d060149eb4365960e6a7a45f334393093061116b197e3240065ff2d8" + }, + "out": "0x", + "post": { + "Berlin": [ + { + "hash": "2a38a040bab1e1fa499253d98b2fd363e5756ecc52db47dd59af7116c068368c", + "logs": "1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347", + "indexes": { + "data": 0, + "gas": 0, + "value": 0 + } + } + ] + } + } +} +``` + From ca5bc676d10d324dddf63bb24932385d0c0b9418 Mon Sep 17 00:00:00 2001 From: Zachinquarantine Date: Tue, 7 Sep 2021 04:51:33 -0400 Subject: [PATCH 39/73] internal/debug: remove deprecated flags (#23368) * internal/debug: remove deprecated flags The removed flags are removed in the main portion of geth, this removes it internally too. * internal/debug: remove legacy --debug and legacy --backtrace flag * Update flags.go Co-authored-by: Martin Holst Swende --- internal/debug/flags.go | 68 +---------------------------------------- 1 file changed, 1 insertion(+), 67 deletions(-) diff --git a/internal/debug/flags.go b/internal/debug/flags.go index 126ee09a7..3aa990adf 100644 --- a/internal/debug/flags.go +++ b/internal/debug/flags.go @@ -90,39 +90,6 @@ var ( Name: "trace", Usage: "Write execution trace to the given file", } - // (Deprecated April 2020) - legacyPprofPortFlag = cli.IntFlag{ - Name: "pprofport", - Usage: "pprof HTTP server listening port (deprecated, use --pprof.port)", - Value: 6060, - } - legacyPprofAddrFlag = cli.StringFlag{ - Name: "pprofaddr", - Usage: "pprof HTTP server listening interface (deprecated, use --pprof.addr)", - Value: "127.0.0.1", - } - legacyMemprofilerateFlag = cli.IntFlag{ - Name: "memprofilerate", - Usage: "Turn on memory profiling with the given rate (deprecated, use --pprof.memprofilerate)", - Value: runtime.MemProfileRate, - } - legacyBlockprofilerateFlag = cli.IntFlag{ - Name: "blockprofilerate", - Usage: "Turn on block profiling with the given rate (deprecated, use --pprof.blockprofilerate)", - } - legacyCpuprofileFlag = cli.StringFlag{ - Name: "cpuprofile", - Usage: "Write CPU profile to the given file (deprecated, use --pprof.cpuprofile)", - } - legacyBacktraceAtFlag = cli.StringFlag{ - Name: "backtrace", - Usage: "Request a stack trace at a specific logging statement (e.g. \"block.go:271\") (deprecated, use --log.backtrace)", - Value: "", - } - legacyDebugFlag = cli.BoolFlag{ - Name: "debug", - Usage: "Prepends log messages with call-site location (file and line number) (deprecated, use --log.debug)", - } ) // Flags holds all command-line flags required for debugging. @@ -141,17 +108,6 @@ var Flags = []cli.Flag{ traceFlag, } -// This is the list of deprecated debugging flags. -var DeprecatedFlags = []cli.Flag{ - legacyPprofPortFlag, - legacyPprofAddrFlag, - legacyMemprofilerateFlag, - legacyBlockprofilerateFlag, - legacyCpuprofileFlag, - legacyBacktraceAtFlag, - legacyDebugFlag, -} - var glogger *log.GlogHandler func init() { @@ -183,45 +139,23 @@ func Setup(ctx *cli.Context) error { glogger.Vmodule(vmodule) debug := ctx.GlobalBool(debugFlag.Name) - if ctx.GlobalIsSet(legacyDebugFlag.Name) { - debug = ctx.GlobalBool(legacyDebugFlag.Name) - log.Warn("The flag --debug is deprecated and will be removed in the future, please use --log.debug") - } if ctx.GlobalIsSet(debugFlag.Name) { debug = ctx.GlobalBool(debugFlag.Name) } log.PrintOrigins(debug) backtrace := ctx.GlobalString(backtraceAtFlag.Name) - if b := ctx.GlobalString(legacyBacktraceAtFlag.Name); b != "" { - backtrace = b - log.Warn("The flag --backtrace is deprecated and will be removed in the future, please use --log.backtrace") - } - if b := ctx.GlobalString(backtraceAtFlag.Name); b != "" { - backtrace = b - } glogger.BacktraceAt(backtrace) log.Root().SetHandler(glogger) // profiling, tracing runtime.MemProfileRate = memprofilerateFlag.Value - if ctx.GlobalIsSet(legacyMemprofilerateFlag.Name) { - runtime.MemProfileRate = ctx.GlobalInt(legacyMemprofilerateFlag.Name) - log.Warn("The flag --memprofilerate is deprecated and will be removed in the future, please use --pprof.memprofilerate") - } if ctx.GlobalIsSet(memprofilerateFlag.Name) { runtime.MemProfileRate = ctx.GlobalInt(memprofilerateFlag.Name) } - blockProfileRate := blockprofilerateFlag.Value - if ctx.GlobalIsSet(legacyBlockprofilerateFlag.Name) { - blockProfileRate = ctx.GlobalInt(legacyBlockprofilerateFlag.Name) - log.Warn("The flag --blockprofilerate is deprecated and will be removed in the future, please use --pprof.blockprofilerate") - } - if ctx.GlobalIsSet(blockprofilerateFlag.Name) { - blockProfileRate = ctx.GlobalInt(blockprofilerateFlag.Name) - } + blockProfileRate := ctx.GlobalInt(blockprofilerateFlag.Name) Handler.SetBlockProfileRate(blockProfileRate) if traceFile := ctx.GlobalString(traceFlag.Name); traceFile != "" { From 9a0df80bbcfd849a9a56ecb0937bf8dbea8a8c79 Mon Sep 17 00:00:00 2001 From: Zachinquarantine Date: Tue, 7 Sep 2021 04:52:58 -0400 Subject: [PATCH 40/73] cmd/geth, cmd/utils: remove legacy rpc flags (#23358) * remove rpc flags * remove legacy rpc flags * remove legacy rpc flags * remove legacy rpc commands * (hopefully) fix most of the build errors * fix build errors https://app.travis-ci.com/github/ethereum/go-ethereum/jobs/530318686 * cmd/utils: fix syntax error * empty commit to unbreak travis ci * fix syntax error * syntax fixes * syntax fixes * fix fixes "cmd/geth/usage.go:234:7: expected '(', found init (typecheck)" * fix * various fixes in usage.go * various fixes in flags.go * adds extra space reverts the spacing to how it was before I resolved the merge conflict * more fixes in usage.go * fix fix for cmd/geth/usage.go:243:17: expected operand, found ':=' (typecheck) in travis * Update cmd/utils/flags.go Co-authored-by: Martin Holst Swende * fix error fixes these errors: cmd/utils/flags_legacy.go:21:2: "strings" imported but not used (typecheck) "strings" ^ cmd/utils/flags_legacy.go:24:2: "github.com/ethereum/go-ethereum/node" imported but not used (typecheck) "github.com/ethereum/go-ethereum/node" ^ * goimports Co-authored-by: Martin Holst Swende --- cmd/geth/main.go | 6 ------ cmd/geth/usage.go | 7 ------- cmd/utils/flags.go | 24 ------------------------ cmd/utils/flags_legacy.go | 31 ------------------------------- 4 files changed, 68 deletions(-) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index cfc1a62b6..d003d590e 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -158,12 +158,6 @@ var ( utils.HTTPPortFlag, utils.HTTPCORSDomainFlag, utils.HTTPVirtualHostsFlag, - utils.LegacyRPCEnabledFlag, - utils.LegacyRPCListenAddrFlag, - utils.LegacyRPCPortFlag, - utils.LegacyRPCCORSDomainFlag, - utils.LegacyRPCVirtualHostsFlag, - utils.LegacyRPCApiFlag, utils.GraphQLEnabledFlag, utils.GraphQLCORSDomainFlag, utils.GraphQLVirtualHostsFlag, diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index f39e4f402..16446f83e 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -218,13 +218,6 @@ var AppHelpFlagGroups = []flags.FlagGroup{ Name: "ALIASED (deprecated)", Flags: []cli.Flag{ utils.NoUSBFlag, - utils.LegacyRPCEnabledFlag, - utils.LegacyRPCListenAddrFlag, - utils.LegacyRPCPortFlag, - utils.LegacyRPCCORSDomainFlag, - utils.LegacyRPCVirtualHostsFlag, - utils.LegacyRPCApiFlag, - utils.LegacyMinerGasTargetFlag, }, }, { diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 2953a0032..52c15c68e 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -920,14 +920,6 @@ func SplitAndTrim(input string) (ret []string) { // setHTTP creates the HTTP RPC listener interface string from the set // command line flags, returning empty if the HTTP endpoint is disabled. func setHTTP(ctx *cli.Context, cfg *node.Config) { - if ctx.GlobalBool(LegacyRPCEnabledFlag.Name) && cfg.HTTPHost == "" { - log.Warn("The flag --rpc is deprecated and will be removed June 2021, please use --http") - cfg.HTTPHost = "127.0.0.1" - if ctx.GlobalIsSet(LegacyRPCListenAddrFlag.Name) { - cfg.HTTPHost = ctx.GlobalString(LegacyRPCListenAddrFlag.Name) - log.Warn("The flag --rpcaddr is deprecated and will be removed June 2021, please use --http.addr") - } - } if ctx.GlobalBool(HTTPEnabledFlag.Name) && cfg.HTTPHost == "" { cfg.HTTPHost = "127.0.0.1" if ctx.GlobalIsSet(HTTPListenAddrFlag.Name) { @@ -935,34 +927,18 @@ func setHTTP(ctx *cli.Context, cfg *node.Config) { } } - if ctx.GlobalIsSet(LegacyRPCPortFlag.Name) { - cfg.HTTPPort = ctx.GlobalInt(LegacyRPCPortFlag.Name) - log.Warn("The flag --rpcport is deprecated and will be removed June 2021, please use --http.port") - } if ctx.GlobalIsSet(HTTPPortFlag.Name) { cfg.HTTPPort = ctx.GlobalInt(HTTPPortFlag.Name) } - if ctx.GlobalIsSet(LegacyRPCCORSDomainFlag.Name) { - cfg.HTTPCors = SplitAndTrim(ctx.GlobalString(LegacyRPCCORSDomainFlag.Name)) - log.Warn("The flag --rpccorsdomain is deprecated and will be removed June 2021, please use --http.corsdomain") - } if ctx.GlobalIsSet(HTTPCORSDomainFlag.Name) { cfg.HTTPCors = SplitAndTrim(ctx.GlobalString(HTTPCORSDomainFlag.Name)) } - if ctx.GlobalIsSet(LegacyRPCApiFlag.Name) { - cfg.HTTPModules = SplitAndTrim(ctx.GlobalString(LegacyRPCApiFlag.Name)) - log.Warn("The flag --rpcapi is deprecated and will be removed June 2021, please use --http.api") - } if ctx.GlobalIsSet(HTTPApiFlag.Name) { cfg.HTTPModules = SplitAndTrim(ctx.GlobalString(HTTPApiFlag.Name)) } - if ctx.GlobalIsSet(LegacyRPCVirtualHostsFlag.Name) { - cfg.HTTPVirtualHosts = SplitAndTrim(ctx.GlobalString(LegacyRPCVirtualHostsFlag.Name)) - log.Warn("The flag --rpcvhosts is deprecated and will be removed June 2021, please use --http.vhosts") - } if ctx.GlobalIsSet(HTTPVirtualHostsFlag.Name) { cfg.HTTPVirtualHosts = SplitAndTrim(ctx.GlobalString(HTTPVirtualHostsFlag.Name)) } diff --git a/cmd/utils/flags_legacy.go b/cmd/utils/flags_legacy.go index 627f77475..a0f64f609 100644 --- a/cmd/utils/flags_legacy.go +++ b/cmd/utils/flags_legacy.go @@ -18,10 +18,8 @@ package utils import ( "fmt" - "strings" "github.com/ethereum/go-ethereum/eth/ethconfig" - "github.com/ethereum/go-ethereum/node" "gopkg.in/urfave/cli.v1" ) @@ -45,35 +43,6 @@ var ( Name: "nousb", Usage: "Disables monitoring for and managing USB hardware wallets (deprecated)", } - LegacyRPCEnabledFlag = cli.BoolFlag{ - Name: "rpc", - Usage: "Enable the HTTP-RPC server (deprecated and will be removed June 2021, use --http)", - } - LegacyRPCListenAddrFlag = cli.StringFlag{ - Name: "rpcaddr", - Usage: "HTTP-RPC server listening interface (deprecated and will be removed June 2021, use --http.addr)", - Value: node.DefaultHTTPHost, - } - LegacyRPCPortFlag = cli.IntFlag{ - Name: "rpcport", - Usage: "HTTP-RPC server listening port (deprecated and will be removed June 2021, use --http.port)", - Value: node.DefaultHTTPPort, - } - LegacyRPCCORSDomainFlag = cli.StringFlag{ - Name: "rpccorsdomain", - Usage: "Comma separated list of domains from which to accept cross origin requests (browser enforced) (deprecated and will be removed June 2021, use --http.corsdomain)", - Value: "", - } - LegacyRPCVirtualHostsFlag = cli.StringFlag{ - Name: "rpcvhosts", - Usage: "Comma separated list of virtual hostnames from which to accept requests (server enforced). Accepts '*' wildcard. (deprecated and will be removed June 2021, use --http.vhosts)", - Value: strings.Join(node.DefaultConfig.HTTPVirtualHosts, ","), - } - LegacyRPCApiFlag = cli.StringFlag{ - Name: "rpcapi", - Usage: "API's offered over the HTTP-RPC interface (deprecated and will be removed June 2021, use --http.api)", - Value: "", - } // (Deprecated July 2021, shown in aliased flags section) LegacyMinerGasTargetFlag = cli.Uint64Flag{ Name: "miner.gastarget", From 794c6133efa2c7e8376d9d141c900ea541790bce Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 7 Sep 2021 12:31:17 +0200 Subject: [PATCH 41/73] core/rawdb: freezer batch write (#23462) This change is a rewrite of the freezer code. When writing ancient chain data to the freezer, the previous version first encoded each individual item to a temporary buffer, then wrote the buffer. For small item sizes (for example, in the block hash freezer table), this strategy causes a lot of system calls for writing tiny chunks of data. It also allocated a lot of temporary []byte buffers. In the new version, we instead encode multiple items into a re-useable batch buffer, which is then written to the file all at once. This avoids performing a system call for every inserted item. To make the internal batching work, the ancient database API had to be changed. While integrating this new API in BlockChain.InsertReceiptChain, additional optimizations were also added there. Co-authored-by: Felix Lange --- core/blockchain.go | 203 +++++++-------- core/blockchain_test.go | 121 ++++++--- core/error.go | 2 + core/rawdb/accessors_chain.go | 92 +++++-- core/rawdb/accessors_chain_test.go | 146 ++++++++++- core/rawdb/database.go | 12 +- core/rawdb/freezer.go | 232 ++++++++++------- core/rawdb/freezer_batch.go | 248 ++++++++++++++++++ core/rawdb/freezer_table.go | 169 +++++-------- core/rawdb/freezer_table_test.go | 390 +++++++++++++++-------------- core/rawdb/freezer_test.go | 301 ++++++++++++++++++++++ core/rawdb/table.go | 7 +- ethdb/database.go | 16 +- 13 files changed, 1349 insertions(+), 590 deletions(-) create mode 100644 core/rawdb/freezer_batch.go create mode 100644 core/rawdb/freezer_test.go diff --git a/core/blockchain.go b/core/blockchain.go index e14291aa5..66bc395c9 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -207,8 +207,7 @@ type BlockChain struct { processor Processor // Block transaction processor interface vmConfig vm.Config - shouldPreserve func(*types.Block) bool // Function used to determine whether should preserve the given block. - terminateInsert func(common.Hash, uint64) bool // Testing hook used to terminate ancient receipt chain insertion. + shouldPreserve func(*types.Block) bool // Function used to determine whether should preserve the given block. } // NewBlockChain returns a fully initialised block chain using information @@ -1085,38 +1084,6 @@ const ( SideStatTy ) -// truncateAncient rewinds the blockchain to the specified header and deletes all -// data in the ancient store that exceeds the specified header. -func (bc *BlockChain) truncateAncient(head uint64) error { - frozen, err := bc.db.Ancients() - if err != nil { - return err - } - // Short circuit if there is no data to truncate in ancient store. - if frozen <= head+1 { - return nil - } - // Truncate all the data in the freezer beyond the specified head - if err := bc.db.TruncateAncients(head + 1); err != nil { - return err - } - // Clear out any stale content from the caches - bc.hc.headerCache.Purge() - bc.hc.tdCache.Purge() - bc.hc.numberCache.Purge() - - // Clear out any stale content from the caches - bc.bodyCache.Purge() - bc.bodyRLPCache.Purge() - bc.receiptsCache.Purge() - bc.blockCache.Purge() - bc.txLookupCache.Purge() - bc.futureBlocks.Purge() - - log.Info("Rewind ancient data", "number", head) - return nil -} - // numberHash is just a container for a number and a hash, to represent a block type numberHash struct { number uint64 @@ -1155,12 +1122,14 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ var ( stats = struct{ processed, ignored int32 }{} start = time.Now() - size = 0 + size = int64(0) ) + // updateHead updates the head fast sync block if the inserted blocks are better // and returns an indicator whether the inserted blocks are canonical. updateHead := func(head *types.Block) bool { bc.chainmu.Lock() + defer bc.chainmu.Unlock() // Rewind may have occurred, skip in that case. if bc.CurrentHeader().Number.Cmp(head.Number()) >= 0 { @@ -1169,68 +1138,63 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ rawdb.WriteHeadFastBlockHash(bc.db, head.Hash()) bc.currentFastBlock.Store(head) headFastBlockGauge.Update(int64(head.NumberU64())) - bc.chainmu.Unlock() return true } } - bc.chainmu.Unlock() return false } + // writeAncient writes blockchain and corresponding receipt chain into ancient store. // // this function only accepts canonical chain data. All side chain will be reverted // eventually. writeAncient := func(blockChain types.Blocks, receiptChain []types.Receipts) (int, error) { - var ( - previous = bc.CurrentFastBlock() - batch = bc.db.NewBatch() - ) - // If any error occurs before updating the head or we are inserting a side chain, - // all the data written this time wll be rolled back. - defer func() { - if previous != nil { - if err := bc.truncateAncient(previous.NumberU64()); err != nil { - log.Crit("Truncate ancient store failed", "err", err) - } - } - }() - var deleted []*numberHash - for i, block := range blockChain { - // Short circuit insertion if shutting down or processing failed - if bc.insertStopped() { - return 0, errInsertionInterrupted - } - // Short circuit insertion if it is required(used in testing only) - if bc.terminateInsert != nil && bc.terminateInsert(block.Hash(), block.NumberU64()) { - return i, errors.New("insertion is terminated for testing purpose") - } - // Short circuit if the owner header is unknown - if !bc.HasHeader(block.Hash(), block.NumberU64()) { - return i, fmt.Errorf("containing header #%d [%x..] unknown", block.Number(), block.Hash().Bytes()[:4]) - } - if block.NumberU64() == 1 { - // Make sure to write the genesis into the freezer - if frozen, _ := bc.db.Ancients(); frozen == 0 { - h := rawdb.ReadCanonicalHash(bc.db, 0) - b := rawdb.ReadBlock(bc.db, h, 0) - size += rawdb.WriteAncientBlock(bc.db, b, rawdb.ReadReceipts(bc.db, h, 0, bc.chainConfig), rawdb.ReadTd(bc.db, h, 0)) - log.Info("Wrote genesis to ancients") - } - } - // Flush data into ancient database. - size += rawdb.WriteAncientBlock(bc.db, block, receiptChain[i], bc.GetTd(block.Hash(), block.NumberU64())) + first := blockChain[0] + last := blockChain[len(blockChain)-1] - // Write tx indices if any condition is satisfied: - // * If user requires to reserve all tx indices(txlookuplimit=0) - // * If all ancient tx indices are required to be reserved(txlookuplimit is even higher than ancientlimit) - // * If block number is large enough to be regarded as a recent block - // It means blocks below the ancientLimit-txlookupLimit won't be indexed. - // - // But if the `TxIndexTail` is not nil, e.g. Geth is initialized with - // an external ancient database, during the setup, blockchain will start - // a background routine to re-indexed all indices in [ancients - txlookupLimit, ancients) - // range. In this case, all tx indices of newly imported blocks should be - // generated. + // Ensure genesis is in ancients. + if first.NumberU64() == 1 { + if frozen, _ := bc.db.Ancients(); frozen == 0 { + b := bc.genesisBlock + td := bc.genesisBlock.Difficulty() + writeSize, err := rawdb.WriteAncientBlocks(bc.db, []*types.Block{b}, []types.Receipts{nil}, td) + size += writeSize + if err != nil { + log.Error("Error writing genesis to ancients", "err", err) + return 0, err + } + log.Info("Wrote genesis to ancients") + } + } + // Before writing the blocks to the ancients, we need to ensure that + // they correspond to the what the headerchain 'expects'. + // We only check the last block/header, since it's a contiguous chain. + if !bc.HasHeader(last.Hash(), last.NumberU64()) { + return 0, fmt.Errorf("containing header #%d [%x..] unknown", last.Number(), last.Hash().Bytes()[:4]) + } + + // Write all chain data to ancients. + td := bc.GetTd(first.Hash(), first.NumberU64()) + writeSize, err := rawdb.WriteAncientBlocks(bc.db, blockChain, receiptChain, td) + size += writeSize + if err != nil { + log.Error("Error importing chain data to ancients", "err", err) + return 0, err + } + + // Write tx indices if any condition is satisfied: + // * If user requires to reserve all tx indices(txlookuplimit=0) + // * If all ancient tx indices are required to be reserved(txlookuplimit is even higher than ancientlimit) + // * If block number is large enough to be regarded as a recent block + // It means blocks below the ancientLimit-txlookupLimit won't be indexed. + // + // But if the `TxIndexTail` is not nil, e.g. Geth is initialized with + // an external ancient database, during the setup, blockchain will start + // a background routine to re-indexed all indices in [ancients - txlookupLimit, ancients) + // range. In this case, all tx indices of newly imported blocks should be + // generated. + var batch = bc.db.NewBatch() + for _, block := range blockChain { if bc.txLookupLimit == 0 || ancientLimit <= bc.txLookupLimit || block.NumberU64() >= ancientLimit-bc.txLookupLimit { rawdb.WriteTxLookupEntriesByBlock(batch, block) } else if rawdb.ReadTxIndexTail(bc.db) != nil { @@ -1238,51 +1202,50 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ } stats.processed++ } + // Flush all tx-lookup index data. - size += batch.ValueSize() + size += int64(batch.ValueSize()) if err := batch.Write(); err != nil { + // The tx index data could not be written. + // Roll back the ancient store update. + fastBlock := bc.CurrentFastBlock().NumberU64() + if err := bc.db.TruncateAncients(fastBlock + 1); err != nil { + log.Error("Can't truncate ancient store after failed insert", "err", err) + } return 0, err } - batch.Reset() // Sync the ancient store explicitly to ensure all data has been flushed to disk. if err := bc.db.Sync(); err != nil { return 0, err } + + // Update the current fast block because all block data is now present in DB. + previousFastBlock := bc.CurrentFastBlock().NumberU64() if !updateHead(blockChain[len(blockChain)-1]) { - return 0, errors.New("side blocks can't be accepted as the ancient chain data") - } - previous = nil // disable rollback explicitly - - // Wipe out canonical block data. - for _, nh := range deleted { - rawdb.DeleteBlockWithoutNumber(batch, nh.hash, nh.number) - rawdb.DeleteCanonicalHash(batch, nh.number) - } - for _, block := range blockChain { - // Always keep genesis block in active database. - if block.NumberU64() != 0 { - rawdb.DeleteBlockWithoutNumber(batch, block.Hash(), block.NumberU64()) - rawdb.DeleteCanonicalHash(batch, block.NumberU64()) + // We end up here if the header chain has reorg'ed, and the blocks/receipts + // don't match the canonical chain. + if err := bc.db.TruncateAncients(previousFastBlock + 1); err != nil { + log.Error("Can't truncate ancient store after failed insert", "err", err) } + return 0, errSideChainReceipts } - if err := batch.Write(); err != nil { - return 0, err - } + + // Delete block data from the main database. batch.Reset() - - // Wipe out side chain too. - for _, nh := range deleted { - for _, hash := range rawdb.ReadAllHashes(bc.db, nh.number) { - rawdb.DeleteBlock(batch, hash, nh.number) - } - } + canonHashes := make(map[common.Hash]struct{}) for _, block := range blockChain { - // Always keep genesis block in active database. - if block.NumberU64() != 0 { - for _, hash := range rawdb.ReadAllHashes(bc.db, block.NumberU64()) { - rawdb.DeleteBlock(batch, hash, block.NumberU64()) - } + canonHashes[block.Hash()] = struct{}{} + if block.NumberU64() == 0 { + continue + } + rawdb.DeleteCanonicalHash(batch, block.NumberU64()) + rawdb.DeleteBlockWithoutNumber(batch, block.Hash(), block.NumberU64()) + } + // Delete side chain hash-to-number mappings. + for _, nh := range rawdb.ReadAllHashesInRange(bc.db, first.NumberU64(), last.NumberU64()) { + if _, canon := canonHashes[nh.Hash]; !canon { + rawdb.DeleteHeader(batch, nh.Hash, nh.Number) } } if err := batch.Write(); err != nil { @@ -1290,6 +1253,7 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ } return 0, nil } + // writeLive writes blockchain and corresponding receipt chain into active store. writeLive := func(blockChain types.Blocks, receiptChain []types.Receipts) (int, error) { skipPresenceCheck := false @@ -1327,7 +1291,7 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ if err := batch.Write(); err != nil { return 0, err } - size += batch.ValueSize() + size += int64(batch.ValueSize()) batch.Reset() } stats.processed++ @@ -1336,7 +1300,7 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ // we can ensure all components of body is completed(body, receipts, // tx indexes) if batch.ValueSize() > 0 { - size += batch.ValueSize() + size += int64(batch.ValueSize()) if err := batch.Write(); err != nil { return 0, err } @@ -1344,6 +1308,7 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ updateHead(blockChain[len(blockChain)-1]) return 0, nil } + // Write downloaded chain data and corresponding receipt chain data if len(ancientBlocks) > 0 { if n, err := writeAncient(ancientBlocks, ancientReceipts); err != nil { diff --git a/core/blockchain_test.go b/core/blockchain_test.go index 4e5df633b..8d94f17aa 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -670,6 +670,7 @@ func TestFastVsFullChains(t *testing.T) { if n, err := ancient.InsertReceiptChain(blocks, receipts, uint64(len(blocks)/2)); err != nil { t.Fatalf("failed to insert receipt %d: %v", n, err) } + // Iterate over all chain data components, and cross reference for i := 0; i < len(blocks); i++ { num, hash := blocks[i].NumberU64(), blocks[i].Hash() @@ -693,10 +694,27 @@ func TestFastVsFullChains(t *testing.T) { } else if types.CalcUncleHash(fblock.Uncles()) != types.CalcUncleHash(arblock.Uncles()) || types.CalcUncleHash(anblock.Uncles()) != types.CalcUncleHash(arblock.Uncles()) { t.Errorf("block #%d [%x]: uncles mismatch: fastdb %v, ancientdb %v, archivedb %v", num, hash, fblock.Uncles(), anblock, arblock.Uncles()) } - if freceipts, anreceipts, areceipts := rawdb.ReadReceipts(fastDb, hash, *rawdb.ReadHeaderNumber(fastDb, hash), fast.Config()), rawdb.ReadReceipts(ancientDb, hash, *rawdb.ReadHeaderNumber(ancientDb, hash), fast.Config()), rawdb.ReadReceipts(archiveDb, hash, *rawdb.ReadHeaderNumber(archiveDb, hash), fast.Config()); types.DeriveSha(freceipts, trie.NewStackTrie(nil)) != types.DeriveSha(areceipts, trie.NewStackTrie(nil)) { + + // Check receipts. + freceipts := rawdb.ReadReceipts(fastDb, hash, num, fast.Config()) + anreceipts := rawdb.ReadReceipts(ancientDb, hash, num, fast.Config()) + areceipts := rawdb.ReadReceipts(archiveDb, hash, num, fast.Config()) + if types.DeriveSha(freceipts, trie.NewStackTrie(nil)) != types.DeriveSha(areceipts, trie.NewStackTrie(nil)) { t.Errorf("block #%d [%x]: receipts mismatch: fastdb %v, ancientdb %v, archivedb %v", num, hash, freceipts, anreceipts, areceipts) } + + // Check that hash-to-number mappings are present in all databases. + if m := rawdb.ReadHeaderNumber(fastDb, hash); m == nil || *m != num { + t.Errorf("block #%d [%x]: wrong hash-to-number mapping in fastdb: %v", num, hash, m) + } + if m := rawdb.ReadHeaderNumber(ancientDb, hash); m == nil || *m != num { + t.Errorf("block #%d [%x]: wrong hash-to-number mapping in ancientdb: %v", num, hash, m) + } + if m := rawdb.ReadHeaderNumber(archiveDb, hash); m == nil || *m != num { + t.Errorf("block #%d [%x]: wrong hash-to-number mapping in archivedb: %v", num, hash, m) + } } + // Check that the canonical chains are the same between the databases for i := 0; i < len(blocks)+1; i++ { if fhash, ahash := rawdb.ReadCanonicalHash(fastDb, uint64(i)), rawdb.ReadCanonicalHash(archiveDb, uint64(i)); fhash != ahash { @@ -1639,20 +1657,34 @@ func TestBlockchainRecovery(t *testing.T) { } } -func TestIncompleteAncientReceiptChainInsertion(t *testing.T) { - // Configure and generate a sample block chain - var ( - gendb = rawdb.NewMemoryDatabase() - key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - address = crypto.PubkeyToAddress(key.PublicKey) - funds = big.NewInt(1000000000) - gspec = &Genesis{Config: params.TestChainConfig, Alloc: GenesisAlloc{address: {Balance: funds}}} - genesis = gspec.MustCommit(gendb) - ) - height := uint64(1024) - blocks, receipts := GenerateChain(gspec.Config, genesis, ethash.NewFaker(), gendb, int(height), nil) +// This test checks that InsertReceiptChain will roll back correctly when attempting to insert a side chain. +func TestInsertReceiptChainRollback(t *testing.T) { + // Generate forked chain. The returned BlockChain object is used to process the side chain blocks. + tmpChain, sideblocks, canonblocks, err := getLongAndShortChains() + if err != nil { + t.Fatal(err) + } + defer tmpChain.Stop() + // Get the side chain receipts. + if _, err := tmpChain.InsertChain(sideblocks); err != nil { + t.Fatal("processing side chain failed:", err) + } + t.Log("sidechain head:", tmpChain.CurrentBlock().Number(), tmpChain.CurrentBlock().Hash()) + sidechainReceipts := make([]types.Receipts, len(sideblocks)) + for i, block := range sideblocks { + sidechainReceipts[i] = tmpChain.GetReceiptsByHash(block.Hash()) + } + // Get the canon chain receipts. + if _, err := tmpChain.InsertChain(canonblocks); err != nil { + t.Fatal("processing canon chain failed:", err) + } + t.Log("canon head:", tmpChain.CurrentBlock().Number(), tmpChain.CurrentBlock().Hash()) + canonReceipts := make([]types.Receipts, len(canonblocks)) + for i, block := range canonblocks { + canonReceipts[i] = tmpChain.GetReceiptsByHash(block.Hash()) + } - // Import the chain as a ancient-first node and ensure all pointers are updated + // Set up a BlockChain that uses the ancient store. frdir, err := ioutil.TempDir("", "") if err != nil { t.Fatalf("failed to create temp freezer dir: %v", err) @@ -1662,38 +1694,43 @@ func TestIncompleteAncientReceiptChainInsertion(t *testing.T) { if err != nil { t.Fatalf("failed to create temp freezer db: %v", err) } + gspec := Genesis{Config: params.AllEthashProtocolChanges} gspec.MustCommit(ancientDb) - ancient, _ := NewBlockChain(ancientDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) - defer ancient.Stop() + ancientChain, _ := NewBlockChain(ancientDb, nil, gspec.Config, ethash.NewFaker(), vm.Config{}, nil, nil) + defer ancientChain.Stop() - headers := make([]*types.Header, len(blocks)) - for i, block := range blocks { - headers[i] = block.Header() + // Import the canonical header chain. + canonHeaders := make([]*types.Header, len(canonblocks)) + for i, block := range canonblocks { + canonHeaders[i] = block.Header() } - if n, err := ancient.InsertHeaderChain(headers, 1); err != nil { - t.Fatalf("failed to insert header %d: %v", n, err) + if _, err = ancientChain.InsertHeaderChain(canonHeaders, 1); err != nil { + t.Fatal("can't import canon headers:", err) } - // Abort ancient receipt chain insertion deliberately - ancient.terminateInsert = func(hash common.Hash, number uint64) bool { - return number == blocks[len(blocks)/2].NumberU64() + + // Try to insert blocks/receipts of the side chain. + _, err = ancientChain.InsertReceiptChain(sideblocks, sidechainReceipts, uint64(len(sideblocks))) + if err == nil { + t.Fatal("expected error from InsertReceiptChain.") } - previousFastBlock := ancient.CurrentFastBlock() - if n, err := ancient.InsertReceiptChain(blocks, receipts, uint64(3*len(blocks)/4)); err == nil { - t.Fatalf("failed to insert receipt %d: %v", n, err) + if ancientChain.CurrentFastBlock().NumberU64() != 0 { + t.Fatalf("failed to rollback ancient data, want %d, have %d", 0, ancientChain.CurrentFastBlock().NumberU64()) } - if ancient.CurrentFastBlock().NumberU64() != previousFastBlock.NumberU64() { - t.Fatalf("failed to rollback ancient data, want %d, have %d", previousFastBlock.NumberU64(), ancient.CurrentFastBlock().NumberU64()) + if frozen, err := ancientChain.db.Ancients(); err != nil || frozen != 1 { + t.Fatalf("failed to truncate ancient data, frozen index is %d", frozen) } - if frozen, err := ancient.db.Ancients(); err != nil || frozen != 1 { - t.Fatalf("failed to truncate ancient data") + + // Insert blocks/receipts of the canonical chain. + _, err = ancientChain.InsertReceiptChain(canonblocks, canonReceipts, uint64(len(canonblocks))) + if err != nil { + t.Fatalf("can't import canon chain receipts: %v", err) } - ancient.terminateInsert = nil - if n, err := ancient.InsertReceiptChain(blocks, receipts, uint64(3*len(blocks)/4)); err != nil { - t.Fatalf("failed to insert receipt %d: %v", n, err) - } - if ancient.CurrentFastBlock().NumberU64() != blocks[len(blocks)-1].NumberU64() { + if ancientChain.CurrentFastBlock().NumberU64() != canonblocks[len(canonblocks)-1].NumberU64() { t.Fatalf("failed to insert ancient recept chain after rollback") } + if frozen, _ := ancientChain.db.Ancients(); frozen != uint64(len(canonblocks))+1 { + t.Fatalf("wrong ancients count %d", frozen) + } } // Tests that importing a very large side fork, which is larger than the canon chain, @@ -1958,9 +1995,8 @@ func testInsertKnownChainData(t *testing.T, typ string) { asserter(t, blocks2[len(blocks2)-1]) } -// getLongAndShortChains returns two chains, -// A is longer, B is heavier -func getLongAndShortChains() (*BlockChain, []*types.Block, []*types.Block, error) { +// getLongAndShortChains returns two chains: A is longer, B is heavier. +func getLongAndShortChains() (bc *BlockChain, longChain []*types.Block, heavyChain []*types.Block, err error) { // Generate a canonical chain to act as the main dataset engine := ethash.NewFaker() db := rawdb.NewMemoryDatabase() @@ -1968,7 +2004,7 @@ func getLongAndShortChains() (*BlockChain, []*types.Block, []*types.Block, error // Generate and import the canonical chain, // Offset the time, to keep the difficulty low - longChain, _ := GenerateChain(params.TestChainConfig, genesis, engine, db, 80, func(i int, b *BlockGen) { + longChain, _ = GenerateChain(params.TestChainConfig, genesis, engine, db, 80, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{1}) }) diskdb := rawdb.NewMemoryDatabase() @@ -1982,10 +2018,13 @@ func getLongAndShortChains() (*BlockChain, []*types.Block, []*types.Block, error // Generate fork chain, make it shorter than canon, with common ancestor pretty early parentIndex := 3 parent := longChain[parentIndex] - heavyChain, _ := GenerateChain(params.TestChainConfig, parent, engine, db, 75, func(i int, b *BlockGen) { + heavyChainExt, _ := GenerateChain(params.TestChainConfig, parent, engine, db, 75, func(i int, b *BlockGen) { b.SetCoinbase(common.Address{2}) b.OffsetTime(-9) }) + heavyChain = append(heavyChain, longChain[:parentIndex+1]...) + heavyChain = append(heavyChain, heavyChainExt...) + // Verify that the test is sane var ( longerTd = new(big.Int) diff --git a/core/error.go b/core/error.go index 33cf874d1..594f3a26e 100644 --- a/core/error.go +++ b/core/error.go @@ -31,6 +31,8 @@ var ( // ErrNoGenesis is returned when there is no Genesis Block. ErrNoGenesis = errors.New("genesis not found in chain") + + errSideChainReceipts = errors.New("side blocks can't be accepted as ancient chain data") ) // List of evm-call-message pre-checking errors. All state transition messages will diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index 76132bf37..58226fb04 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -19,6 +19,7 @@ package rawdb import ( "bytes" "encoding/binary" + "fmt" "math/big" "sort" @@ -81,6 +82,37 @@ func ReadAllHashes(db ethdb.Iteratee, number uint64) []common.Hash { return hashes } +type NumberHash struct { + Number uint64 + Hash common.Hash +} + +// ReadAllHashes retrieves all the hashes assigned to blocks at a certain heights, +// both canonical and reorged forks included. +// This method considers both limits to be _inclusive_. +func ReadAllHashesInRange(db ethdb.Iteratee, first, last uint64) []*NumberHash { + var ( + start = encodeBlockNumber(first) + keyLength = len(headerPrefix) + 8 + 32 + hashes = make([]*NumberHash, 0, 1+last-first) + it = db.NewIterator(headerPrefix, start) + ) + defer it.Release() + for it.Next() { + key := it.Key() + if len(key) != keyLength { + continue + } + num := binary.BigEndian.Uint64(key[len(headerPrefix) : len(headerPrefix)+8]) + if num > last { + break + } + hash := common.BytesToHash(key[len(key)-32:]) + hashes = append(hashes, &NumberHash{num, hash}) + } + return hashes +} + // ReadAllCanonicalHashes retrieves all canonical number and hash mappings at the // certain chain range. If the accumulated entries reaches the given threshold, // abort the iteration and return the semi-finish result. @@ -656,34 +688,48 @@ func WriteBlock(db ethdb.KeyValueWriter, block *types.Block) { } // WriteAncientBlock writes entire block data into ancient store and returns the total written size. -func WriteAncientBlock(db ethdb.AncientWriter, block *types.Block, receipts types.Receipts, td *big.Int) int { - // Encode all block components to RLP format. - headerBlob, err := rlp.EncodeToBytes(block.Header()) - if err != nil { - log.Crit("Failed to RLP encode block header", "err", err) +func WriteAncientBlocks(db ethdb.AncientWriter, blocks []*types.Block, receipts []types.Receipts, td *big.Int) (int64, error) { + var ( + tdSum = new(big.Int).Set(td) + stReceipts []*types.ReceiptForStorage + ) + return db.ModifyAncients(func(op ethdb.AncientWriteOp) error { + for i, block := range blocks { + // Convert receipts to storage format and sum up total difficulty. + stReceipts = stReceipts[:0] + for _, receipt := range receipts[i] { + stReceipts = append(stReceipts, (*types.ReceiptForStorage)(receipt)) + } + header := block.Header() + if i > 0 { + tdSum.Add(tdSum, header.Difficulty) + } + if err := writeAncientBlock(op, block, header, stReceipts, tdSum); err != nil { + return err + } + } + return nil + }) +} + +func writeAncientBlock(op ethdb.AncientWriteOp, block *types.Block, header *types.Header, receipts []*types.ReceiptForStorage, td *big.Int) error { + num := block.NumberU64() + if err := op.AppendRaw(freezerHashTable, num, block.Hash().Bytes()); err != nil { + return fmt.Errorf("can't add block %d hash: %v", num, err) } - bodyBlob, err := rlp.EncodeToBytes(block.Body()) - if err != nil { - log.Crit("Failed to RLP encode body", "err", err) + if err := op.Append(freezerHeaderTable, num, header); err != nil { + return fmt.Errorf("can't append block header %d: %v", num, err) } - storageReceipts := make([]*types.ReceiptForStorage, len(receipts)) - for i, receipt := range receipts { - storageReceipts[i] = (*types.ReceiptForStorage)(receipt) + if err := op.Append(freezerBodiesTable, num, block.Body()); err != nil { + return fmt.Errorf("can't append block body %d: %v", num, err) } - receiptBlob, err := rlp.EncodeToBytes(storageReceipts) - if err != nil { - log.Crit("Failed to RLP encode block receipts", "err", err) + if err := op.Append(freezerReceiptTable, num, receipts); err != nil { + return fmt.Errorf("can't append block %d receipts: %v", num, err) } - tdBlob, err := rlp.EncodeToBytes(td) - if err != nil { - log.Crit("Failed to RLP encode block total difficulty", "err", err) + if err := op.Append(freezerDifficultyTable, num, td); err != nil { + return fmt.Errorf("can't append block %d total difficulty: %v", num, err) } - // Write all blob to flatten files. - err = db.AppendAncient(block.NumberU64(), block.Hash().Bytes(), headerBlob, bodyBlob, receiptBlob, tdBlob) - if err != nil { - log.Crit("Failed to write block data to ancient store", "err", err) - } - return len(headerBlob) + len(bodyBlob) + len(receiptBlob) + len(tdBlob) + common.HashLength + return nil } // DeleteBlock removes all block data associated with a hash. diff --git a/core/rawdb/accessors_chain_test.go b/core/rawdb/accessors_chain_test.go index f20e8b1ff..58d7645e5 100644 --- a/core/rawdb/accessors_chain_test.go +++ b/core/rawdb/accessors_chain_test.go @@ -29,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" "golang.org/x/crypto/sha3" @@ -438,7 +439,7 @@ func TestAncientStorage(t *testing.T) { if err != nil { t.Fatalf("failed to create temp freezer dir: %v", err) } - defer os.Remove(frdir) + defer os.RemoveAll(frdir) db, err := NewDatabaseWithFreezer(NewMemoryDatabase(), frdir, "", false) if err != nil { @@ -467,8 +468,10 @@ func TestAncientStorage(t *testing.T) { if blob := ReadTdRLP(db, hash, number); len(blob) > 0 { t.Fatalf("non existent td returned") } + // Write and verify the header in the database - WriteAncientBlock(db, block, nil, big.NewInt(100)) + WriteAncientBlocks(db, []*types.Block{block}, []types.Receipts{nil}, big.NewInt(100)) + if blob := ReadHeaderRLP(db, hash, number); len(blob) == 0 { t.Fatalf("no header returned") } @@ -481,6 +484,7 @@ func TestAncientStorage(t *testing.T) { if blob := ReadTdRLP(db, hash, number); len(blob) == 0 { t.Fatalf("no td returned") } + // Use a fake hash for data retrieval, nothing should be returned. fakeHash := common.BytesToHash([]byte{0x01, 0x02, 0x03}) if blob := ReadHeaderRLP(db, fakeHash, number); len(blob) != 0 { @@ -528,3 +532,141 @@ func TestCanonicalHashIteration(t *testing.T) { } } } + +func TestHashesInRange(t *testing.T) { + mkHeader := func(number, seq int) *types.Header { + h := types.Header{ + Difficulty: new(big.Int), + Number: big.NewInt(int64(number)), + GasLimit: uint64(seq), + } + return &h + } + db := NewMemoryDatabase() + // For each number, write N versions of that particular number + total := 0 + for i := 0; i < 15; i++ { + for ii := 0; ii < i; ii++ { + WriteHeader(db, mkHeader(i, ii)) + total++ + } + } + if have, want := len(ReadAllHashesInRange(db, 10, 10)), 10; have != want { + t.Fatalf("Wrong number of hashes read, want %d, got %d", want, have) + } + if have, want := len(ReadAllHashesInRange(db, 10, 9)), 0; have != want { + t.Fatalf("Wrong number of hashes read, want %d, got %d", want, have) + } + if have, want := len(ReadAllHashesInRange(db, 0, 100)), total; have != want { + t.Fatalf("Wrong number of hashes read, want %d, got %d", want, have) + } + if have, want := len(ReadAllHashesInRange(db, 9, 10)), 9+10; have != want { + t.Fatalf("Wrong number of hashes read, want %d, got %d", want, have) + } + if have, want := len(ReadAllHashes(db, 10)), 10; have != want { + t.Fatalf("Wrong number of hashes read, want %d, got %d", want, have) + } + if have, want := len(ReadAllHashes(db, 16)), 0; have != want { + t.Fatalf("Wrong number of hashes read, want %d, got %d", want, have) + } + if have, want := len(ReadAllHashes(db, 1)), 1; have != want { + t.Fatalf("Wrong number of hashes read, want %d, got %d", want, have) + } +} + +// This measures the write speed of the WriteAncientBlocks operation. +func BenchmarkWriteAncientBlocks(b *testing.B) { + // Open freezer database. + frdir, err := ioutil.TempDir("", "") + if err != nil { + b.Fatalf("failed to create temp freezer dir: %v", err) + } + defer os.RemoveAll(frdir) + db, err := NewDatabaseWithFreezer(NewMemoryDatabase(), frdir, "", false) + if err != nil { + b.Fatalf("failed to create database with ancient backend") + } + + // Create the data to insert. The blocks must have consecutive numbers, so we create + // all of them ahead of time. However, there is no need to create receipts + // individually for each block, just make one batch here and reuse it for all writes. + const batchSize = 128 + const blockTxs = 20 + allBlocks := makeTestBlocks(b.N, blockTxs) + batchReceipts := makeTestReceipts(batchSize, blockTxs) + b.ResetTimer() + + // The benchmark loop writes batches of blocks, but note that the total block count is + // b.N. This means the resulting ns/op measurement is the time it takes to write a + // single block and its associated data. + var td = big.NewInt(55) + var totalSize int64 + for i := 0; i < b.N; i += batchSize { + length := batchSize + if i+batchSize > b.N { + length = b.N - i + } + + blocks := allBlocks[i : i+length] + receipts := batchReceipts[:length] + writeSize, err := WriteAncientBlocks(db, blocks, receipts, td) + if err != nil { + b.Fatal(err) + } + totalSize += writeSize + } + + // Enable MB/s reporting. + b.SetBytes(totalSize / int64(b.N)) +} + +// makeTestBlocks creates fake blocks for the ancient write benchmark. +func makeTestBlocks(nblock int, txsPerBlock int) []*types.Block { + key, _ := crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + signer := types.LatestSignerForChainID(big.NewInt(8)) + + // Create transactions. + txs := make([]*types.Transaction, txsPerBlock) + for i := 0; i < len(txs); i++ { + var err error + to := common.Address{1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1} + txs[i], err = types.SignNewTx(key, signer, &types.LegacyTx{ + Nonce: 2, + GasPrice: big.NewInt(30000), + Gas: 0x45454545, + To: &to, + }) + if err != nil { + panic(err) + } + } + + // Create the blocks. + blocks := make([]*types.Block, nblock) + for i := 0; i < nblock; i++ { + header := &types.Header{ + Number: big.NewInt(int64(i)), + Extra: []byte("test block"), + } + blocks[i] = types.NewBlockWithHeader(header).WithBody(txs, nil) + blocks[i].Hash() // pre-cache the block hash + } + return blocks +} + +// makeTestReceipts creates fake receipts for the ancient write benchmark. +func makeTestReceipts(n int, nPerBlock int) []types.Receipts { + receipts := make([]*types.Receipt, nPerBlock) + for i := 0; i < len(receipts); i++ { + receipts[i] = &types.Receipt{ + Status: types.ReceiptStatusSuccessful, + CumulativeGasUsed: 0x888888888, + Logs: make([]*types.Log, 5), + } + } + allReceipts := make([]types.Receipts, n) + for i := 0; i < n; i++ { + allReceipts[i] = receipts + } + return allReceipts +} diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 90619169a..0e116ef99 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -104,9 +104,9 @@ func (db *nofreezedb) AncientSize(kind string) (uint64, error) { return 0, errNotSupported } -// AppendAncient returns an error as we don't have a backing chain freezer. -func (db *nofreezedb) AppendAncient(number uint64, hash, header, body, receipts, td []byte) error { - return errNotSupported +// ModifyAncients is not supported. +func (db *nofreezedb) ModifyAncients(func(ethdb.AncientWriteOp) error) (int64, error) { + return 0, errNotSupported } // TruncateAncients returns an error as we don't have a backing chain freezer. @@ -122,9 +122,7 @@ func (db *nofreezedb) Sync() error { // NewDatabase creates a high level database on top of a given key-value data // store without a freezer moving immutable chain segments into cold storage. func NewDatabase(db ethdb.KeyValueStore) ethdb.Database { - return &nofreezedb{ - KeyValueStore: db, - } + return &nofreezedb{KeyValueStore: db} } // NewDatabaseWithFreezer creates a high level database on top of a given key- @@ -132,7 +130,7 @@ func NewDatabase(db ethdb.KeyValueStore) ethdb.Database { // storage. func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace string, readonly bool) (ethdb.Database, error) { // Create the idle freezer instance - frdb, err := newFreezer(freezer, namespace, readonly) + frdb, err := newFreezer(freezer, namespace, readonly, freezerTableSize, FreezerNoSnappy) if err != nil { return nil, err } diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index 4262a615a..4cecbc50f 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -61,6 +61,9 @@ const ( // freezerBatchLimit is the maximum number of blocks to freeze in one batch // before doing an fsync and deleting it from the key-value store. freezerBatchLimit = 30000 + + // freezerTableSize defines the maximum size of freezer data files. + freezerTableSize = 2 * 1000 * 1000 * 1000 ) // freezer is an memory mapped append-only database to store immutable chain data @@ -77,6 +80,10 @@ type freezer struct { frozen uint64 // Number of blocks already frozen threshold uint64 // Number of recent blocks not to freeze (params.FullImmutabilityThreshold apart from tests) + // This lock synchronizes writers and the truncate operation. + writeLock sync.Mutex + writeBatch *freezerBatch + readonly bool tables map[string]*freezerTable // Data tables for storing everything instanceLock fileutil.Releaser // File-system lock to prevent double opens @@ -90,7 +97,10 @@ type freezer struct { // newFreezer creates a chain freezer that moves ancient chain data into // append-only flat file containers. -func newFreezer(datadir string, namespace string, readonly bool) (*freezer, error) { +// +// The 'tables' argument defines the data tables. If the value of a map +// entry is true, snappy compression is disabled for the table. +func newFreezer(datadir string, namespace string, readonly bool, maxTableSize uint32, tables map[string]bool) (*freezer, error) { // Create the initial freezer object var ( readMeter = metrics.NewRegisteredMeter(namespace+"ancient/read", nil) @@ -119,8 +129,10 @@ func newFreezer(datadir string, namespace string, readonly bool) (*freezer, erro trigger: make(chan chan struct{}), quit: make(chan struct{}), } - for name, disableSnappy := range FreezerNoSnappy { - table, err := newTable(datadir, name, readMeter, writeMeter, sizeGauge, disableSnappy) + + // Create the tables. + for name, disableSnappy := range tables { + table, err := newTable(datadir, name, readMeter, writeMeter, sizeGauge, maxTableSize, disableSnappy) if err != nil { for _, table := range freezer.tables { table.Close() @@ -130,6 +142,8 @@ func newFreezer(datadir string, namespace string, readonly bool) (*freezer, erro } freezer.tables[name] = table } + + // Truncate all tables to common length. if err := freezer.repair(); err != nil { for _, table := range freezer.tables { table.Close() @@ -137,12 +151,19 @@ func newFreezer(datadir string, namespace string, readonly bool) (*freezer, erro lock.Release() return nil, err } + + // Create the write batch. + freezer.writeBatch = newFreezerBatch(freezer) + log.Info("Opened ancient database", "database", datadir, "readonly", readonly) return freezer, nil } // Close terminates the chain freezer, unmapping all the data files. func (f *freezer) Close() error { + f.writeLock.Lock() + defer f.writeLock.Unlock() + var errs []error f.closeOnce.Do(func() { close(f.quit) @@ -199,60 +220,49 @@ func (f *freezer) Ancients() (uint64, error) { // AncientSize returns the ancient size of the specified category. func (f *freezer) AncientSize(kind string) (uint64, error) { + // This needs the write lock to avoid data races on table fields. + // Speed doesn't matter here, AncientSize is for debugging. + f.writeLock.Lock() + defer f.writeLock.Unlock() + if table := f.tables[kind]; table != nil { return table.size() } return 0, errUnknownTable } -// AppendAncient injects all binary blobs belong to block at the end of the -// append-only immutable table files. -// -// Notably, this function is lock free but kind of thread-safe. All out-of-order -// injection will be rejected. But if two injections with same number happen at -// the same time, we can get into the trouble. -func (f *freezer) AppendAncient(number uint64, hash, header, body, receipts, td []byte) (err error) { +// ModifyAncients runs the given write operation. +func (f *freezer) ModifyAncients(fn func(ethdb.AncientWriteOp) error) (writeSize int64, err error) { if f.readonly { - return errReadOnly + return 0, errReadOnly } - // Ensure the binary blobs we are appending is continuous with freezer. - if atomic.LoadUint64(&f.frozen) != number { - return errOutOrderInsertion - } - // Rollback all inserted data if any insertion below failed to ensure - // the tables won't out of sync. + f.writeLock.Lock() + defer f.writeLock.Unlock() + + // Roll back all tables to the starting position in case of error. + prevItem := f.frozen defer func() { if err != nil { - rerr := f.repair() - if rerr != nil { - log.Crit("Failed to repair freezer", "err", rerr) + // The write operation has failed. Go back to the previous item position. + for name, table := range f.tables { + err := table.truncate(prevItem) + if err != nil { + log.Error("Freezer table roll-back failed", "table", name, "index", prevItem, "err", err) + } } - log.Info("Append ancient failed", "number", number, "err", err) } }() - // Inject all the components into the relevant data tables - if err := f.tables[freezerHashTable].Append(f.frozen, hash[:]); err != nil { - log.Error("Failed to append ancient hash", "number", f.frozen, "hash", hash, "err", err) - return err + + f.writeBatch.reset() + if err := fn(f.writeBatch); err != nil { + return 0, err } - if err := f.tables[freezerHeaderTable].Append(f.frozen, header); err != nil { - log.Error("Failed to append ancient header", "number", f.frozen, "hash", hash, "err", err) - return err + item, writeSize, err := f.writeBatch.commit() + if err != nil { + return 0, err } - if err := f.tables[freezerBodiesTable].Append(f.frozen, body); err != nil { - log.Error("Failed to append ancient body", "number", f.frozen, "hash", hash, "err", err) - return err - } - if err := f.tables[freezerReceiptTable].Append(f.frozen, receipts); err != nil { - log.Error("Failed to append ancient receipts", "number", f.frozen, "hash", hash, "err", err) - return err - } - if err := f.tables[freezerDifficultyTable].Append(f.frozen, td); err != nil { - log.Error("Failed to append ancient difficulty", "number", f.frozen, "hash", hash, "err", err) - return err - } - atomic.AddUint64(&f.frozen, 1) // Only modify atomically - return nil + atomic.StoreUint64(&f.frozen, item) + return writeSize, nil } // TruncateAncients discards any recent data above the provided threshold number. @@ -260,6 +270,9 @@ func (f *freezer) TruncateAncients(items uint64) error { if f.readonly { return errReadOnly } + f.writeLock.Lock() + defer f.writeLock.Unlock() + if atomic.LoadUint64(&f.frozen) <= items { return nil } @@ -286,6 +299,24 @@ func (f *freezer) Sync() error { return nil } +// repair truncates all data tables to the same length. +func (f *freezer) repair() error { + min := uint64(math.MaxUint64) + for _, table := range f.tables { + items := atomic.LoadUint64(&table.items) + if min > items { + min = items + } + } + for _, table := range f.tables { + if err := table.truncate(min); err != nil { + return err + } + } + atomic.StoreUint64(&f.frozen, min) + return nil +} + // freeze is a background thread that periodically checks the blockchain for any // import progress and moves ancient data from the fast database into the freezer. // @@ -352,54 +383,28 @@ func (f *freezer) freeze(db ethdb.KeyValueStore) { backoff = true continue } + // Seems we have data ready to be frozen, process in usable batches - limit := *number - threshold - if limit-f.frozen > freezerBatchLimit { - limit = f.frozen + freezerBatchLimit - } var ( start = time.Now() - first = f.frozen - ancients = make([]common.Hash, 0, limit-f.frozen) + first, _ = f.Ancients() + limit = *number - threshold ) - for f.frozen <= limit { - // Retrieves all the components of the canonical block - hash := ReadCanonicalHash(nfdb, f.frozen) - if hash == (common.Hash{}) { - log.Error("Canonical hash missing, can't freeze", "number", f.frozen) - break - } - header := ReadHeaderRLP(nfdb, hash, f.frozen) - if len(header) == 0 { - log.Error("Block header missing, can't freeze", "number", f.frozen, "hash", hash) - break - } - body := ReadBodyRLP(nfdb, hash, f.frozen) - if len(body) == 0 { - log.Error("Block body missing, can't freeze", "number", f.frozen, "hash", hash) - break - } - receipts := ReadReceiptsRLP(nfdb, hash, f.frozen) - if len(receipts) == 0 { - log.Error("Block receipts missing, can't freeze", "number", f.frozen, "hash", hash) - break - } - td := ReadTdRLP(nfdb, hash, f.frozen) - if len(td) == 0 { - log.Error("Total difficulty missing, can't freeze", "number", f.frozen, "hash", hash) - break - } - log.Trace("Deep froze ancient block", "number", f.frozen, "hash", hash) - // Inject all the components into the relevant data tables - if err := f.AppendAncient(f.frozen, hash[:], header, body, receipts, td); err != nil { - break - } - ancients = append(ancients, hash) + if limit-first > freezerBatchLimit { + limit = first + freezerBatchLimit } + ancients, err := f.freezeRange(nfdb, first, limit) + if err != nil { + log.Error("Error in block freeze operation", "err", err) + backoff = true + continue + } + // Batch of blocks have been frozen, flush them before wiping from leveldb if err := f.Sync(); err != nil { log.Crit("Failed to flush frozen tables", "err", err) } + // Wipe out all data from the active database batch := db.NewBatch() for i := 0; i < len(ancients); i++ { @@ -464,6 +469,7 @@ func (f *freezer) freeze(db ethdb.KeyValueStore) { log.Crit("Failed to delete dangling side blocks", "err", err) } } + // Log something friendly for the user context := []interface{}{ "blocks", f.frozen - first, "elapsed", common.PrettyDuration(time.Since(start)), "number", f.frozen - 1, @@ -480,20 +486,54 @@ func (f *freezer) freeze(db ethdb.KeyValueStore) { } } -// repair truncates all data tables to the same length. -func (f *freezer) repair() error { - min := uint64(math.MaxUint64) - for _, table := range f.tables { - items := atomic.LoadUint64(&table.items) - if min > items { - min = items +func (f *freezer) freezeRange(nfdb *nofreezedb, number, limit uint64) (hashes []common.Hash, err error) { + hashes = make([]common.Hash, 0, limit-number) + + _, err = f.ModifyAncients(func(op ethdb.AncientWriteOp) error { + for ; number <= limit; number++ { + // Retrieve all the components of the canonical block. + hash := ReadCanonicalHash(nfdb, number) + if hash == (common.Hash{}) { + return fmt.Errorf("canonical hash missing, can't freeze block %d", number) + } + header := ReadHeaderRLP(nfdb, hash, number) + if len(header) == 0 { + return fmt.Errorf("block header missing, can't freeze block %d", number) + } + body := ReadBodyRLP(nfdb, hash, number) + if len(body) == 0 { + return fmt.Errorf("block body missing, can't freeze block %d", number) + } + receipts := ReadReceiptsRLP(nfdb, hash, number) + if len(receipts) == 0 { + return fmt.Errorf("block receipts missing, can't freeze block %d", number) + } + td := ReadTdRLP(nfdb, hash, number) + if len(td) == 0 { + return fmt.Errorf("total difficulty missing, can't freeze block %d", number) + } + + // Write to the batch. + if err := op.AppendRaw(freezerHashTable, number, hash[:]); err != nil { + return fmt.Errorf("can't write hash to freezer: %v", err) + } + if err := op.AppendRaw(freezerHeaderTable, number, header); err != nil { + return fmt.Errorf("can't write header to freezer: %v", err) + } + if err := op.AppendRaw(freezerBodiesTable, number, body); err != nil { + return fmt.Errorf("can't write body to freezer: %v", err) + } + if err := op.AppendRaw(freezerReceiptTable, number, receipts); err != nil { + return fmt.Errorf("can't write receipts to freezer: %v", err) + } + if err := op.AppendRaw(freezerDifficultyTable, number, td); err != nil { + return fmt.Errorf("can't write td to freezer: %v", err) + } + + hashes = append(hashes, hash) } - } - for _, table := range f.tables { - if err := table.truncate(min); err != nil { - return err - } - } - atomic.StoreUint64(&f.frozen, min) - return nil + return nil + }) + + return hashes, err } diff --git a/core/rawdb/freezer_batch.go b/core/rawdb/freezer_batch.go new file mode 100644 index 000000000..8297c0ac1 --- /dev/null +++ b/core/rawdb/freezer_batch.go @@ -0,0 +1,248 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rawdb + +import ( + "fmt" + "sync/atomic" + + "github.com/ethereum/go-ethereum/common/math" + "github.com/ethereum/go-ethereum/rlp" + "github.com/golang/snappy" +) + +// This is the maximum amount of data that will be buffered in memory +// for a single freezer table batch. +const freezerBatchBufferLimit = 2 * 1024 * 1024 + +// freezerBatch is a write operation of multiple items on a freezer. +type freezerBatch struct { + tables map[string]*freezerTableBatch +} + +func newFreezerBatch(f *freezer) *freezerBatch { + batch := &freezerBatch{tables: make(map[string]*freezerTableBatch, len(f.tables))} + for kind, table := range f.tables { + batch.tables[kind] = table.newBatch() + } + return batch +} + +// Append adds an RLP-encoded item of the given kind. +func (batch *freezerBatch) Append(kind string, num uint64, item interface{}) error { + return batch.tables[kind].Append(num, item) +} + +// AppendRaw adds an item of the given kind. +func (batch *freezerBatch) AppendRaw(kind string, num uint64, item []byte) error { + return batch.tables[kind].AppendRaw(num, item) +} + +// reset initializes the batch. +func (batch *freezerBatch) reset() { + for _, tb := range batch.tables { + tb.reset() + } +} + +// commit is called at the end of a write operation and +// writes all remaining data to tables. +func (batch *freezerBatch) commit() (item uint64, writeSize int64, err error) { + // Check that count agrees on all batches. + item = uint64(math.MaxUint64) + for name, tb := range batch.tables { + if item < math.MaxUint64 && tb.curItem != item { + return 0, 0, fmt.Errorf("table %s is at item %d, want %d", name, tb.curItem, item) + } + item = tb.curItem + } + + // Commit all table batches. + for _, tb := range batch.tables { + if err := tb.commit(); err != nil { + return 0, 0, err + } + writeSize += tb.totalBytes + } + return item, writeSize, nil +} + +// freezerTableBatch is a batch for a freezer table. +type freezerTableBatch struct { + t *freezerTable + + sb *snappyBuffer + encBuffer writeBuffer + dataBuffer []byte + indexBuffer []byte + curItem uint64 // expected index of next append + totalBytes int64 // counts written bytes since reset +} + +// newBatch creates a new batch for the freezer table. +func (t *freezerTable) newBatch() *freezerTableBatch { + batch := &freezerTableBatch{t: t} + if !t.noCompression { + batch.sb = new(snappyBuffer) + } + batch.reset() + return batch +} + +// reset clears the batch for reuse. +func (batch *freezerTableBatch) reset() { + batch.dataBuffer = batch.dataBuffer[:0] + batch.indexBuffer = batch.indexBuffer[:0] + batch.curItem = atomic.LoadUint64(&batch.t.items) + batch.totalBytes = 0 +} + +// Append rlp-encodes and adds data at the end of the freezer table. The item number is a +// precautionary parameter to ensure data correctness, but the table will reject already +// existing data. +func (batch *freezerTableBatch) Append(item uint64, data interface{}) error { + if item != batch.curItem { + return errOutOrderInsertion + } + + // Encode the item. + batch.encBuffer.Reset() + if err := rlp.Encode(&batch.encBuffer, data); err != nil { + return err + } + encItem := batch.encBuffer.data + if batch.sb != nil { + encItem = batch.sb.compress(encItem) + } + return batch.appendItem(encItem) +} + +// AppendRaw injects a binary blob at the end of the freezer table. The item number is a +// precautionary parameter to ensure data correctness, but the table will reject already +// existing data. +func (batch *freezerTableBatch) AppendRaw(item uint64, blob []byte) error { + if item != batch.curItem { + return errOutOrderInsertion + } + + encItem := blob + if batch.sb != nil { + encItem = batch.sb.compress(blob) + } + return batch.appendItem(encItem) +} + +func (batch *freezerTableBatch) appendItem(data []byte) error { + // Check if item fits into current data file. + itemSize := int64(len(data)) + itemOffset := batch.t.headBytes + int64(len(batch.dataBuffer)) + if itemOffset+itemSize > int64(batch.t.maxFileSize) { + // It doesn't fit, go to next file first. + if err := batch.commit(); err != nil { + return err + } + if err := batch.t.advanceHead(); err != nil { + return err + } + itemOffset = 0 + } + + // Put data to buffer. + batch.dataBuffer = append(batch.dataBuffer, data...) + batch.totalBytes += itemSize + + // Put index entry to buffer. + entry := indexEntry{filenum: batch.t.headId, offset: uint32(itemOffset + itemSize)} + batch.indexBuffer = entry.append(batch.indexBuffer) + batch.curItem++ + + return batch.maybeCommit() +} + +// maybeCommit writes the buffered data if the buffer is full enough. +func (batch *freezerTableBatch) maybeCommit() error { + if len(batch.dataBuffer) > freezerBatchBufferLimit { + return batch.commit() + } + return nil +} + +// commit writes the batched items to the backing freezerTable. +func (batch *freezerTableBatch) commit() error { + // Write data. + _, err := batch.t.head.Write(batch.dataBuffer) + if err != nil { + return err + } + dataSize := int64(len(batch.dataBuffer)) + batch.dataBuffer = batch.dataBuffer[:0] + + // Write index. + _, err = batch.t.index.Write(batch.indexBuffer) + if err != nil { + return err + } + indexSize := int64(len(batch.indexBuffer)) + batch.indexBuffer = batch.indexBuffer[:0] + + // Update headBytes of table. + batch.t.headBytes += dataSize + atomic.StoreUint64(&batch.t.items, batch.curItem) + + // Update metrics. + batch.t.sizeGauge.Inc(dataSize + indexSize) + batch.t.writeMeter.Mark(dataSize + indexSize) + return nil +} + +// snappyBuffer writes snappy in block format, and can be reused. It is +// reset when WriteTo is called. +type snappyBuffer struct { + dst []byte +} + +// compress snappy-compresses the data. +func (s *snappyBuffer) compress(data []byte) []byte { + // The snappy library does not care what the capacity of the buffer is, + // but only checks the length. If the length is too small, it will + // allocate a brand new buffer. + // To avoid that, we check the required size here, and grow the size of the + // buffer to utilize the full capacity. + if n := snappy.MaxEncodedLen(len(data)); len(s.dst) < n { + if cap(s.dst) < n { + s.dst = make([]byte, n) + } + s.dst = s.dst[:n] + } + + s.dst = snappy.Encode(s.dst, data) + return s.dst +} + +// writeBuffer implements io.Writer for a byte slice. +type writeBuffer struct { + data []byte +} + +func (wb *writeBuffer) Write(data []byte) (int, error) { + wb.data = append(wb.data, data...) + return len(data), nil +} + +func (wb *writeBuffer) Reset() { + wb.data = wb.data[:0] +} diff --git a/core/rawdb/freezer_table.go b/core/rawdb/freezer_table.go index 9d052f7cd..22405cf9b 100644 --- a/core/rawdb/freezer_table.go +++ b/core/rawdb/freezer_table.go @@ -17,6 +17,7 @@ package rawdb import ( + "bytes" "encoding/binary" "errors" "fmt" @@ -55,19 +56,20 @@ type indexEntry struct { const indexEntrySize = 6 -// unmarshallBinary deserializes binary b into the rawIndex entry. +// unmarshalBinary deserializes binary b into the rawIndex entry. func (i *indexEntry) unmarshalBinary(b []byte) error { i.filenum = uint32(binary.BigEndian.Uint16(b[:2])) i.offset = binary.BigEndian.Uint32(b[2:6]) return nil } -// marshallBinary serializes the rawIndex entry into binary. -func (i *indexEntry) marshallBinary() []byte { - b := make([]byte, indexEntrySize) - binary.BigEndian.PutUint16(b[:2], uint16(i.filenum)) - binary.BigEndian.PutUint32(b[2:6], i.offset) - return b +// append adds the encoded entry to the end of b. +func (i *indexEntry) append(b []byte) []byte { + offset := len(b) + out := append(b, make([]byte, indexEntrySize)...) + binary.BigEndian.PutUint16(out[offset:], uint16(i.filenum)) + binary.BigEndian.PutUint32(out[offset+2:], i.offset) + return out } // bounds returns the start- and end- offsets, and the file number of where to @@ -107,7 +109,7 @@ type freezerTable struct { // to count how many historic items have gone missing. itemOffset uint32 // Offset (number of discarded items) - headBytes uint32 // Number of bytes written to the head file + headBytes int64 // Number of bytes written to the head file readMeter metrics.Meter // Meter for measuring the effective amount of data read writeMeter metrics.Meter // Meter for measuring the effective amount of data written sizeGauge metrics.Gauge // Gauge for tracking the combined size of all freezer tables @@ -118,12 +120,7 @@ type freezerTable struct { // NewFreezerTable opens the given path as a freezer table. func NewFreezerTable(path, name string, disableSnappy bool) (*freezerTable, error) { - return newTable(path, name, metrics.NilMeter{}, metrics.NilMeter{}, metrics.NilGauge{}, disableSnappy) -} - -// newTable opens a freezer table with default settings - 2G files -func newTable(path string, name string, readMeter metrics.Meter, writeMeter metrics.Meter, sizeGauge metrics.Gauge, disableSnappy bool) (*freezerTable, error) { - return newCustomTable(path, name, readMeter, writeMeter, sizeGauge, 2*1000*1000*1000, disableSnappy) + return newTable(path, name, metrics.NilMeter{}, metrics.NilMeter{}, metrics.NilGauge{}, freezerTableSize, disableSnappy) } // openFreezerFileForAppend opens a freezer table file and seeks to the end @@ -164,10 +161,10 @@ func truncateFreezerFile(file *os.File, size int64) error { return nil } -// newCustomTable opens a freezer table, creating the data and index files if they are +// newTable opens a freezer table, creating the data and index files if they are // non existent. Both files are truncated to the shortest common length to ensure // they don't go out of sync. -func newCustomTable(path string, name string, readMeter metrics.Meter, writeMeter metrics.Meter, sizeGauge metrics.Gauge, maxFilesize uint32, noCompression bool) (*freezerTable, error) { +func newTable(path string, name string, readMeter metrics.Meter, writeMeter metrics.Meter, sizeGauge metrics.Gauge, maxFilesize uint32, noCompression bool) (*freezerTable, error) { // Ensure the containing directory exists and open the indexEntry file if err := os.MkdirAll(path, 0755); err != nil { return nil, err @@ -313,7 +310,7 @@ func (t *freezerTable) repair() error { } // Update the item and byte counters and return t.items = uint64(t.itemOffset) + uint64(offsetsSize/indexEntrySize-1) // last indexEntry points to the end of the data file - t.headBytes = uint32(contentSize) + t.headBytes = contentSize t.headId = lastIndex.filenum // Close opened files and preopen all files @@ -387,14 +384,14 @@ func (t *freezerTable) truncate(items uint64) error { t.releaseFilesAfter(expected.filenum, true) // Set back the historic head t.head = newHead - atomic.StoreUint32(&t.headId, expected.filenum) + t.headId = expected.filenum } if err := truncateFreezerFile(t.head, int64(expected.offset)); err != nil { return err } // All data files truncated, set internal counters and return + t.headBytes = int64(expected.offset) atomic.StoreUint64(&t.items, items) - atomic.StoreUint32(&t.headBytes, expected.offset) // Retrieve the new size and update the total size counter newSize, err := t.sizeNolock() @@ -471,94 +468,6 @@ func (t *freezerTable) releaseFilesAfter(num uint32, remove bool) { } } -// Append injects a binary blob at the end of the freezer table. The item number -// is a precautionary parameter to ensure data correctness, but the table will -// reject already existing data. -// -// Note, this method will *not* flush any data to disk so be sure to explicitly -// fsync before irreversibly deleting data from the database. -func (t *freezerTable) Append(item uint64, blob []byte) error { - // Encode the blob before the lock portion - if !t.noCompression { - blob = snappy.Encode(nil, blob) - } - // Read lock prevents competition with truncate - retry, err := t.append(item, blob, false) - if err != nil { - return err - } - if retry { - // Read lock was insufficient, retry with a writelock - _, err = t.append(item, blob, true) - } - return err -} - -// append injects a binary blob at the end of the freezer table. -// Normally, inserts do not require holding the write-lock, so it should be invoked with 'wlock' set to -// false. -// However, if the data will grown the current file out of bounds, then this -// method will return 'true, nil', indicating that the caller should retry, this time -// with 'wlock' set to true. -func (t *freezerTable) append(item uint64, encodedBlob []byte, wlock bool) (bool, error) { - if wlock { - t.lock.Lock() - defer t.lock.Unlock() - } else { - t.lock.RLock() - defer t.lock.RUnlock() - } - // Ensure the table is still accessible - if t.index == nil || t.head == nil { - return false, errClosed - } - // Ensure only the next item can be written, nothing else - if atomic.LoadUint64(&t.items) != item { - return false, fmt.Errorf("appending unexpected item: want %d, have %d", t.items, item) - } - bLen := uint32(len(encodedBlob)) - if t.headBytes+bLen < bLen || - t.headBytes+bLen > t.maxFileSize { - // Writing would overflow, so we need to open a new data file. - // If we don't already hold the writelock, abort and let the caller - // invoke this method a second time. - if !wlock { - return true, nil - } - nextID := atomic.LoadUint32(&t.headId) + 1 - // We open the next file in truncated mode -- if this file already - // exists, we need to start over from scratch on it - newHead, err := t.openFile(nextID, openFreezerFileTruncated) - if err != nil { - return false, err - } - // Close old file, and reopen in RDONLY mode - t.releaseFile(t.headId) - t.openFile(t.headId, openFreezerFileForReadOnly) - - // Swap out the current head - t.head = newHead - atomic.StoreUint32(&t.headBytes, 0) - atomic.StoreUint32(&t.headId, nextID) - } - if _, err := t.head.Write(encodedBlob); err != nil { - return false, err - } - newOffset := atomic.AddUint32(&t.headBytes, bLen) - idx := indexEntry{ - filenum: atomic.LoadUint32(&t.headId), - offset: newOffset, - } - // Write indexEntry - t.index.Write(idx.marshallBinary()) - - t.writeMeter.Mark(int64(bLen + indexEntrySize)) - t.sizeGauge.Inc(int64(bLen + indexEntrySize)) - - atomic.AddUint64(&t.items, 1) - return false, nil -} - // getIndices returns the index entries for the given from-item, covering 'count' items. // N.B: The actual number of returned indices for N items will always be N+1 (unless an // error is returned). @@ -651,6 +560,7 @@ func (t *freezerTable) RetrieveItems(start, count, maxBytes uint64) ([][]byte, e func (t *freezerTable) retrieveItems(start, count, maxBytes uint64) ([]byte, []int, error) { t.lock.RLock() defer t.lock.RUnlock() + // Ensure the table and the item is accessible if t.index == nil || t.head == nil { return nil, nil, errClosed @@ -763,6 +673,32 @@ func (t *freezerTable) sizeNolock() (uint64, error) { return total, nil } +// advanceHead should be called when the current head file would outgrow the file limits, +// and a new file must be opened. The caller of this method must hold the write-lock +// before calling this method. +func (t *freezerTable) advanceHead() error { + t.lock.Lock() + defer t.lock.Unlock() + + // We open the next file in truncated mode -- if this file already + // exists, we need to start over from scratch on it. + nextID := t.headId + 1 + newHead, err := t.openFile(nextID, openFreezerFileTruncated) + if err != nil { + return err + } + + // Close old file, and reopen in RDONLY mode. + t.releaseFile(t.headId) + t.openFile(t.headId, openFreezerFileForReadOnly) + + // Swap out the current head. + t.head = newHead + t.headBytes = 0 + t.headId = nextID + return nil +} + // Sync pushes any pending data from memory out to disk. This is an expensive // operation, so use it with care. func (t *freezerTable) Sync() error { @@ -775,10 +711,21 @@ func (t *freezerTable) Sync() error { // DumpIndex is a debug print utility function, mainly for testing. It can also // be used to analyse a live freezer table index. func (t *freezerTable) DumpIndex(start, stop int64) { + t.dumpIndex(os.Stdout, start, stop) +} + +func (t *freezerTable) dumpIndexString(start, stop int64) string { + var out bytes.Buffer + out.WriteString("\n") + t.dumpIndex(&out, start, stop) + return out.String() +} + +func (t *freezerTable) dumpIndex(w io.Writer, start, stop int64) { buf := make([]byte, indexEntrySize) - fmt.Printf("| number | fileno | offset |\n") - fmt.Printf("|--------|--------|--------|\n") + fmt.Fprintf(w, "| number | fileno | offset |\n") + fmt.Fprintf(w, "|--------|--------|--------|\n") for i := uint64(start); ; i++ { if _, err := t.index.ReadAt(buf, int64(i*indexEntrySize)); err != nil { @@ -786,10 +733,10 @@ func (t *freezerTable) DumpIndex(start, stop int64) { } var entry indexEntry entry.unmarshalBinary(buf) - fmt.Printf("| %03d | %03d | %03d | \n", i, entry.filenum, entry.offset) + fmt.Fprintf(w, "| %03d | %03d | %03d | \n", i, entry.filenum, entry.offset) if stop > 0 && i >= uint64(stop) { break } } - fmt.Printf("|--------------------------|\n") + fmt.Fprintf(w, "|--------------------------|\n") } diff --git a/core/rawdb/freezer_table_test.go b/core/rawdb/freezer_table_test.go index e8a8b5c46..803809b52 100644 --- a/core/rawdb/freezer_table_test.go +++ b/core/rawdb/freezer_table_test.go @@ -18,49 +18,36 @@ package rawdb import ( "bytes" - "encoding/binary" "fmt" - "io/ioutil" "math/rand" "os" "path/filepath" - "sync" "testing" "time" "github.com/ethereum/go-ethereum/metrics" + "github.com/stretchr/testify/require" ) func init() { rand.Seed(time.Now().Unix()) } -// Gets a chunk of data, filled with 'b' -func getChunk(size int, b int) []byte { - data := make([]byte, size) - for i := range data { - data[i] = byte(b) - } - return data -} - // TestFreezerBasics test initializing a freezertable from scratch, writing to the table, // and reading it back. func TestFreezerBasics(t *testing.T) { t.Parallel() // set cutoff at 50 bytes - f, err := newCustomTable(os.TempDir(), + f, err := newTable(os.TempDir(), fmt.Sprintf("unittest-%d", rand.Uint64()), metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge(), 50, true) if err != nil { t.Fatal(err) } defer f.Close() + // Write 15 bytes 255 times, results in 85 files - for x := 0; x < 255; x++ { - data := getChunk(15, x) - f.Append(uint64(x), data) - } + writeChunks(t, f, 255, 15) //print(t, f, 0) //print(t, f, 1) @@ -98,16 +85,21 @@ func TestFreezerBasicsClosing(t *testing.T) { f *freezerTable err error ) - f, err = newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true) + f, err = newTable(os.TempDir(), fname, rm, wm, sg, 50, true) if err != nil { t.Fatal(err) } - // Write 15 bytes 255 times, results in 85 files + + // Write 15 bytes 255 times, results in 85 files. + // In-between writes, the table is closed and re-opened. for x := 0; x < 255; x++ { data := getChunk(15, x) - f.Append(uint64(x), data) + batch := f.newBatch() + require.NoError(t, batch.AppendRaw(uint64(x), data)) + require.NoError(t, batch.commit()) f.Close() - f, err = newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true) + + f, err = newTable(os.TempDir(), fname, rm, wm, sg, 50, true) if err != nil { t.Fatal(err) } @@ -124,7 +116,7 @@ func TestFreezerBasicsClosing(t *testing.T) { t.Fatalf("test %d, got \n%x != \n%x", y, got, exp) } f.Close() - f, err = newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true) + f, err = newTable(os.TempDir(), fname, rm, wm, sg, 50, true) if err != nil { t.Fatal(err) } @@ -137,22 +129,22 @@ func TestFreezerRepairDanglingHead(t *testing.T) { rm, wm, sg := metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge() fname := fmt.Sprintf("dangling_headtest-%d", rand.Uint64()) - { // Fill table - f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true) + // Fill table + { + f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true) if err != nil { t.Fatal(err) } // Write 15 bytes 255 times - for x := 0; x < 255; x++ { - data := getChunk(15, x) - f.Append(uint64(x), data) - } + writeChunks(t, f, 255, 15) + // The last item should be there if _, err = f.Retrieve(0xfe); err != nil { t.Fatal(err) } f.Close() } + // open the index idxFile, err := os.OpenFile(filepath.Join(os.TempDir(), fmt.Sprintf("%s.ridx", fname)), os.O_RDWR, 0644) if err != nil { @@ -165,9 +157,10 @@ func TestFreezerRepairDanglingHead(t *testing.T) { } idxFile.Truncate(stat.Size() - 4) idxFile.Close() + // Now open it again { - f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true) + f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true) if err != nil { t.Fatal(err) } @@ -188,22 +181,22 @@ func TestFreezerRepairDanglingHeadLarge(t *testing.T) { rm, wm, sg := metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge() fname := fmt.Sprintf("dangling_headtest-%d", rand.Uint64()) - { // Fill a table and close it - f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true) + // Fill a table and close it + { + f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true) if err != nil { t.Fatal(err) } // Write 15 bytes 255 times - for x := 0; x < 0xff; x++ { - data := getChunk(15, x) - f.Append(uint64(x), data) - } + writeChunks(t, f, 255, 15) + // The last item should be there if _, err = f.Retrieve(f.items - 1); err != nil { t.Fatal(err) } f.Close() } + // open the index idxFile, err := os.OpenFile(filepath.Join(os.TempDir(), fmt.Sprintf("%s.ridx", fname)), os.O_RDWR, 0644) if err != nil { @@ -213,9 +206,10 @@ func TestFreezerRepairDanglingHeadLarge(t *testing.T) { // 0-indexEntry, 1-indexEntry, corrupt-indexEntry idxFile.Truncate(indexEntrySize + indexEntrySize + indexEntrySize/2) idxFile.Close() + // Now open it again { - f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true) + f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true) if err != nil { t.Fatal(err) } @@ -228,15 +222,17 @@ func TestFreezerRepairDanglingHeadLarge(t *testing.T) { t.Errorf("Expected error for missing index entry") } // We should now be able to store items again, from item = 1 + batch := f.newBatch() for x := 1; x < 0xff; x++ { - data := getChunk(15, ^x) - f.Append(uint64(x), data) + require.NoError(t, batch.AppendRaw(uint64(x), getChunk(15, ^x))) } + require.NoError(t, batch.commit()) f.Close() } + // And if we open it, we should now be able to read all of them (new values) { - f, _ := newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true) + f, _ := newTable(os.TempDir(), fname, rm, wm, sg, 50, true) for y := 1; y < 255; y++ { exp := getChunk(15, ^y) got, err := f.Retrieve(uint64(y)) @@ -255,22 +251,21 @@ func TestSnappyDetection(t *testing.T) { t.Parallel() rm, wm, sg := metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge() fname := fmt.Sprintf("snappytest-%d", rand.Uint64()) + // Open with snappy { - f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true) + f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true) if err != nil { t.Fatal(err) } // Write 15 bytes 255 times - for x := 0; x < 0xff; x++ { - data := getChunk(15, x) - f.Append(uint64(x), data) - } + writeChunks(t, f, 255, 15) f.Close() } + // Open without snappy { - f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, false) + f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, false) if err != nil { t.Fatal(err) } @@ -282,7 +277,7 @@ func TestSnappyDetection(t *testing.T) { // Open with snappy { - f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true) + f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true) if err != nil { t.Fatal(err) } @@ -292,8 +287,8 @@ func TestSnappyDetection(t *testing.T) { t.Fatalf("expected no error, got %v", err) } } - } + func assertFileSize(f string, size int64) error { stat, err := os.Stat(f) if err != nil { @@ -303,7 +298,6 @@ func assertFileSize(f string, size int64) error { return fmt.Errorf("error, expected size %d, got %d", size, stat.Size()) } return nil - } // TestFreezerRepairDanglingIndex checks that if the index has more entries than there are data, @@ -313,16 +307,15 @@ func TestFreezerRepairDanglingIndex(t *testing.T) { rm, wm, sg := metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge() fname := fmt.Sprintf("dangling_indextest-%d", rand.Uint64()) - { // Fill a table and close it - f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true) + // Fill a table and close it + { + f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true) if err != nil { t.Fatal(err) } // Write 15 bytes 9 times : 150 bytes - for x := 0; x < 9; x++ { - data := getChunk(15, x) - f.Append(uint64(x), data) - } + writeChunks(t, f, 9, 15) + // The last item should be there if _, err = f.Retrieve(f.items - 1); err != nil { f.Close() @@ -331,6 +324,7 @@ func TestFreezerRepairDanglingIndex(t *testing.T) { f.Close() // File sizes should be 45, 45, 45 : items[3, 3, 3) } + // Crop third file fileToCrop := filepath.Join(os.TempDir(), fmt.Sprintf("%s.0002.rdat", fname)) // Truncate third file: 45 ,45, 20 @@ -345,17 +339,18 @@ func TestFreezerRepairDanglingIndex(t *testing.T) { file.Truncate(20) file.Close() } + // Open db it again // It should restore the file(s) to // 45, 45, 15 // with 3+3+1 items { - f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true) + f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true) if err != nil { t.Fatal(err) } + defer f.Close() if f.items != 7 { - f.Close() t.Fatalf("expected %d items, got %d", 7, f.items) } if err := assertFileSize(fileToCrop, 15); err != nil { @@ -365,30 +360,29 @@ func TestFreezerRepairDanglingIndex(t *testing.T) { } func TestFreezerTruncate(t *testing.T) { - t.Parallel() rm, wm, sg := metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge() fname := fmt.Sprintf("truncation-%d", rand.Uint64()) - { // Fill table - f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true) + // Fill table + { + f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true) if err != nil { t.Fatal(err) } // Write 15 bytes 30 times - for x := 0; x < 30; x++ { - data := getChunk(15, x) - f.Append(uint64(x), data) - } + writeChunks(t, f, 30, 15) + // The last item should be there if _, err = f.Retrieve(f.items - 1); err != nil { t.Fatal(err) } f.Close() } + // Reopen, truncate { - f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true) + f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true) if err != nil { t.Fatal(err) } @@ -401,9 +395,7 @@ func TestFreezerTruncate(t *testing.T) { if f.headBytes != 15 { t.Fatalf("expected %d bytes, got %d", 15, f.headBytes) } - } - } // TestFreezerRepairFirstFile tests a head file with the very first item only half-written. @@ -412,20 +404,26 @@ func TestFreezerRepairFirstFile(t *testing.T) { t.Parallel() rm, wm, sg := metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge() fname := fmt.Sprintf("truncationfirst-%d", rand.Uint64()) - { // Fill table - f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true) + + // Fill table + { + f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true) if err != nil { t.Fatal(err) } // Write 80 bytes, splitting out into two files - f.Append(0, getChunk(40, 0xFF)) - f.Append(1, getChunk(40, 0xEE)) + batch := f.newBatch() + require.NoError(t, batch.AppendRaw(0, getChunk(40, 0xFF))) + require.NoError(t, batch.AppendRaw(1, getChunk(40, 0xEE))) + require.NoError(t, batch.commit()) + // The last item should be there - if _, err = f.Retrieve(f.items - 1); err != nil { + if _, err = f.Retrieve(1); err != nil { t.Fatal(err) } f.Close() } + // Truncate the file in half fileToCrop := filepath.Join(os.TempDir(), fmt.Sprintf("%s.0001.rdat", fname)) { @@ -439,9 +437,10 @@ func TestFreezerRepairFirstFile(t *testing.T) { file.Truncate(20) file.Close() } + // Reopen { - f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true) + f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true) if err != nil { t.Fatal(err) } @@ -449,9 +448,14 @@ func TestFreezerRepairFirstFile(t *testing.T) { f.Close() t.Fatalf("expected %d items, got %d", 0, f.items) } + // Write 40 bytes - f.Append(1, getChunk(40, 0xDD)) + batch := f.newBatch() + require.NoError(t, batch.AppendRaw(1, getChunk(40, 0xDD))) + require.NoError(t, batch.commit()) + f.Close() + // Should have been truncated down to zero and then 40 written if err := assertFileSize(fileToCrop, 40); err != nil { t.Fatal(err) @@ -468,25 +472,26 @@ func TestFreezerReadAndTruncate(t *testing.T) { t.Parallel() rm, wm, sg := metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge() fname := fmt.Sprintf("read_truncate-%d", rand.Uint64()) - { // Fill table - f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true) + + // Fill table + { + f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true) if err != nil { t.Fatal(err) } // Write 15 bytes 30 times - for x := 0; x < 30; x++ { - data := getChunk(15, x) - f.Append(uint64(x), data) - } + writeChunks(t, f, 30, 15) + // The last item should be there if _, err = f.Retrieve(f.items - 1); err != nil { t.Fatal(err) } f.Close() } + // Reopen and read all files { - f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true) + f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true) if err != nil { t.Fatal(err) } @@ -497,40 +502,48 @@ func TestFreezerReadAndTruncate(t *testing.T) { for y := byte(0); y < 30; y++ { f.Retrieve(uint64(y)) } + // Now, truncate back to zero f.truncate(0) + // Write the data again + batch := f.newBatch() for x := 0; x < 30; x++ { - data := getChunk(15, ^x) - if err := f.Append(uint64(x), data); err != nil { - t.Fatalf("error %v", err) - } + require.NoError(t, batch.AppendRaw(uint64(x), getChunk(15, ^x))) } + require.NoError(t, batch.commit()) f.Close() } } -func TestOffset(t *testing.T) { +func TestFreezerOffset(t *testing.T) { t.Parallel() rm, wm, sg := metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge() fname := fmt.Sprintf("offset-%d", rand.Uint64()) - { // Fill table - f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 40, true) + + // Fill table + { + f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, true) if err != nil { t.Fatal(err) } + // Write 6 x 20 bytes, splitting out into three files - f.Append(0, getChunk(20, 0xFF)) - f.Append(1, getChunk(20, 0xEE)) + batch := f.newBatch() + require.NoError(t, batch.AppendRaw(0, getChunk(20, 0xFF))) + require.NoError(t, batch.AppendRaw(1, getChunk(20, 0xEE))) - f.Append(2, getChunk(20, 0xdd)) - f.Append(3, getChunk(20, 0xcc)) + require.NoError(t, batch.AppendRaw(2, getChunk(20, 0xdd))) + require.NoError(t, batch.AppendRaw(3, getChunk(20, 0xcc))) - f.Append(4, getChunk(20, 0xbb)) - f.Append(5, getChunk(20, 0xaa)) - f.DumpIndex(0, 100) + require.NoError(t, batch.AppendRaw(4, getChunk(20, 0xbb))) + require.NoError(t, batch.AppendRaw(5, getChunk(20, 0xaa))) + require.NoError(t, batch.commit()) + + t.Log(f.dumpIndexString(0, 100)) f.Close() } + // Now crop it. { // delete files 0 and 1 @@ -558,7 +571,7 @@ func TestOffset(t *testing.T) { filenum: tailId, offset: itemOffset, } - buf := zeroIndex.marshallBinary() + buf := zeroIndex.append(nil) // Overwrite index zero copy(indexBuf, buf) // Remove the four next indices by overwriting @@ -567,44 +580,36 @@ func TestOffset(t *testing.T) { // Need to truncate the moved index items indexFile.Truncate(indexEntrySize * (1 + 2)) indexFile.Close() - } + // Now open again - checkPresent := func(numDeleted uint64) { - f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 40, true) + { + f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, true) if err != nil { t.Fatal(err) } - f.DumpIndex(0, 100) - // It should allow writing item 6 - f.Append(numDeleted+2, getChunk(20, 0x99)) + defer f.Close() + t.Log(f.dumpIndexString(0, 100)) - // It should be fine to fetch 4,5,6 - if got, err := f.Retrieve(numDeleted); err != nil { - t.Fatal(err) - } else if exp := getChunk(20, 0xbb); !bytes.Equal(got, exp) { - t.Fatalf("expected %x got %x", exp, got) - } - if got, err := f.Retrieve(numDeleted + 1); err != nil { - t.Fatal(err) - } else if exp := getChunk(20, 0xaa); !bytes.Equal(got, exp) { - t.Fatalf("expected %x got %x", exp, got) - } - if got, err := f.Retrieve(numDeleted + 2); err != nil { - t.Fatal(err) - } else if exp := getChunk(20, 0x99); !bytes.Equal(got, exp) { - t.Fatalf("expected %x got %x", exp, got) - } + // It should allow writing item 6. + batch := f.newBatch() + require.NoError(t, batch.AppendRaw(6, getChunk(20, 0x99))) + require.NoError(t, batch.commit()) - // It should error at 0, 1,2,3 - for i := numDeleted - 1; i > numDeleted-10; i-- { - if _, err := f.Retrieve(i); err == nil { - t.Fatal("expected err") - } - } + checkRetrieveError(t, f, map[uint64]error{ + 0: errOutOfBounds, + 1: errOutOfBounds, + 2: errOutOfBounds, + 3: errOutOfBounds, + }) + checkRetrieve(t, f, map[uint64][]byte{ + 4: getChunk(20, 0xbb), + 5: getChunk(20, 0xaa), + 6: getChunk(20, 0x99), + }) } - checkPresent(4) - // Now, let's pretend we have deleted 1M items + + // Edit the index again, with a much larger initial offset of 1M. { // Read the index file p := filepath.Join(os.TempDir(), fmt.Sprintf("%v.ridx", fname)) @@ -624,13 +629,71 @@ func TestOffset(t *testing.T) { offset: itemOffset, filenum: tailId, } - buf := zeroIndex.marshallBinary() + buf := zeroIndex.append(nil) // Overwrite index zero copy(indexBuf, buf) indexFile.WriteAt(indexBuf, 0) indexFile.Close() } - checkPresent(1000000) + + // Check that existing items have been moved to index 1M. + { + f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, true) + if err != nil { + t.Fatal(err) + } + defer f.Close() + t.Log(f.dumpIndexString(0, 100)) + + checkRetrieveError(t, f, map[uint64]error{ + 0: errOutOfBounds, + 1: errOutOfBounds, + 2: errOutOfBounds, + 3: errOutOfBounds, + 999999: errOutOfBounds, + }) + checkRetrieve(t, f, map[uint64][]byte{ + 1000000: getChunk(20, 0xbb), + 1000001: getChunk(20, 0xaa), + }) + } +} + +func checkRetrieve(t *testing.T, f *freezerTable, items map[uint64][]byte) { + t.Helper() + + for item, wantBytes := range items { + value, err := f.Retrieve(item) + if err != nil { + t.Fatalf("can't get expected item %d: %v", item, err) + } + if !bytes.Equal(value, wantBytes) { + t.Fatalf("item %d has wrong value %x (want %x)", item, value, wantBytes) + } + } +} + +func checkRetrieveError(t *testing.T, f *freezerTable, items map[uint64]error) { + t.Helper() + + for item, wantError := range items { + value, err := f.Retrieve(item) + if err == nil { + t.Fatalf("unexpected value %x for item %d, want error %v", item, value, wantError) + } + if err != wantError { + t.Fatalf("wrong error for item %d: %v", item, err) + } + } +} + +// Gets a chunk of data, filled with 'b' +func getChunk(size int, b int) []byte { + data := make([]byte, size) + for i := range data { + data[i] = byte(b) + } + return data } // TODO (?) @@ -644,53 +707,18 @@ func TestOffset(t *testing.T) { // should be handled already, and the case described above can only (?) happen if an // external process/user deletes files from the filesystem. -// TestAppendTruncateParallel is a test to check if the Append/truncate operations are -// racy. -// -// The reason why it's not a regular fuzzer, within tests/fuzzers, is that it is dependent -// on timing rather than 'clever' input -- there's no determinism. -func TestAppendTruncateParallel(t *testing.T) { - dir, err := ioutil.TempDir("", "freezer") - if err != nil { - t.Fatal(err) - } - defer os.RemoveAll(dir) +func writeChunks(t *testing.T, ft *freezerTable, n int, length int) { + t.Helper() - f, err := newCustomTable(dir, "tmp", metrics.NilMeter{}, metrics.NilMeter{}, metrics.NilGauge{}, 8, true) - if err != nil { - t.Fatal(err) - } - - fill := func(mark uint64) []byte { - data := make([]byte, 8) - binary.LittleEndian.PutUint64(data, mark) - return data - } - - for i := 0; i < 5000; i++ { - f.truncate(0) - data0 := fill(0) - f.Append(0, data0) - data1 := fill(1) - - var wg sync.WaitGroup - wg.Add(2) - go func() { - f.truncate(0) - wg.Done() - }() - go func() { - f.Append(1, data1) - wg.Done() - }() - wg.Wait() - - if have, err := f.Retrieve(0); err == nil { - if !bytes.Equal(have, data0) { - t.Fatalf("have %x want %x", have, data0) - } + batch := ft.newBatch() + for i := 0; i < n; i++ { + if err := batch.AppendRaw(uint64(i), getChunk(length, i)); err != nil { + t.Fatalf("AppendRaw(%d, ...) returned error: %v", i, err) } } + if err := batch.commit(); err != nil { + t.Fatalf("Commit returned error: %v", err) + } } // TestSequentialRead does some basic tests on the RetrieveItems. @@ -698,20 +726,17 @@ func TestSequentialRead(t *testing.T) { rm, wm, sg := metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge() fname := fmt.Sprintf("batchread-%d", rand.Uint64()) { // Fill table - f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true) + f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true) if err != nil { t.Fatal(err) } // Write 15 bytes 30 times - for x := 0; x < 30; x++ { - data := getChunk(15, x) - f.Append(uint64(x), data) - } + writeChunks(t, f, 30, 15) f.DumpIndex(0, 30) f.Close() } { // Open it, iterate, verify iteration - f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true) + f, err := newTable(os.TempDir(), fname, rm, wm, sg, 50, true) if err != nil { t.Fatal(err) } @@ -732,7 +757,7 @@ func TestSequentialRead(t *testing.T) { } { // Open it, iterate, verify byte limit. The byte limit is less than item // size, so each lookup should only return one item - f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 40, true) + f, err := newTable(os.TempDir(), fname, rm, wm, sg, 40, true) if err != nil { t.Fatal(err) } @@ -761,16 +786,13 @@ func TestSequentialReadByteLimit(t *testing.T) { rm, wm, sg := metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge() fname := fmt.Sprintf("batchread-2-%d", rand.Uint64()) { // Fill table - f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 100, true) + f, err := newTable(os.TempDir(), fname, rm, wm, sg, 100, true) if err != nil { t.Fatal(err) } // Write 10 bytes 30 times, // Splitting it at every 100 bytes (10 items) - for x := 0; x < 30; x++ { - data := getChunk(10, x) - f.Append(uint64(x), data) - } + writeChunks(t, f, 30, 10) f.Close() } for i, tc := range []struct { @@ -786,7 +808,7 @@ func TestSequentialReadByteLimit(t *testing.T) { {100, 109, 10}, } { { - f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 100, true) + f, err := newTable(os.TempDir(), fname, rm, wm, sg, 100, true) if err != nil { t.Fatal(err) } diff --git a/core/rawdb/freezer_test.go b/core/rawdb/freezer_test.go new file mode 100644 index 000000000..7359131c8 --- /dev/null +++ b/core/rawdb/freezer_test.go @@ -0,0 +1,301 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rawdb + +import ( + "bytes" + "errors" + "fmt" + "io/ioutil" + "math/big" + "math/rand" + "os" + "sync" + "testing" + + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/rlp" + "github.com/stretchr/testify/require" +) + +var freezerTestTableDef = map[string]bool{"test": true} + +func TestFreezerModify(t *testing.T) { + t.Parallel() + + // Create test data. + var valuesRaw [][]byte + var valuesRLP []*big.Int + for x := 0; x < 100; x++ { + v := getChunk(256, x) + valuesRaw = append(valuesRaw, v) + iv := big.NewInt(int64(x)) + iv = iv.Exp(iv, iv, nil) + valuesRLP = append(valuesRLP, iv) + } + + tables := map[string]bool{"raw": true, "rlp": false} + f, dir := newFreezerForTesting(t, tables) + defer os.RemoveAll(dir) + defer f.Close() + + // Commit test data. + _, err := f.ModifyAncients(func(op ethdb.AncientWriteOp) error { + for i := range valuesRaw { + if err := op.AppendRaw("raw", uint64(i), valuesRaw[i]); err != nil { + return err + } + if err := op.Append("rlp", uint64(i), valuesRLP[i]); err != nil { + return err + } + } + return nil + }) + if err != nil { + t.Fatal("ModifyAncients failed:", err) + } + + // Dump indexes. + for _, table := range f.tables { + t.Log(table.name, "index:", table.dumpIndexString(0, int64(len(valuesRaw)))) + } + + // Read back test data. + checkAncientCount(t, f, "raw", uint64(len(valuesRaw))) + checkAncientCount(t, f, "rlp", uint64(len(valuesRLP))) + for i := range valuesRaw { + v, _ := f.Ancient("raw", uint64(i)) + if !bytes.Equal(v, valuesRaw[i]) { + t.Fatalf("wrong raw value at %d: %x", i, v) + } + ivEnc, _ := f.Ancient("rlp", uint64(i)) + want, _ := rlp.EncodeToBytes(valuesRLP[i]) + if !bytes.Equal(ivEnc, want) { + t.Fatalf("wrong RLP value at %d: %x", i, ivEnc) + } + } +} + +// This checks that ModifyAncients rolls back freezer updates +// when the function passed to it returns an error. +func TestFreezerModifyRollback(t *testing.T) { + t.Parallel() + + f, dir := newFreezerForTesting(t, freezerTestTableDef) + defer os.RemoveAll(dir) + + theError := errors.New("oops") + _, err := f.ModifyAncients(func(op ethdb.AncientWriteOp) error { + // Append three items. This creates two files immediately, + // because the table size limit of the test freezer is 2048. + require.NoError(t, op.AppendRaw("test", 0, make([]byte, 2048))) + require.NoError(t, op.AppendRaw("test", 1, make([]byte, 2048))) + require.NoError(t, op.AppendRaw("test", 2, make([]byte, 2048))) + return theError + }) + if err != theError { + t.Errorf("ModifyAncients returned wrong error %q", err) + } + checkAncientCount(t, f, "test", 0) + f.Close() + + // Reopen and check that the rolled-back data doesn't reappear. + tables := map[string]bool{"test": true} + f2, err := newFreezer(dir, "", false, 2049, tables) + if err != nil { + t.Fatalf("can't reopen freezer after failed ModifyAncients: %v", err) + } + defer f2.Close() + checkAncientCount(t, f2, "test", 0) +} + +// This test runs ModifyAncients and Ancient concurrently with each other. +func TestFreezerConcurrentModifyRetrieve(t *testing.T) { + t.Parallel() + + f, dir := newFreezerForTesting(t, freezerTestTableDef) + defer os.RemoveAll(dir) + defer f.Close() + + var ( + numReaders = 5 + writeBatchSize = uint64(50) + written = make(chan uint64, numReaders*6) + wg sync.WaitGroup + ) + wg.Add(numReaders + 1) + + // Launch the writer. It appends 10000 items in batches. + go func() { + defer wg.Done() + defer close(written) + for item := uint64(0); item < 10000; item += writeBatchSize { + _, err := f.ModifyAncients(func(op ethdb.AncientWriteOp) error { + for i := uint64(0); i < writeBatchSize; i++ { + item := item + i + value := getChunk(32, int(item)) + if err := op.AppendRaw("test", item, value); err != nil { + return err + } + } + return nil + }) + if err != nil { + panic(err) + } + for i := 0; i < numReaders; i++ { + written <- item + writeBatchSize + } + } + }() + + // Launch the readers. They read random items from the freezer up to the + // current frozen item count. + for i := 0; i < numReaders; i++ { + go func() { + defer wg.Done() + for frozen := range written { + for rc := 0; rc < 80; rc++ { + num := uint64(rand.Intn(int(frozen))) + value, err := f.Ancient("test", num) + if err != nil { + panic(fmt.Errorf("error reading %d (frozen %d): %v", num, frozen, err)) + } + if !bytes.Equal(value, getChunk(32, int(num))) { + panic(fmt.Errorf("wrong value at %d", num)) + } + } + } + }() + } + + wg.Wait() +} + +// This test runs ModifyAncients and TruncateAncients concurrently with each other. +func TestFreezerConcurrentModifyTruncate(t *testing.T) { + f, dir := newFreezerForTesting(t, freezerTestTableDef) + defer os.RemoveAll(dir) + defer f.Close() + + var item = make([]byte, 256) + + for i := 0; i < 1000; i++ { + // First reset and write 100 items. + if err := f.TruncateAncients(0); err != nil { + t.Fatal("truncate failed:", err) + } + _, err := f.ModifyAncients(func(op ethdb.AncientWriteOp) error { + for i := uint64(0); i < 100; i++ { + if err := op.AppendRaw("test", i, item); err != nil { + return err + } + } + return nil + }) + if err != nil { + t.Fatal("modify failed:", err) + } + checkAncientCount(t, f, "test", 100) + + // Now append 100 more items and truncate concurrently. + var ( + wg sync.WaitGroup + truncateErr error + modifyErr error + ) + wg.Add(3) + go func() { + _, modifyErr = f.ModifyAncients(func(op ethdb.AncientWriteOp) error { + for i := uint64(100); i < 200; i++ { + if err := op.AppendRaw("test", i, item); err != nil { + return err + } + } + return nil + }) + wg.Done() + }() + go func() { + truncateErr = f.TruncateAncients(10) + wg.Done() + }() + go func() { + f.AncientSize("test") + wg.Done() + }() + wg.Wait() + + // Now check the outcome. If the truncate operation went through first, the append + // fails, otherwise it succeeds. In either case, the freezer should be positioned + // at 10 after both operations are done. + if truncateErr != nil { + t.Fatal("concurrent truncate failed:", err) + } + if !(modifyErr == nil || modifyErr == errOutOrderInsertion) { + t.Fatal("wrong error from concurrent modify:", modifyErr) + } + checkAncientCount(t, f, "test", 10) + } +} + +func newFreezerForTesting(t *testing.T, tables map[string]bool) (*freezer, string) { + t.Helper() + + dir, err := ioutil.TempDir("", "freezer") + if err != nil { + t.Fatal(err) + } + // note: using low max table size here to ensure the tests actually + // switch between multiple files. + f, err := newFreezer(dir, "", false, 2049, tables) + if err != nil { + t.Fatal("can't open freezer", err) + } + return f, dir +} + +// checkAncientCount verifies that the freezer contains n items. +func checkAncientCount(t *testing.T, f *freezer, kind string, n uint64) { + t.Helper() + + if frozen, _ := f.Ancients(); frozen != n { + t.Fatalf("Ancients() returned %d, want %d", frozen, n) + } + + // Check at index n-1. + if n > 0 { + index := n - 1 + if ok, _ := f.HasAncient(kind, index); !ok { + t.Errorf("HasAncient(%q, %d) returned false unexpectedly", kind, index) + } + if _, err := f.Ancient(kind, index); err != nil { + t.Errorf("Ancient(%q, %d) returned unexpected error %q", kind, index, err) + } + } + + // Check at index n. + index := n + if ok, _ := f.HasAncient(kind, index); ok { + t.Errorf("HasAncient(%q, %d) returned true unexpectedly", kind, index) + } + if _, err := f.Ancient(kind, index); err == nil { + t.Errorf("Ancient(%q, %d) didn't return expected error", kind, index) + } else if err != errOutOfBounds { + t.Errorf("Ancient(%q, %d) returned unexpected error %q", kind, index, err) + } +} diff --git a/core/rawdb/table.go b/core/rawdb/table.go index 586451c06..02e23b517 100644 --- a/core/rawdb/table.go +++ b/core/rawdb/table.go @@ -80,10 +80,9 @@ func (t *table) AncientSize(kind string) (uint64, error) { return t.db.AncientSize(kind) } -// AppendAncient is a noop passthrough that just forwards the request to the underlying -// database. -func (t *table) AppendAncient(number uint64, hash, header, body, receipts, td []byte) error { - return t.db.AppendAncient(number, hash, header, body, receipts, td) +// ModifyAncients runs an ancient write operation on the underlying database. +func (t *table) ModifyAncients(fn func(ethdb.AncientWriteOp) error) (int64, error) { + return t.db.ModifyAncients(fn) } // TruncateAncients is a noop passthrough that just forwards the request to the underlying diff --git a/ethdb/database.go b/ethdb/database.go index bdc09d5e9..3c6500d1d 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -92,9 +92,10 @@ type AncientReader interface { // AncientWriter contains the methods required to write to immutable ancient data. type AncientWriter interface { - // AppendAncient injects all binary blobs belong to block at the end of the - // append-only immutable table files. - AppendAncient(number uint64, hash, header, body, receipt, td []byte) error + // ModifyAncients runs a write operation on the ancient store. + // If the function returns an error, any changes to the underlying store are reverted. + // The integer return value is the total size of the written data. + ModifyAncients(func(AncientWriteOp) error) (int64, error) // TruncateAncients discards all but the first n ancient data from the ancient store. TruncateAncients(n uint64) error @@ -103,6 +104,15 @@ type AncientWriter interface { Sync() error } +// AncientWriteOp is given to the function argument of ModifyAncients. +type AncientWriteOp interface { + // Append adds an RLP-encoded item. + Append(kind string, number uint64, item interface{}) error + + // AppendRaw adds an item without RLP-encoding it. + AppendRaw(kind string, number uint64, item []byte) error +} + // Reader contains the methods required to read data from both key-value as well as // immutable ancient data. type Reader interface { From 6ef3a168692e34f6e356454e1c361803ded6e712 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 7 Sep 2021 13:36:48 +0300 Subject: [PATCH 42/73] p2p/enode: use unix timestamp as base ENR sequence number (#19903) This PR ensures that wiping all data associated with a node (apart from its nodekey) will not generate already used sequence number for the ENRs, since all remote nodes would reject them until they out-number the previously published largest one. The big complication with this scheme is that every local update to the ENR can potentially bump the sequence number by one. In order to ensure that local updates do not outrun the clock, the sequence number is a millisecond-precision timestamp, and updates are throttled to occur at most once per millisecond. Co-authored-by: Felix Lange --- p2p/discover/v5wire/encoding_test.go | 3 -- p2p/enode/localnode.go | 45 ++++++++++++++++++++++++++-- p2p/enode/localnode_test.go | 33 ++++++++++++-------- p2p/enode/nodedb.go | 9 ++++-- 4 files changed, 69 insertions(+), 21 deletions(-) diff --git a/p2p/discover/v5wire/encoding_test.go b/p2p/discover/v5wire/encoding_test.go index d9c807e0a..b13041d1b 100644 --- a/p2p/discover/v5wire/encoding_test.go +++ b/p2p/discover/v5wire/encoding_test.go @@ -512,9 +512,6 @@ func (n *handshakeTestNode) init(key *ecdsa.PrivateKey, ip net.IP, clock mclock. db, _ := enode.OpenDB("") n.ln = enode.NewLocalNode(db, key) n.ln.SetStaticIP(ip) - if n.ln.Node().Seq() != 1 { - panic(fmt.Errorf("unexpected seq %d", n.ln.Node().Seq())) - } n.c = NewCodec(n.ln, key, clock) } diff --git a/p2p/enode/localnode.go b/p2p/enode/localnode.go index d8aa02a77..4827b6c0a 100644 --- a/p2p/enode/localnode.go +++ b/p2p/enode/localnode.go @@ -36,20 +36,25 @@ const ( iptrackMinStatements = 10 iptrackWindow = 5 * time.Minute iptrackContactWindow = 10 * time.Minute + + // time needed to wait between two updates to the local ENR + recordUpdateThrottle = time.Millisecond ) // LocalNode produces the signed node record of a local node, i.e. a node run in the // current process. Setting ENR entries via the Set method updates the record. A new version // of the record is signed on demand when the Node method is called. type LocalNode struct { - cur atomic.Value // holds a non-nil node pointer while the record is up-to-date. + cur atomic.Value // holds a non-nil node pointer while the record is up-to-date + id ID key *ecdsa.PrivateKey db *DB // everything below is protected by a lock - mu sync.Mutex + mu sync.RWMutex seq uint64 + update time.Time // timestamp when the record was last updated entries map[string]enr.Entry endpoint4 lnEndpoint endpoint6 lnEndpoint @@ -76,7 +81,8 @@ func NewLocalNode(db *DB, key *ecdsa.PrivateKey) *LocalNode { }, } ln.seq = db.localSeq(ln.id) - ln.invalidate() + ln.update = time.Now() + ln.cur.Store((*Node)(nil)) return ln } @@ -87,14 +93,34 @@ func (ln *LocalNode) Database() *DB { // Node returns the current version of the local node record. func (ln *LocalNode) Node() *Node { + // If we have a valid record, return that n := ln.cur.Load().(*Node) if n != nil { return n } + // Record was invalidated, sign a new copy. ln.mu.Lock() defer ln.mu.Unlock() + + // Double check the current record, since multiple goroutines might be waiting + // on the write mutex. + if n = ln.cur.Load().(*Node); n != nil { + return n + } + + // The initial sequence number is the current timestamp in milliseconds. To ensure + // that the initial sequence number will always be higher than any previous sequence + // number (assuming the clock is correct), we want to avoid updating the record faster + // than once per ms. So we need to sleep here until the next possible update time has + // arrived. + lastChange := time.Since(ln.update) + if lastChange < recordUpdateThrottle { + time.Sleep(recordUpdateThrottle - lastChange) + } + ln.sign() + ln.update = time.Now() return ln.cur.Load().(*Node) } @@ -114,6 +140,10 @@ func (ln *LocalNode) ID() ID { // Set puts the given entry into the local record, overwriting any existing value. // Use Set*IP and SetFallbackUDP to set IP addresses and UDP port, otherwise they'll // be overwritten by the endpoint predictor. +// +// Since node record updates are throttled to one per second, Set is asynchronous. +// Any update will be queued up and published when at least one second passes from +// the last change. func (ln *LocalNode) Set(e enr.Entry) { ln.mu.Lock() defer ln.mu.Unlock() @@ -288,3 +318,12 @@ func (ln *LocalNode) bumpSeq() { ln.seq++ ln.db.storeLocalSeq(ln.id, ln.seq) } + +// nowMilliseconds gives the current timestamp at millisecond precision. +func nowMilliseconds() uint64 { + ns := time.Now().UnixNano() + if ns < 0 { + return 0 + } + return uint64(ns / 1000 / 1000) +} diff --git a/p2p/enode/localnode_test.go b/p2p/enode/localnode_test.go index 00746a8d2..312df813b 100644 --- a/p2p/enode/localnode_test.go +++ b/p2p/enode/localnode_test.go @@ -49,32 +49,39 @@ func TestLocalNode(t *testing.T) { } } +// This test checks that the sequence number is persisted between restarts. func TestLocalNodeSeqPersist(t *testing.T) { + timestamp := nowMilliseconds() + ln, db := newLocalNodeForTesting() defer db.Close() - if s := ln.Node().Seq(); s != 1 { - t.Fatalf("wrong initial seq %d, want 1", s) + initialSeq := ln.Node().Seq() + if initialSeq < timestamp { + t.Fatalf("wrong initial seq %d, want at least %d", initialSeq, timestamp) } + ln.Set(enr.WithEntry("x", uint(1))) - if s := ln.Node().Seq(); s != 2 { - t.Fatalf("wrong seq %d after set, want 2", s) + if s := ln.Node().Seq(); s != initialSeq+1 { + t.Fatalf("wrong seq %d after set, want %d", s, initialSeq+1) } // Create a new instance, it should reload the sequence number. // The number increases just after that because a new record is // created without the "x" entry. ln2 := NewLocalNode(db, ln.key) - if s := ln2.Node().Seq(); s != 3 { - t.Fatalf("wrong seq %d on new instance, want 3", s) + if s := ln2.Node().Seq(); s != initialSeq+2 { + t.Fatalf("wrong seq %d on new instance, want %d", s, initialSeq+2) } + finalSeq := ln2.Node().Seq() + // Create a new instance with a different node key on the same database. // This should reset the sequence number. key, _ := crypto.GenerateKey() ln3 := NewLocalNode(db, key) - if s := ln3.Node().Seq(); s != 1 { - t.Fatalf("wrong seq %d on instance with changed key, want 1", s) + if s := ln3.Node().Seq(); s < finalSeq { + t.Fatalf("wrong seq %d on instance with changed key, want >= %d", s, finalSeq) } } @@ -91,20 +98,20 @@ func TestLocalNodeEndpoint(t *testing.T) { // Nothing is set initially. assert.Equal(t, net.IP(nil), ln.Node().IP()) assert.Equal(t, 0, ln.Node().UDP()) - assert.Equal(t, uint64(1), ln.Node().Seq()) + initialSeq := ln.Node().Seq() // Set up fallback address. ln.SetFallbackIP(fallback.IP) ln.SetFallbackUDP(fallback.Port) assert.Equal(t, fallback.IP, ln.Node().IP()) assert.Equal(t, fallback.Port, ln.Node().UDP()) - assert.Equal(t, uint64(2), ln.Node().Seq()) + assert.Equal(t, initialSeq+1, ln.Node().Seq()) // Add endpoint statements from random hosts. for i := 0; i < iptrackMinStatements; i++ { assert.Equal(t, fallback.IP, ln.Node().IP()) assert.Equal(t, fallback.Port, ln.Node().UDP()) - assert.Equal(t, uint64(2), ln.Node().Seq()) + assert.Equal(t, initialSeq+1, ln.Node().Seq()) from := &net.UDPAddr{IP: make(net.IP, 4), Port: 90} rand.Read(from.IP) @@ -112,11 +119,11 @@ func TestLocalNodeEndpoint(t *testing.T) { } assert.Equal(t, predicted.IP, ln.Node().IP()) assert.Equal(t, predicted.Port, ln.Node().UDP()) - assert.Equal(t, uint64(3), ln.Node().Seq()) + assert.Equal(t, initialSeq+2, ln.Node().Seq()) // Static IP overrides prediction. ln.SetStaticIP(staticIP) assert.Equal(t, staticIP, ln.Node().IP()) assert.Equal(t, fallback.Port, ln.Node().UDP()) - assert.Equal(t, uint64(4), ln.Node().Seq()) + assert.Equal(t, initialSeq+3, ln.Node().Seq()) } diff --git a/p2p/enode/nodedb.go b/p2p/enode/nodedb.go index d62f383f0..d1712f759 100644 --- a/p2p/enode/nodedb.go +++ b/p2p/enode/nodedb.go @@ -427,9 +427,14 @@ func (db *DB) UpdateFindFailsV5(id ID, ip net.IP, fails int) error { return db.storeInt64(v5Key(id, ip, dbNodeFindFails), int64(fails)) } -// LocalSeq retrieves the local record sequence counter. +// localSeq retrieves the local record sequence counter, defaulting to the current +// timestamp if no previous exists. This ensures that wiping all data associated +// with a node (apart from its key) will not generate already used sequence nums. func (db *DB) localSeq(id ID) uint64 { - return db.fetchUint64(localItemKey(id, dbLocalSeq)) + if seq := db.fetchUint64(localItemKey(id, dbLocalSeq)); seq > 0 { + return seq + } + return nowMilliseconds() } // storeLocalSeq stores the local record sequence counter. From 51ed39c0933d0d2fffbc5d093d45872a84d960eb Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 7 Sep 2021 16:15:11 +0200 Subject: [PATCH 43/73] core: make txPricedList.stales the first field for 64bit alignment (#23542) This fixes crashes on 32bit builds. --- core/tx_list.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/core/tx_list.go b/core/tx_list.go index 1d5c336a1..ea96f3ebb 100644 --- a/core/tx_list.go +++ b/core/tx_list.go @@ -480,9 +480,14 @@ func (h *priceHeap) Pop() interface{} { // better candidates for inclusion while in other cases (at the top of the baseFee peak) // the floating heap is better. When baseFee is decreasing they behave similarly. type txPricedList struct { + // Number of stale price points to (re-heap trigger). + // This field is accessed atomically, and must be the first field + // to ensure it has correct alignment for atomic.AddInt64. + // See https://golang.org/pkg/sync/atomic/#pkg-note-BUG. + stales int64 + all *txLookup // Pointer to the map of all transactions urgent, floating priceHeap // Heaps of prices of all the stored **remote** transactions - stales int64 // Number of stale price points to (re-heap trigger) reheapMu sync.Mutex // Mutex asserts that only one routine is reheaping the list } From 5c1fc3bf547f1e983a95c734bdf8ab42a2d848c4 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 7 Sep 2021 17:54:15 +0200 Subject: [PATCH 44/73] core: 64bit alignment of testBlockChain.gasLimit (#23543) This fixes crashes in txpool tests on 32bit platforms. --- core/tx_pool_test.go | 42 +++++++++++++++++++++--------------------- 1 file changed, 21 insertions(+), 21 deletions(-) diff --git a/core/tx_pool_test.go b/core/tx_pool_test.go index ce2fe7755..9b5208de3 100644 --- a/core/tx_pool_test.go +++ b/core/tx_pool_test.go @@ -58,8 +58,8 @@ func init() { } type testBlockChain struct { + gasLimit uint64 // must be first field for 64 bit alignment (atomic access) statedb *state.StateDB - gasLimit uint64 chainHeadFeed *event.Feed } @@ -119,7 +119,7 @@ func setupTxPool() (*TxPool, *ecdsa.PrivateKey) { func setupTxPoolWithConfig(config *params.ChainConfig) (*TxPool, *ecdsa.PrivateKey) { statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) - blockchain := &testBlockChain{statedb, 10000000, new(event.Feed)} + blockchain := &testBlockChain{10000000, statedb, new(event.Feed)} key, _ := crypto.GenerateKey() pool := NewTxPool(testTxPoolConfig, config, blockchain) @@ -231,7 +231,7 @@ func TestStateChangeDuringTransactionPoolReset(t *testing.T) { // setup pool with 2 transaction in it statedb.SetBalance(address, new(big.Int).SetUint64(params.Ether)) - blockchain := &testChain{&testBlockChain{statedb, 1000000000, new(event.Feed)}, address, &trigger} + blockchain := &testChain{&testBlockChain{1000000000, statedb, new(event.Feed)}, address, &trigger} tx0 := transaction(0, 100000, key) tx1 := transaction(1, 100000, key) @@ -429,7 +429,7 @@ func TestTransactionChainFork(t *testing.T) { statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) statedb.AddBalance(addr, big.NewInt(100000000000000)) - pool.chain = &testBlockChain{statedb, 1000000, new(event.Feed)} + pool.chain = &testBlockChain{1000000, statedb, new(event.Feed)} <-pool.requestReset(nil, nil) } resetState() @@ -458,7 +458,7 @@ func TestTransactionDoubleNonce(t *testing.T) { statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) statedb.AddBalance(addr, big.NewInt(100000000000000)) - pool.chain = &testBlockChain{statedb, 1000000, new(event.Feed)} + pool.chain = &testBlockChain{1000000, statedb, new(event.Feed)} <-pool.requestReset(nil, nil) } resetState() @@ -656,7 +656,7 @@ func TestTransactionPostponing(t *testing.T) { // Create the pool to test the postponing with statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) - blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} + blockchain := &testBlockChain{1000000, statedb, new(event.Feed)} pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain) defer pool.Stop() @@ -869,7 +869,7 @@ func testTransactionQueueGlobalLimiting(t *testing.T, nolocals bool) { // Create the pool to test the limit enforcement with statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) - blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} + blockchain := &testBlockChain{1000000, statedb, new(event.Feed)} config := testTxPoolConfig config.NoLocals = nolocals @@ -961,7 +961,7 @@ func testTransactionQueueTimeLimiting(t *testing.T, nolocals bool) { // Create the pool to test the non-expiration enforcement statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) - blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} + blockchain := &testBlockChain{1000000, statedb, new(event.Feed)} config := testTxPoolConfig config.Lifetime = time.Second @@ -1146,7 +1146,7 @@ func TestTransactionPendingGlobalLimiting(t *testing.T) { // Create the pool to test the limit enforcement with statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) - blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} + blockchain := &testBlockChain{1000000, statedb, new(event.Feed)} config := testTxPoolConfig config.GlobalSlots = config.AccountSlots * 10 @@ -1248,7 +1248,7 @@ func TestTransactionCapClearsFromAll(t *testing.T) { // Create the pool to test the limit enforcement with statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) - blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} + blockchain := &testBlockChain{1000000, statedb, new(event.Feed)} config := testTxPoolConfig config.AccountSlots = 2 @@ -1282,7 +1282,7 @@ func TestTransactionPendingMinimumAllowance(t *testing.T) { // Create the pool to test the limit enforcement with statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) - blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} + blockchain := &testBlockChain{1000000, statedb, new(event.Feed)} config := testTxPoolConfig config.GlobalSlots = 1 @@ -1330,7 +1330,7 @@ func TestTransactionPoolRepricing(t *testing.T) { // Create the pool to test the pricing enforcement with statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) - blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} + blockchain := &testBlockChain{1000000, statedb, new(event.Feed)} pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain) defer pool.Stop() @@ -1578,7 +1578,7 @@ func TestTransactionPoolRepricingKeepsLocals(t *testing.T) { // Create the pool to test the pricing enforcement with statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) - blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} + blockchain := &testBlockChain{1000000, statedb, new(event.Feed)} pool := NewTxPool(testTxPoolConfig, eip1559Config, blockchain) defer pool.Stop() @@ -1651,7 +1651,7 @@ func TestTransactionPoolUnderpricing(t *testing.T) { // Create the pool to test the pricing enforcement with statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) - blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} + blockchain := &testBlockChain{1000000, statedb, new(event.Feed)} config := testTxPoolConfig config.GlobalSlots = 2 @@ -1757,7 +1757,7 @@ func TestTransactionPoolStableUnderpricing(t *testing.T) { // Create the pool to test the pricing enforcement with statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) - blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} + blockchain := &testBlockChain{1000000, statedb, new(event.Feed)} config := testTxPoolConfig config.GlobalSlots = 128 @@ -1989,7 +1989,7 @@ func TestTransactionDeduplication(t *testing.T) { // Create the pool to test the pricing enforcement with statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) - blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} + blockchain := &testBlockChain{1000000, statedb, new(event.Feed)} pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain) defer pool.Stop() @@ -2055,7 +2055,7 @@ func TestTransactionReplacement(t *testing.T) { // Create the pool to test the pricing enforcement with statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) - blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} + blockchain := &testBlockChain{1000000, statedb, new(event.Feed)} pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain) defer pool.Stop() @@ -2260,7 +2260,7 @@ func testTransactionJournaling(t *testing.T, nolocals bool) { // Create the original pool to inject transaction into the journal statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) - blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} + blockchain := &testBlockChain{1000000, statedb, new(event.Feed)} config := testTxPoolConfig config.NoLocals = nolocals @@ -2302,7 +2302,7 @@ func testTransactionJournaling(t *testing.T, nolocals bool) { // Terminate the old pool, bump the local nonce, create a new pool and ensure relevant transaction survive pool.Stop() statedb.SetNonce(crypto.PubkeyToAddress(local.PublicKey), 1) - blockchain = &testBlockChain{statedb, 1000000, new(event.Feed)} + blockchain = &testBlockChain{1000000, statedb, new(event.Feed)} pool = NewTxPool(config, params.TestChainConfig, blockchain) @@ -2329,7 +2329,7 @@ func testTransactionJournaling(t *testing.T, nolocals bool) { pool.Stop() statedb.SetNonce(crypto.PubkeyToAddress(local.PublicKey), 1) - blockchain = &testBlockChain{statedb, 1000000, new(event.Feed)} + blockchain = &testBlockChain{1000000, statedb, new(event.Feed)} pool = NewTxPool(config, params.TestChainConfig, blockchain) pending, queued = pool.Stats() @@ -2358,7 +2358,7 @@ func TestTransactionStatusCheck(t *testing.T) { // Create the pool to test the status retrievals with statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) - blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} + blockchain := &testBlockChain{1000000, statedb, new(event.Feed)} pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain) defer pool.Stop() From 90987db7334c1d10eb866ca550efedb66dea8a20 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 8 Sep 2021 14:21:59 +0200 Subject: [PATCH 45/73] appveyor.yml: add linux/amd64 build job (#23548) --- .golangci.yml | 2 +- appveyor.yml | 60 +++++++++++++++++++++++++++++++++++++-------------- 2 files changed, 45 insertions(+), 17 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 18b325e20..395a91fe1 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -1,7 +1,7 @@ # This file configures github.com/golangci/golangci-lint. run: - timeout: 3m + timeout: 5m tests: true # default is true. Enables skipping of directories: # vendor$, third_party$, testdata$, examples$, Godeps$, builtin$ diff --git a/appveyor.yml b/appveyor.yml index a72163382..65b5f9684 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -1,29 +1,57 @@ -os: Visual Studio 2019 clone_depth: 5 version: "{branch}.{build}" + +image: + - Ubuntu + - Visual Studio 2019 + environment: matrix: - # We use gcc from MSYS2 because it is the most recent compiler version available on - # AppVeyor. Note: gcc.exe only works properly if the corresponding bin/ directory is - # contained in PATH. - GETH_ARCH: amd64 - GETH_CC: C:\msys64\mingw64\bin\gcc.exe - PATH: C:\msys64\mingw64\bin;C:\Program Files (x86)\NSIS\;%PATH% + GETH_MINGW: 'C:\msys64\mingw64' - GETH_ARCH: 386 - GETH_CC: C:\msys64\mingw32\bin\gcc.exe - PATH: C:\msys64\mingw32\bin;C:\Program Files (x86)\NSIS\;%PATH% + GETH_MINGW: 'C:\msys64\mingw32' install: - git submodule update --init --depth 1 - go version - - "%GETH_CC% --version" -build_script: - - go run build\ci.go install -dlgo -arch %GETH_ARCH% -cc %GETH_CC% +for: + # Linux has its own script without -arch and -cc. + # The linux builder also runs lint. + - matrix: + only: + - image: Ubuntu + build_script: + - go run build/ci.go lint + - go run build/ci.go install -dlgo + test_script: + - go run build/ci.go test -dlgo -coverage -after_build: - - go run build\ci.go archive -arch %GETH_ARCH% -type zip -signer WINDOWS_SIGNING_KEY -upload gethstore/builds - - go run build\ci.go nsis -arch %GETH_ARCH% -signer WINDOWS_SIGNING_KEY -upload gethstore/builds + # linux/386 is disabled. + - matrix: + exclude: + - image: Ubuntu + GETH_ARCH: 386 -test_script: - - go run build\ci.go test -dlgo -arch %GETH_ARCH% -cc %GETH_CC% -coverage + # Windows builds for amd64 + 386. + - matrix: + only: + - image: Visual Studio 2019 + environment: + # We use gcc from MSYS2 because it is the most recent compiler version available on + # AppVeyor. Note: gcc.exe only works properly if the corresponding bin/ directory is + # contained in PATH. + GETH_CC: '%GETH_MINGW%\bin\gcc.exe' + PATH: '%GETH_MINGW%\bin;C:\Program Files (x86)\NSIS\;%PATH%' + build_script: + - 'echo %GETH_ARCH%' + - 'echo %GETH_CC%' + - '%GETH_CC% --version' + - go run build/ci.go install -dlgo -arch %GETH_ARCH% -cc %GETH_CC% + after_build: + # Upload builds. Note that ci.go makes this a no-op PR builds. + - go run build/ci.go archive -arch %GETH_ARCH% -type zip -signer WINDOWS_SIGNING_KEY -upload gethstore/builds + - go run build/ci.go nsis -arch %GETH_ARCH% -signer WINDOWS_SIGNING_KEY -upload gethstore/builds + test_script: + - go run build/ci.go test -dlgo -arch %GETH_ARCH% -cc %GETH_CC% -coverage From 9e17648d8c991c7243ca99123f010c152b87fce7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Fri, 10 Sep 2021 10:55:48 +0300 Subject: [PATCH 46/73] les: duplicate downloader and fetcher to allow progressive refactoring --- eth/api_backend.go | 6 +- ethstats/ethstats.go | 8 +- graphql/graphql.go | 2 +- internal/ethapi/api.go | 2 +- internal/ethapi/backend.go | 5 +- les/api_backend.go | 6 +- les/api_test.go | 7 +- les/client.go | 2 +- les/client_handler.go | 2 +- les/downloader/api.go | 166 +++ les/downloader/downloader.go | 2014 +++++++++++++++++++++++++++++ les/downloader/downloader_test.go | 1622 +++++++++++++++++++++++ les/downloader/events.go | 25 + les/downloader/metrics.go | 45 + les/downloader/modes.go | 81 ++ les/downloader/peer.go | 501 +++++++ les/downloader/queue.go | 913 +++++++++++++ les/downloader/queue_test.go | 452 +++++++ les/downloader/resultstore.go | 194 +++ les/downloader/statesync.go | 615 +++++++++ les/downloader/testchain_test.go | 230 ++++ les/downloader/types.go | 79 ++ les/fetcher.go | 2 +- les/fetcher/block_fetcher.go | 889 +++++++++++++ les/fetcher/block_fetcher_test.go | 896 +++++++++++++ les/handler_test.go | 2 +- les/sync.go | 2 +- 27 files changed, 8746 insertions(+), 22 deletions(-) create mode 100644 les/downloader/api.go create mode 100644 les/downloader/downloader.go create mode 100644 les/downloader/downloader_test.go create mode 100644 les/downloader/events.go create mode 100644 les/downloader/metrics.go create mode 100644 les/downloader/modes.go create mode 100644 les/downloader/peer.go create mode 100644 les/downloader/queue.go create mode 100644 les/downloader/queue_test.go create mode 100644 les/downloader/resultstore.go create mode 100644 les/downloader/statesync.go create mode 100644 les/downloader/testchain_test.go create mode 100644 les/downloader/types.go create mode 100644 les/fetcher/block_fetcher.go create mode 100644 les/fetcher/block_fetcher_test.go diff --git a/eth/api_backend.go b/eth/api_backend.go index 49de70e21..7b40a7edd 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -21,6 +21,7 @@ import ( "errors" "math/big" + "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus" @@ -30,7 +31,6 @@ import ( "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" - "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/gasprice" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" @@ -279,8 +279,8 @@ func (b *EthAPIBackend) SubscribeNewTxsEvent(ch chan<- core.NewTxsEvent) event.S return b.eth.TxPool().SubscribeNewTxsEvent(ch) } -func (b *EthAPIBackend) Downloader() *downloader.Downloader { - return b.eth.Downloader() +func (b *EthAPIBackend) SyncProgress() ethereum.SyncProgress { + return b.eth.Downloader().Progress() } func (b *EthAPIBackend) SuggestGasTipCap(ctx context.Context) (*big.Int, error) { diff --git a/ethstats/ethstats.go b/ethstats/ethstats.go index 148359110..55c0c880f 100644 --- a/ethstats/ethstats.go +++ b/ethstats/ethstats.go @@ -30,12 +30,12 @@ import ( "sync" "time" + "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/mclock" "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/eth/downloader" ethproto "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/les" @@ -67,7 +67,7 @@ type backend interface { HeaderByNumber(ctx context.Context, number rpc.BlockNumber) (*types.Header, error) GetTd(ctx context.Context, hash common.Hash) *big.Int Stats() (pending int, queued int) - Downloader() *downloader.Downloader + SyncProgress() ethereum.SyncProgress } // fullNodeBackend encompasses the functionality necessary for a full node @@ -777,7 +777,7 @@ func (s *Service) reportStats(conn *connWrapper) error { mining = fullBackend.Miner().Mining() hashrate = int(fullBackend.Miner().Hashrate()) - sync := fullBackend.Downloader().Progress() + sync := fullBackend.SyncProgress() syncing = fullBackend.CurrentHeader().Number.Uint64() >= sync.HighestBlock price, _ := fullBackend.SuggestGasTipCap(context.Background()) @@ -786,7 +786,7 @@ func (s *Service) reportStats(conn *connWrapper) error { gasprice += int(basefee.Uint64()) } } else { - sync := s.backend.Downloader().Progress() + sync := s.backend.SyncProgress() syncing = s.backend.CurrentHeader().Number.Uint64() >= sync.HighestBlock } // Assemble the node stats and send it to the server diff --git a/graphql/graphql.go b/graphql/graphql.go index d35994234..4dd96c4b9 100644 --- a/graphql/graphql.go +++ b/graphql/graphql.go @@ -1248,7 +1248,7 @@ func (s *SyncState) KnownStates() *hexutil.Uint64 { // - pulledStates: number of state entries processed until now // - knownStates: number of known state entries that still need to be pulled func (r *Resolver) Syncing() (*SyncState, error) { - progress := r.backend.Downloader().Progress() + progress := r.backend.SyncProgress() // Return not syncing if the synchronisation already completed if progress.CurrentBlock >= progress.HighestBlock { diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 9a82824ad..6997f2c82 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -122,7 +122,7 @@ func (s *PublicEthereumAPI) FeeHistory(ctx context.Context, blockCount rpc.Decim // - pulledStates: number of state entries processed until now // - knownStates: number of known state entries that still need to be pulled func (s *PublicEthereumAPI) Syncing() (interface{}, error) { - progress := s.b.Downloader().Progress() + progress := s.b.SyncProgress() // Return not syncing if the synchronisation already completed if progress.CurrentBlock >= progress.HighestBlock { diff --git a/internal/ethapi/backend.go b/internal/ethapi/backend.go index 995454582..1624f4963 100644 --- a/internal/ethapi/backend.go +++ b/internal/ethapi/backend.go @@ -21,6 +21,7 @@ import ( "context" "math/big" + "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus" @@ -29,7 +30,6 @@ import ( "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" - "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/params" @@ -40,7 +40,8 @@ import ( // both full and light clients) with access to necessary functions. type Backend interface { // General Ethereum API - Downloader() *downloader.Downloader + SyncProgress() ethereum.SyncProgress + SuggestGasTipCap(ctx context.Context) (*big.Int, error) FeeHistory(ctx context.Context, blockCount int, lastBlock rpc.BlockNumber, rewardPercentiles []float64) (*big.Int, [][]*big.Int, []*big.Int, []float64, error) ChainDb() ethdb.Database diff --git a/les/api_backend.go b/les/api_backend.go index 9c80270da..e12984cb4 100644 --- a/les/api_backend.go +++ b/les/api_backend.go @@ -21,6 +21,7 @@ import ( "errors" "math/big" + "github.com/ethereum/go-ethereum" "github.com/ethereum/go-ethereum/accounts" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus" @@ -30,7 +31,6 @@ import ( "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" - "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/gasprice" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" @@ -257,8 +257,8 @@ func (b *LesApiBackend) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEven return b.eth.blockchain.SubscribeRemovedLogsEvent(ch) } -func (b *LesApiBackend) Downloader() *downloader.Downloader { - return b.eth.Downloader() +func (b *LesApiBackend) SyncProgress() ethereum.SyncProgress { + return b.eth.Downloader().Progress() } func (b *LesApiBackend) ProtocolVersion() int { diff --git a/les/api_test.go b/les/api_test.go index f7017c5d9..6a19b0fe4 100644 --- a/les/api_test.go +++ b/les/api_test.go @@ -32,8 +32,9 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/eth" - "github.com/ethereum/go-ethereum/eth/downloader" + ethdownloader "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/ethconfig" + "github.com/ethereum/go-ethereum/les/downloader" "github.com/ethereum/go-ethereum/les/flowcontrol" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" @@ -494,14 +495,14 @@ func testSim(t *testing.T, serverCount, clientCount int, serverDir, clientDir [] func newLesClientService(ctx *adapters.ServiceContext, stack *node.Node) (node.Lifecycle, error) { config := ethconfig.Defaults - config.SyncMode = downloader.LightSync + config.SyncMode = (ethdownloader.SyncMode)(downloader.LightSync) config.Ethash.PowMode = ethash.ModeFake return New(stack, &config) } func newLesServerService(ctx *adapters.ServiceContext, stack *node.Node) (node.Lifecycle, error) { config := ethconfig.Defaults - config.SyncMode = downloader.FullSync + config.SyncMode = (ethdownloader.SyncMode)(downloader.FullSync) config.LightServ = testServerCapacity config.LightPeers = testMaxClients ethereum, err := eth.New(stack, &config) diff --git a/les/client.go b/les/client.go index 1d8a2c6f9..5d07c783e 100644 --- a/les/client.go +++ b/les/client.go @@ -30,12 +30,12 @@ import ( "github.com/ethereum/go-ethereum/core/bloombits" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/ethconfig" "github.com/ethereum/go-ethereum/eth/filters" "github.com/ethereum/go-ethereum/eth/gasprice" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/internal/ethapi" + "github.com/ethereum/go-ethereum/les/downloader" "github.com/ethereum/go-ethereum/les/vflux" vfc "github.com/ethereum/go-ethereum/les/vflux/client" "github.com/ethereum/go-ethereum/light" diff --git a/les/client_handler.go b/les/client_handler.go index 4a550b207..9583bd57c 100644 --- a/les/client_handler.go +++ b/les/client_handler.go @@ -28,8 +28,8 @@ import ( "github.com/ethereum/go-ethereum/common/mclock" "github.com/ethereum/go-ethereum/core/forkid" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/protocols/eth" + "github.com/ethereum/go-ethereum/les/downloader" "github.com/ethereum/go-ethereum/light" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p" diff --git a/les/downloader/api.go b/les/downloader/api.go new file mode 100644 index 000000000..2024d23de --- /dev/null +++ b/les/downloader/api.go @@ -0,0 +1,166 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package downloader + +import ( + "context" + "sync" + + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/rpc" +) + +// PublicDownloaderAPI provides an API which gives information about the current synchronisation status. +// It offers only methods that operates on data that can be available to anyone without security risks. +type PublicDownloaderAPI struct { + d *Downloader + mux *event.TypeMux + installSyncSubscription chan chan interface{} + uninstallSyncSubscription chan *uninstallSyncSubscriptionRequest +} + +// NewPublicDownloaderAPI create a new PublicDownloaderAPI. The API has an internal event loop that +// listens for events from the downloader through the global event mux. In case it receives one of +// these events it broadcasts it to all syncing subscriptions that are installed through the +// installSyncSubscription channel. +func NewPublicDownloaderAPI(d *Downloader, m *event.TypeMux) *PublicDownloaderAPI { + api := &PublicDownloaderAPI{ + d: d, + mux: m, + installSyncSubscription: make(chan chan interface{}), + uninstallSyncSubscription: make(chan *uninstallSyncSubscriptionRequest), + } + + go api.eventLoop() + + return api +} + +// eventLoop runs a loop until the event mux closes. It will install and uninstall new +// sync subscriptions and broadcasts sync status updates to the installed sync subscriptions. +func (api *PublicDownloaderAPI) eventLoop() { + var ( + sub = api.mux.Subscribe(StartEvent{}, DoneEvent{}, FailedEvent{}) + syncSubscriptions = make(map[chan interface{}]struct{}) + ) + + for { + select { + case i := <-api.installSyncSubscription: + syncSubscriptions[i] = struct{}{} + case u := <-api.uninstallSyncSubscription: + delete(syncSubscriptions, u.c) + close(u.uninstalled) + case event := <-sub.Chan(): + if event == nil { + return + } + + var notification interface{} + switch event.Data.(type) { + case StartEvent: + notification = &SyncingResult{ + Syncing: true, + Status: api.d.Progress(), + } + case DoneEvent, FailedEvent: + notification = false + } + // broadcast + for c := range syncSubscriptions { + c <- notification + } + } + } +} + +// Syncing provides information when this nodes starts synchronising with the Ethereum network and when it's finished. +func (api *PublicDownloaderAPI) Syncing(ctx context.Context) (*rpc.Subscription, error) { + notifier, supported := rpc.NotifierFromContext(ctx) + if !supported { + return &rpc.Subscription{}, rpc.ErrNotificationsUnsupported + } + + rpcSub := notifier.CreateSubscription() + + go func() { + statuses := make(chan interface{}) + sub := api.SubscribeSyncStatus(statuses) + + for { + select { + case status := <-statuses: + notifier.Notify(rpcSub.ID, status) + case <-rpcSub.Err(): + sub.Unsubscribe() + return + case <-notifier.Closed(): + sub.Unsubscribe() + return + } + } + }() + + return rpcSub, nil +} + +// SyncingResult provides information about the current synchronisation status for this node. +type SyncingResult struct { + Syncing bool `json:"syncing"` + Status ethereum.SyncProgress `json:"status"` +} + +// uninstallSyncSubscriptionRequest uninstalles a syncing subscription in the API event loop. +type uninstallSyncSubscriptionRequest struct { + c chan interface{} + uninstalled chan interface{} +} + +// SyncStatusSubscription represents a syncing subscription. +type SyncStatusSubscription struct { + api *PublicDownloaderAPI // register subscription in event loop of this api instance + c chan interface{} // channel where events are broadcasted to + unsubOnce sync.Once // make sure unsubscribe logic is executed once +} + +// Unsubscribe uninstalls the subscription from the DownloadAPI event loop. +// The status channel that was passed to subscribeSyncStatus isn't used anymore +// after this method returns. +func (s *SyncStatusSubscription) Unsubscribe() { + s.unsubOnce.Do(func() { + req := uninstallSyncSubscriptionRequest{s.c, make(chan interface{})} + s.api.uninstallSyncSubscription <- &req + + for { + select { + case <-s.c: + // drop new status events until uninstall confirmation + continue + case <-req.uninstalled: + return + } + } + }) +} + +// SubscribeSyncStatus creates a subscription that will broadcast new synchronisation updates. +// The given channel must receive interface values, the result can either +func (api *PublicDownloaderAPI) SubscribeSyncStatus(status chan interface{}) *SyncStatusSubscription { + api.installSyncSubscription <- status + return &SyncStatusSubscription{api: api, c: status} +} diff --git a/les/downloader/downloader.go b/les/downloader/downloader.go new file mode 100644 index 000000000..e7dfc4158 --- /dev/null +++ b/les/downloader/downloader.go @@ -0,0 +1,2014 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// This is a temporary package whilst working on the eth/66 blocking refactors. +// After that work is done, les needs to be refactored to use the new package, +// or alternatively use a stripped down version of it. Either way, we need to +// keep the changes scoped so duplicating temporarily seems the sanest. +package downloader + +import ( + "errors" + "fmt" + "math/big" + "sync" + "sync/atomic" + "time" + + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/state/snapshot" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/eth/protocols/eth" + "github.com/ethereum/go-ethereum/eth/protocols/snap" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/trie" +) + +var ( + MaxBlockFetch = 128 // Amount of blocks to be fetched per retrieval request + MaxHeaderFetch = 192 // Amount of block headers to be fetched per retrieval request + MaxSkeletonSize = 128 // Number of header fetches to need for a skeleton assembly + MaxReceiptFetch = 256 // Amount of transaction receipts to allow fetching per request + MaxStateFetch = 384 // Amount of node state values to allow fetching per request + + maxQueuedHeaders = 32 * 1024 // [eth/62] Maximum number of headers to queue for import (DOS protection) + maxHeadersProcess = 2048 // Number of header download results to import at once into the chain + maxResultsProcess = 2048 // Number of content download results to import at once into the chain + fullMaxForkAncestry uint64 = params.FullImmutabilityThreshold // Maximum chain reorganisation (locally redeclared so tests can reduce it) + lightMaxForkAncestry uint64 = params.LightImmutabilityThreshold // Maximum chain reorganisation (locally redeclared so tests can reduce it) + + reorgProtThreshold = 48 // Threshold number of recent blocks to disable mini reorg protection + reorgProtHeaderDelay = 2 // Number of headers to delay delivering to cover mini reorgs + + fsHeaderCheckFrequency = 100 // Verification frequency of the downloaded headers during fast sync + fsHeaderSafetyNet = 2048 // Number of headers to discard in case a chain violation is detected + fsHeaderForceVerify = 24 // Number of headers to verify before and after the pivot to accept it + fsHeaderContCheck = 3 * time.Second // Time interval to check for header continuations during state download + fsMinFullBlocks = 64 // Number of blocks to retrieve fully even in fast sync +) + +var ( + errBusy = errors.New("busy") + errUnknownPeer = errors.New("peer is unknown or unhealthy") + errBadPeer = errors.New("action from bad peer ignored") + errStallingPeer = errors.New("peer is stalling") + errUnsyncedPeer = errors.New("unsynced peer") + errNoPeers = errors.New("no peers to keep download active") + errTimeout = errors.New("timeout") + errEmptyHeaderSet = errors.New("empty header set by peer") + errPeersUnavailable = errors.New("no peers available or all tried for download") + errInvalidAncestor = errors.New("retrieved ancestor is invalid") + errInvalidChain = errors.New("retrieved hash chain is invalid") + errInvalidBody = errors.New("retrieved block body is invalid") + errInvalidReceipt = errors.New("retrieved receipt is invalid") + errCancelStateFetch = errors.New("state data download canceled (requested)") + errCancelContentProcessing = errors.New("content processing canceled (requested)") + errCanceled = errors.New("syncing canceled (requested)") + errNoSyncActive = errors.New("no sync active") + errTooOld = errors.New("peer's protocol version too old") + errNoAncestorFound = errors.New("no common ancestor found") +) + +type Downloader struct { + mode uint32 // Synchronisation mode defining the strategy used (per sync cycle), use d.getMode() to get the SyncMode + mux *event.TypeMux // Event multiplexer to announce sync operation events + + checkpoint uint64 // Checkpoint block number to enforce head against (e.g. fast sync) + genesis uint64 // Genesis block number to limit sync to (e.g. light client CHT) + queue *queue // Scheduler for selecting the hashes to download + peers *peerSet // Set of active peers from which download can proceed + + stateDB ethdb.Database // Database to state sync into (and deduplicate via) + stateBloom *trie.SyncBloom // Bloom filter for fast trie node and contract code existence checks + + // Statistics + syncStatsChainOrigin uint64 // Origin block number where syncing started at + syncStatsChainHeight uint64 // Highest block number known when syncing started + syncStatsState stateSyncStats + syncStatsLock sync.RWMutex // Lock protecting the sync stats fields + + lightchain LightChain + blockchain BlockChain + + // Callbacks + dropPeer peerDropFn // Drops a peer for misbehaving + + // Status + synchroniseMock func(id string, hash common.Hash) error // Replacement for synchronise during testing + synchronising int32 + notified int32 + committed int32 + ancientLimit uint64 // The maximum block number which can be regarded as ancient data. + + // Channels + headerCh chan dataPack // Channel receiving inbound block headers + bodyCh chan dataPack // Channel receiving inbound block bodies + receiptCh chan dataPack // Channel receiving inbound receipts + bodyWakeCh chan bool // Channel to signal the block body fetcher of new tasks + receiptWakeCh chan bool // Channel to signal the receipt fetcher of new tasks + headerProcCh chan []*types.Header // Channel to feed the header processor new tasks + + // State sync + pivotHeader *types.Header // Pivot block header to dynamically push the syncing state root + pivotLock sync.RWMutex // Lock protecting pivot header reads from updates + + snapSync bool // Whether to run state sync over the snap protocol + SnapSyncer *snap.Syncer // TODO(karalabe): make private! hack for now + stateSyncStart chan *stateSync + trackStateReq chan *stateReq + stateCh chan dataPack // Channel receiving inbound node state data + + // Cancellation and termination + cancelPeer string // Identifier of the peer currently being used as the master (cancel on drop) + cancelCh chan struct{} // Channel to cancel mid-flight syncs + cancelLock sync.RWMutex // Lock to protect the cancel channel and peer in delivers + cancelWg sync.WaitGroup // Make sure all fetcher goroutines have exited. + + quitCh chan struct{} // Quit channel to signal termination + quitLock sync.Mutex // Lock to prevent double closes + + // Testing hooks + syncInitHook func(uint64, uint64) // Method to call upon initiating a new sync run + bodyFetchHook func([]*types.Header) // Method to call upon starting a block body fetch + receiptFetchHook func([]*types.Header) // Method to call upon starting a receipt fetch + chainInsertHook func([]*fetchResult) // Method to call upon inserting a chain of blocks (possibly in multiple invocations) +} + +// LightChain encapsulates functions required to synchronise a light chain. +type LightChain interface { + // HasHeader verifies a header's presence in the local chain. + HasHeader(common.Hash, uint64) bool + + // GetHeaderByHash retrieves a header from the local chain. + GetHeaderByHash(common.Hash) *types.Header + + // CurrentHeader retrieves the head header from the local chain. + CurrentHeader() *types.Header + + // GetTd returns the total difficulty of a local block. + GetTd(common.Hash, uint64) *big.Int + + // InsertHeaderChain inserts a batch of headers into the local chain. + InsertHeaderChain([]*types.Header, int) (int, error) + + // SetHead rewinds the local chain to a new head. + SetHead(uint64) error +} + +// BlockChain encapsulates functions required to sync a (full or fast) blockchain. +type BlockChain interface { + LightChain + + // HasBlock verifies a block's presence in the local chain. + HasBlock(common.Hash, uint64) bool + + // HasFastBlock verifies a fast block's presence in the local chain. + HasFastBlock(common.Hash, uint64) bool + + // GetBlockByHash retrieves a block from the local chain. + GetBlockByHash(common.Hash) *types.Block + + // CurrentBlock retrieves the head block from the local chain. + CurrentBlock() *types.Block + + // CurrentFastBlock retrieves the head fast block from the local chain. + CurrentFastBlock() *types.Block + + // FastSyncCommitHead directly commits the head block to a certain entity. + FastSyncCommitHead(common.Hash) error + + // InsertChain inserts a batch of blocks into the local chain. + InsertChain(types.Blocks) (int, error) + + // InsertReceiptChain inserts a batch of receipts into the local chain. + InsertReceiptChain(types.Blocks, []types.Receipts, uint64) (int, error) + + // Snapshots returns the blockchain snapshot tree to paused it during sync. + Snapshots() *snapshot.Tree +} + +// New creates a new downloader to fetch hashes and blocks from remote peers. +func New(checkpoint uint64, stateDb ethdb.Database, stateBloom *trie.SyncBloom, mux *event.TypeMux, chain BlockChain, lightchain LightChain, dropPeer peerDropFn) *Downloader { + if lightchain == nil { + lightchain = chain + } + dl := &Downloader{ + stateDB: stateDb, + stateBloom: stateBloom, + mux: mux, + checkpoint: checkpoint, + queue: newQueue(blockCacheMaxItems, blockCacheInitialItems), + peers: newPeerSet(), + blockchain: chain, + lightchain: lightchain, + dropPeer: dropPeer, + headerCh: make(chan dataPack, 1), + bodyCh: make(chan dataPack, 1), + receiptCh: make(chan dataPack, 1), + bodyWakeCh: make(chan bool, 1), + receiptWakeCh: make(chan bool, 1), + headerProcCh: make(chan []*types.Header, 1), + quitCh: make(chan struct{}), + stateCh: make(chan dataPack), + SnapSyncer: snap.NewSyncer(stateDb), + stateSyncStart: make(chan *stateSync), + syncStatsState: stateSyncStats{ + processed: rawdb.ReadFastTrieProgress(stateDb), + }, + trackStateReq: make(chan *stateReq), + } + go dl.stateFetcher() + return dl +} + +// Progress retrieves the synchronisation boundaries, specifically the origin +// block where synchronisation started at (may have failed/suspended); the block +// or header sync is currently at; and the latest known block which the sync targets. +// +// In addition, during the state download phase of fast synchronisation the number +// of processed and the total number of known states are also returned. Otherwise +// these are zero. +func (d *Downloader) Progress() ethereum.SyncProgress { + // Lock the current stats and return the progress + d.syncStatsLock.RLock() + defer d.syncStatsLock.RUnlock() + + current := uint64(0) + mode := d.getMode() + switch { + case d.blockchain != nil && mode == FullSync: + current = d.blockchain.CurrentBlock().NumberU64() + case d.blockchain != nil && mode == FastSync: + current = d.blockchain.CurrentFastBlock().NumberU64() + case d.lightchain != nil: + current = d.lightchain.CurrentHeader().Number.Uint64() + default: + log.Error("Unknown downloader chain/mode combo", "light", d.lightchain != nil, "full", d.blockchain != nil, "mode", mode) + } + return ethereum.SyncProgress{ + StartingBlock: d.syncStatsChainOrigin, + CurrentBlock: current, + HighestBlock: d.syncStatsChainHeight, + PulledStates: d.syncStatsState.processed, + KnownStates: d.syncStatsState.processed + d.syncStatsState.pending, + } +} + +// Synchronising returns whether the downloader is currently retrieving blocks. +func (d *Downloader) Synchronising() bool { + return atomic.LoadInt32(&d.synchronising) > 0 +} + +// RegisterPeer injects a new download peer into the set of block source to be +// used for fetching hashes and blocks from. +func (d *Downloader) RegisterPeer(id string, version uint, peer Peer) error { + var logger log.Logger + if len(id) < 16 { + // Tests use short IDs, don't choke on them + logger = log.New("peer", id) + } else { + logger = log.New("peer", id[:8]) + } + logger.Trace("Registering sync peer") + if err := d.peers.Register(newPeerConnection(id, version, peer, logger)); err != nil { + logger.Error("Failed to register sync peer", "err", err) + return err + } + return nil +} + +// RegisterLightPeer injects a light client peer, wrapping it so it appears as a regular peer. +func (d *Downloader) RegisterLightPeer(id string, version uint, peer LightPeer) error { + return d.RegisterPeer(id, version, &lightPeerWrapper{peer}) +} + +// UnregisterPeer remove a peer from the known list, preventing any action from +// the specified peer. An effort is also made to return any pending fetches into +// the queue. +func (d *Downloader) UnregisterPeer(id string) error { + // Unregister the peer from the active peer set and revoke any fetch tasks + var logger log.Logger + if len(id) < 16 { + // Tests use short IDs, don't choke on them + logger = log.New("peer", id) + } else { + logger = log.New("peer", id[:8]) + } + logger.Trace("Unregistering sync peer") + if err := d.peers.Unregister(id); err != nil { + logger.Error("Failed to unregister sync peer", "err", err) + return err + } + d.queue.Revoke(id) + + return nil +} + +// Synchronise tries to sync up our local block chain with a remote peer, both +// adding various sanity checks as well as wrapping it with various log entries. +func (d *Downloader) Synchronise(id string, head common.Hash, td *big.Int, mode SyncMode) error { + err := d.synchronise(id, head, td, mode) + + switch err { + case nil, errBusy, errCanceled: + return err + } + if errors.Is(err, errInvalidChain) || errors.Is(err, errBadPeer) || errors.Is(err, errTimeout) || + errors.Is(err, errStallingPeer) || errors.Is(err, errUnsyncedPeer) || errors.Is(err, errEmptyHeaderSet) || + errors.Is(err, errPeersUnavailable) || errors.Is(err, errTooOld) || errors.Is(err, errInvalidAncestor) { + log.Warn("Synchronisation failed, dropping peer", "peer", id, "err", err) + if d.dropPeer == nil { + // The dropPeer method is nil when `--copydb` is used for a local copy. + // Timeouts can occur if e.g. compaction hits at the wrong time, and can be ignored + log.Warn("Downloader wants to drop peer, but peerdrop-function is not set", "peer", id) + } else { + d.dropPeer(id) + } + return err + } + log.Warn("Synchronisation failed, retrying", "err", err) + return err +} + +// synchronise will select the peer and use it for synchronising. If an empty string is given +// it will use the best peer possible and synchronize if its TD is higher than our own. If any of the +// checks fail an error will be returned. This method is synchronous +func (d *Downloader) synchronise(id string, hash common.Hash, td *big.Int, mode SyncMode) error { + // Mock out the synchronisation if testing + if d.synchroniseMock != nil { + return d.synchroniseMock(id, hash) + } + // Make sure only one goroutine is ever allowed past this point at once + if !atomic.CompareAndSwapInt32(&d.synchronising, 0, 1) { + return errBusy + } + defer atomic.StoreInt32(&d.synchronising, 0) + + // Post a user notification of the sync (only once per session) + if atomic.CompareAndSwapInt32(&d.notified, 0, 1) { + log.Info("Block synchronisation started") + } + // If we are already full syncing, but have a fast-sync bloom filter laying + // around, make sure it doesn't use memory any more. This is a special case + // when the user attempts to fast sync a new empty network. + if mode == FullSync && d.stateBloom != nil { + d.stateBloom.Close() + } + // If snap sync was requested, create the snap scheduler and switch to fast + // sync mode. Long term we could drop fast sync or merge the two together, + // but until snap becomes prevalent, we should support both. TODO(karalabe). + if mode == SnapSync { + if !d.snapSync { + // Snap sync uses the snapshot namespace to store potentially flakey data until + // sync completely heals and finishes. Pause snapshot maintenance in the mean + // time to prevent access. + if snapshots := d.blockchain.Snapshots(); snapshots != nil { // Only nil in tests + snapshots.Disable() + } + log.Warn("Enabling snapshot sync prototype") + d.snapSync = true + } + mode = FastSync + } + // Reset the queue, peer set and wake channels to clean any internal leftover state + d.queue.Reset(blockCacheMaxItems, blockCacheInitialItems) + d.peers.Reset() + + for _, ch := range []chan bool{d.bodyWakeCh, d.receiptWakeCh} { + select { + case <-ch: + default: + } + } + for _, ch := range []chan dataPack{d.headerCh, d.bodyCh, d.receiptCh} { + for empty := false; !empty; { + select { + case <-ch: + default: + empty = true + } + } + } + for empty := false; !empty; { + select { + case <-d.headerProcCh: + default: + empty = true + } + } + // Create cancel channel for aborting mid-flight and mark the master peer + d.cancelLock.Lock() + d.cancelCh = make(chan struct{}) + d.cancelPeer = id + d.cancelLock.Unlock() + + defer d.Cancel() // No matter what, we can't leave the cancel channel open + + // Atomically set the requested sync mode + atomic.StoreUint32(&d.mode, uint32(mode)) + + // Retrieve the origin peer and initiate the downloading process + p := d.peers.Peer(id) + if p == nil { + return errUnknownPeer + } + return d.syncWithPeer(p, hash, td) +} + +func (d *Downloader) getMode() SyncMode { + return SyncMode(atomic.LoadUint32(&d.mode)) +} + +// syncWithPeer starts a block synchronization based on the hash chain from the +// specified peer and head hash. +func (d *Downloader) syncWithPeer(p *peerConnection, hash common.Hash, td *big.Int) (err error) { + d.mux.Post(StartEvent{}) + defer func() { + // reset on error + if err != nil { + d.mux.Post(FailedEvent{err}) + } else { + latest := d.lightchain.CurrentHeader() + d.mux.Post(DoneEvent{latest}) + } + }() + if p.version < eth.ETH66 { + return fmt.Errorf("%w: advertized %d < required %d", errTooOld, p.version, eth.ETH66) + } + mode := d.getMode() + + log.Debug("Synchronising with the network", "peer", p.id, "eth", p.version, "head", hash, "td", td, "mode", mode) + defer func(start time.Time) { + log.Debug("Synchronisation terminated", "elapsed", common.PrettyDuration(time.Since(start))) + }(time.Now()) + + // Look up the sync boundaries: the common ancestor and the target block + latest, pivot, err := d.fetchHead(p) + if err != nil { + return err + } + if mode == FastSync && pivot == nil { + // If no pivot block was returned, the head is below the min full block + // threshold (i.e. new chain). In that case we won't really fast sync + // anyway, but still need a valid pivot block to avoid some code hitting + // nil panics on an access. + pivot = d.blockchain.CurrentBlock().Header() + } + height := latest.Number.Uint64() + + origin, err := d.findAncestor(p, latest) + if err != nil { + return err + } + d.syncStatsLock.Lock() + if d.syncStatsChainHeight <= origin || d.syncStatsChainOrigin > origin { + d.syncStatsChainOrigin = origin + } + d.syncStatsChainHeight = height + d.syncStatsLock.Unlock() + + // Ensure our origin point is below any fast sync pivot point + if mode == FastSync { + if height <= uint64(fsMinFullBlocks) { + origin = 0 + } else { + pivotNumber := pivot.Number.Uint64() + if pivotNumber <= origin { + origin = pivotNumber - 1 + } + // Write out the pivot into the database so a rollback beyond it will + // reenable fast sync + rawdb.WriteLastPivotNumber(d.stateDB, pivotNumber) + } + } + d.committed = 1 + if mode == FastSync && pivot.Number.Uint64() != 0 { + d.committed = 0 + } + if mode == FastSync { + // Set the ancient data limitation. + // If we are running fast sync, all block data older than ancientLimit will be + // written to the ancient store. More recent data will be written to the active + // database and will wait for the freezer to migrate. + // + // If there is a checkpoint available, then calculate the ancientLimit through + // that. Otherwise calculate the ancient limit through the advertised height + // of the remote peer. + // + // The reason for picking checkpoint first is that a malicious peer can give us + // a fake (very high) height, forcing the ancient limit to also be very high. + // The peer would start to feed us valid blocks until head, resulting in all of + // the blocks might be written into the ancient store. A following mini-reorg + // could cause issues. + if d.checkpoint != 0 && d.checkpoint > fullMaxForkAncestry+1 { + d.ancientLimit = d.checkpoint + } else if height > fullMaxForkAncestry+1 { + d.ancientLimit = height - fullMaxForkAncestry - 1 + } else { + d.ancientLimit = 0 + } + frozen, _ := d.stateDB.Ancients() // Ignore the error here since light client can also hit here. + + // If a part of blockchain data has already been written into active store, + // disable the ancient style insertion explicitly. + if origin >= frozen && frozen != 0 { + d.ancientLimit = 0 + log.Info("Disabling direct-ancient mode", "origin", origin, "ancient", frozen-1) + } else if d.ancientLimit > 0 { + log.Debug("Enabling direct-ancient mode", "ancient", d.ancientLimit) + } + // Rewind the ancient store and blockchain if reorg happens. + if origin+1 < frozen { + if err := d.lightchain.SetHead(origin + 1); err != nil { + return err + } + } + } + // Initiate the sync using a concurrent header and content retrieval algorithm + d.queue.Prepare(origin+1, mode) + if d.syncInitHook != nil { + d.syncInitHook(origin, height) + } + fetchers := []func() error{ + func() error { return d.fetchHeaders(p, origin+1) }, // Headers are always retrieved + func() error { return d.fetchBodies(origin + 1) }, // Bodies are retrieved during normal and fast sync + func() error { return d.fetchReceipts(origin + 1) }, // Receipts are retrieved during fast sync + func() error { return d.processHeaders(origin+1, td) }, + } + if mode == FastSync { + d.pivotLock.Lock() + d.pivotHeader = pivot + d.pivotLock.Unlock() + + fetchers = append(fetchers, func() error { return d.processFastSyncContent() }) + } else if mode == FullSync { + fetchers = append(fetchers, d.processFullSyncContent) + } + return d.spawnSync(fetchers) +} + +// spawnSync runs d.process and all given fetcher functions to completion in +// separate goroutines, returning the first error that appears. +func (d *Downloader) spawnSync(fetchers []func() error) error { + errc := make(chan error, len(fetchers)) + d.cancelWg.Add(len(fetchers)) + for _, fn := range fetchers { + fn := fn + go func() { defer d.cancelWg.Done(); errc <- fn() }() + } + // Wait for the first error, then terminate the others. + var err error + for i := 0; i < len(fetchers); i++ { + if i == len(fetchers)-1 { + // Close the queue when all fetchers have exited. + // This will cause the block processor to end when + // it has processed the queue. + d.queue.Close() + } + if err = <-errc; err != nil && err != errCanceled { + break + } + } + d.queue.Close() + d.Cancel() + return err +} + +// cancel aborts all of the operations and resets the queue. However, cancel does +// not wait for the running download goroutines to finish. This method should be +// used when cancelling the downloads from inside the downloader. +func (d *Downloader) cancel() { + // Close the current cancel channel + d.cancelLock.Lock() + defer d.cancelLock.Unlock() + + if d.cancelCh != nil { + select { + case <-d.cancelCh: + // Channel was already closed + default: + close(d.cancelCh) + } + } +} + +// Cancel aborts all of the operations and waits for all download goroutines to +// finish before returning. +func (d *Downloader) Cancel() { + d.cancel() + d.cancelWg.Wait() +} + +// Terminate interrupts the downloader, canceling all pending operations. +// The downloader cannot be reused after calling Terminate. +func (d *Downloader) Terminate() { + // Close the termination channel (make sure double close is allowed) + d.quitLock.Lock() + select { + case <-d.quitCh: + default: + close(d.quitCh) + } + if d.stateBloom != nil { + d.stateBloom.Close() + } + d.quitLock.Unlock() + + // Cancel any pending download requests + d.Cancel() +} + +// fetchHead retrieves the head header and prior pivot block (if available) from +// a remote peer. +func (d *Downloader) fetchHead(p *peerConnection) (head *types.Header, pivot *types.Header, err error) { + p.log.Debug("Retrieving remote chain head") + mode := d.getMode() + + // Request the advertised remote head block and wait for the response + latest, _ := p.peer.Head() + fetch := 1 + if mode == FastSync { + fetch = 2 // head + pivot headers + } + go p.peer.RequestHeadersByHash(latest, fetch, fsMinFullBlocks-1, true) + + ttl := d.peers.rates.TargetTimeout() + timeout := time.After(ttl) + for { + select { + case <-d.cancelCh: + return nil, nil, errCanceled + + case packet := <-d.headerCh: + // Discard anything not from the origin peer + if packet.PeerId() != p.id { + log.Debug("Received headers from incorrect peer", "peer", packet.PeerId()) + break + } + // Make sure the peer gave us at least one and at most the requested headers + headers := packet.(*headerPack).headers + if len(headers) == 0 || len(headers) > fetch { + return nil, nil, fmt.Errorf("%w: returned headers %d != requested %d", errBadPeer, len(headers), fetch) + } + // The first header needs to be the head, validate against the checkpoint + // and request. If only 1 header was returned, make sure there's no pivot + // or there was not one requested. + head := headers[0] + if (mode == FastSync || mode == LightSync) && head.Number.Uint64() < d.checkpoint { + return nil, nil, fmt.Errorf("%w: remote head %d below checkpoint %d", errUnsyncedPeer, head.Number, d.checkpoint) + } + if len(headers) == 1 { + if mode == FastSync && head.Number.Uint64() > uint64(fsMinFullBlocks) { + return nil, nil, fmt.Errorf("%w: no pivot included along head header", errBadPeer) + } + p.log.Debug("Remote head identified, no pivot", "number", head.Number, "hash", head.Hash()) + return head, nil, nil + } + // At this point we have 2 headers in total and the first is the + // validated head of the chain. Check the pivot number and return, + pivot := headers[1] + if pivot.Number.Uint64() != head.Number.Uint64()-uint64(fsMinFullBlocks) { + return nil, nil, fmt.Errorf("%w: remote pivot %d != requested %d", errInvalidChain, pivot.Number, head.Number.Uint64()-uint64(fsMinFullBlocks)) + } + return head, pivot, nil + + case <-timeout: + p.log.Debug("Waiting for head header timed out", "elapsed", ttl) + return nil, nil, errTimeout + + case <-d.bodyCh: + case <-d.receiptCh: + // Out of bounds delivery, ignore + } + } +} + +// calculateRequestSpan calculates what headers to request from a peer when trying to determine the +// common ancestor. +// It returns parameters to be used for peer.RequestHeadersByNumber: +// from - starting block number +// count - number of headers to request +// skip - number of headers to skip +// and also returns 'max', the last block which is expected to be returned by the remote peers, +// given the (from,count,skip) +func calculateRequestSpan(remoteHeight, localHeight uint64) (int64, int, int, uint64) { + var ( + from int + count int + MaxCount = MaxHeaderFetch / 16 + ) + // requestHead is the highest block that we will ask for. If requestHead is not offset, + // the highest block that we will get is 16 blocks back from head, which means we + // will fetch 14 or 15 blocks unnecessarily in the case the height difference + // between us and the peer is 1-2 blocks, which is most common + requestHead := int(remoteHeight) - 1 + if requestHead < 0 { + requestHead = 0 + } + // requestBottom is the lowest block we want included in the query + // Ideally, we want to include the one just below our own head + requestBottom := int(localHeight - 1) + if requestBottom < 0 { + requestBottom = 0 + } + totalSpan := requestHead - requestBottom + span := 1 + totalSpan/MaxCount + if span < 2 { + span = 2 + } + if span > 16 { + span = 16 + } + + count = 1 + totalSpan/span + if count > MaxCount { + count = MaxCount + } + if count < 2 { + count = 2 + } + from = requestHead - (count-1)*span + if from < 0 { + from = 0 + } + max := from + (count-1)*span + return int64(from), count, span - 1, uint64(max) +} + +// findAncestor tries to locate the common ancestor link of the local chain and +// a remote peers blockchain. In the general case when our node was in sync and +// on the correct chain, checking the top N links should already get us a match. +// In the rare scenario when we ended up on a long reorganisation (i.e. none of +// the head links match), we do a binary search to find the common ancestor. +func (d *Downloader) findAncestor(p *peerConnection, remoteHeader *types.Header) (uint64, error) { + // Figure out the valid ancestor range to prevent rewrite attacks + var ( + floor = int64(-1) + localHeight uint64 + remoteHeight = remoteHeader.Number.Uint64() + ) + mode := d.getMode() + switch mode { + case FullSync: + localHeight = d.blockchain.CurrentBlock().NumberU64() + case FastSync: + localHeight = d.blockchain.CurrentFastBlock().NumberU64() + default: + localHeight = d.lightchain.CurrentHeader().Number.Uint64() + } + p.log.Debug("Looking for common ancestor", "local", localHeight, "remote", remoteHeight) + + // Recap floor value for binary search + maxForkAncestry := fullMaxForkAncestry + if d.getMode() == LightSync { + maxForkAncestry = lightMaxForkAncestry + } + if localHeight >= maxForkAncestry { + // We're above the max reorg threshold, find the earliest fork point + floor = int64(localHeight - maxForkAncestry) + } + // If we're doing a light sync, ensure the floor doesn't go below the CHT, as + // all headers before that point will be missing. + if mode == LightSync { + // If we don't know the current CHT position, find it + if d.genesis == 0 { + header := d.lightchain.CurrentHeader() + for header != nil { + d.genesis = header.Number.Uint64() + if floor >= int64(d.genesis)-1 { + break + } + header = d.lightchain.GetHeaderByHash(header.ParentHash) + } + } + // We already know the "genesis" block number, cap floor to that + if floor < int64(d.genesis)-1 { + floor = int64(d.genesis) - 1 + } + } + + ancestor, err := d.findAncestorSpanSearch(p, mode, remoteHeight, localHeight, floor) + if err == nil { + return ancestor, nil + } + // The returned error was not nil. + // If the error returned does not reflect that a common ancestor was not found, return it. + // If the error reflects that a common ancestor was not found, continue to binary search, + // where the error value will be reassigned. + if !errors.Is(err, errNoAncestorFound) { + return 0, err + } + + ancestor, err = d.findAncestorBinarySearch(p, mode, remoteHeight, floor) + if err != nil { + return 0, err + } + return ancestor, nil +} + +func (d *Downloader) findAncestorSpanSearch(p *peerConnection, mode SyncMode, remoteHeight, localHeight uint64, floor int64) (commonAncestor uint64, err error) { + from, count, skip, max := calculateRequestSpan(remoteHeight, localHeight) + + p.log.Trace("Span searching for common ancestor", "count", count, "from", from, "skip", skip) + go p.peer.RequestHeadersByNumber(uint64(from), count, skip, false) + + // Wait for the remote response to the head fetch + number, hash := uint64(0), common.Hash{} + + ttl := d.peers.rates.TargetTimeout() + timeout := time.After(ttl) + + for finished := false; !finished; { + select { + case <-d.cancelCh: + return 0, errCanceled + + case packet := <-d.headerCh: + // Discard anything not from the origin peer + if packet.PeerId() != p.id { + log.Debug("Received headers from incorrect peer", "peer", packet.PeerId()) + break + } + // Make sure the peer actually gave something valid + headers := packet.(*headerPack).headers + if len(headers) == 0 { + p.log.Warn("Empty head header set") + return 0, errEmptyHeaderSet + } + // Make sure the peer's reply conforms to the request + for i, header := range headers { + expectNumber := from + int64(i)*int64(skip+1) + if number := header.Number.Int64(); number != expectNumber { + p.log.Warn("Head headers broke chain ordering", "index", i, "requested", expectNumber, "received", number) + return 0, fmt.Errorf("%w: %v", errInvalidChain, errors.New("head headers broke chain ordering")) + } + } + // Check if a common ancestor was found + finished = true + for i := len(headers) - 1; i >= 0; i-- { + // Skip any headers that underflow/overflow our requested set + if headers[i].Number.Int64() < from || headers[i].Number.Uint64() > max { + continue + } + // Otherwise check if we already know the header or not + h := headers[i].Hash() + n := headers[i].Number.Uint64() + + var known bool + switch mode { + case FullSync: + known = d.blockchain.HasBlock(h, n) + case FastSync: + known = d.blockchain.HasFastBlock(h, n) + default: + known = d.lightchain.HasHeader(h, n) + } + if known { + number, hash = n, h + break + } + } + + case <-timeout: + p.log.Debug("Waiting for head header timed out", "elapsed", ttl) + return 0, errTimeout + + case <-d.bodyCh: + case <-d.receiptCh: + // Out of bounds delivery, ignore + } + } + // If the head fetch already found an ancestor, return + if hash != (common.Hash{}) { + if int64(number) <= floor { + p.log.Warn("Ancestor below allowance", "number", number, "hash", hash, "allowance", floor) + return 0, errInvalidAncestor + } + p.log.Debug("Found common ancestor", "number", number, "hash", hash) + return number, nil + } + return 0, errNoAncestorFound +} + +func (d *Downloader) findAncestorBinarySearch(p *peerConnection, mode SyncMode, remoteHeight uint64, floor int64) (commonAncestor uint64, err error) { + hash := common.Hash{} + + // Ancestor not found, we need to binary search over our chain + start, end := uint64(0), remoteHeight + if floor > 0 { + start = uint64(floor) + } + p.log.Trace("Binary searching for common ancestor", "start", start, "end", end) + + for start+1 < end { + // Split our chain interval in two, and request the hash to cross check + check := (start + end) / 2 + + ttl := d.peers.rates.TargetTimeout() + timeout := time.After(ttl) + + go p.peer.RequestHeadersByNumber(check, 1, 0, false) + + // Wait until a reply arrives to this request + for arrived := false; !arrived; { + select { + case <-d.cancelCh: + return 0, errCanceled + + case packet := <-d.headerCh: + // Discard anything not from the origin peer + if packet.PeerId() != p.id { + log.Debug("Received headers from incorrect peer", "peer", packet.PeerId()) + break + } + // Make sure the peer actually gave something valid + headers := packet.(*headerPack).headers + if len(headers) != 1 { + p.log.Warn("Multiple headers for single request", "headers", len(headers)) + return 0, fmt.Errorf("%w: multiple headers (%d) for single request", errBadPeer, len(headers)) + } + arrived = true + + // Modify the search interval based on the response + h := headers[0].Hash() + n := headers[0].Number.Uint64() + + var known bool + switch mode { + case FullSync: + known = d.blockchain.HasBlock(h, n) + case FastSync: + known = d.blockchain.HasFastBlock(h, n) + default: + known = d.lightchain.HasHeader(h, n) + } + if !known { + end = check + break + } + header := d.lightchain.GetHeaderByHash(h) // Independent of sync mode, header surely exists + if header.Number.Uint64() != check { + p.log.Warn("Received non requested header", "number", header.Number, "hash", header.Hash(), "request", check) + return 0, fmt.Errorf("%w: non-requested header (%d)", errBadPeer, header.Number) + } + start = check + hash = h + + case <-timeout: + p.log.Debug("Waiting for search header timed out", "elapsed", ttl) + return 0, errTimeout + + case <-d.bodyCh: + case <-d.receiptCh: + // Out of bounds delivery, ignore + } + } + } + // Ensure valid ancestry and return + if int64(start) <= floor { + p.log.Warn("Ancestor below allowance", "number", start, "hash", hash, "allowance", floor) + return 0, errInvalidAncestor + } + p.log.Debug("Found common ancestor", "number", start, "hash", hash) + return start, nil +} + +// fetchHeaders keeps retrieving headers concurrently from the number +// requested, until no more are returned, potentially throttling on the way. To +// facilitate concurrency but still protect against malicious nodes sending bad +// headers, we construct a header chain skeleton using the "origin" peer we are +// syncing with, and fill in the missing headers using anyone else. Headers from +// other peers are only accepted if they map cleanly to the skeleton. If no one +// can fill in the skeleton - not even the origin peer - it's assumed invalid and +// the origin is dropped. +func (d *Downloader) fetchHeaders(p *peerConnection, from uint64) error { + p.log.Debug("Directing header downloads", "origin", from) + defer p.log.Debug("Header download terminated") + + // Create a timeout timer, and the associated header fetcher + skeleton := true // Skeleton assembly phase or finishing up + pivoting := false // Whether the next request is pivot verification + request := time.Now() // time of the last skeleton fetch request + timeout := time.NewTimer(0) // timer to dump a non-responsive active peer + <-timeout.C // timeout channel should be initially empty + defer timeout.Stop() + + var ttl time.Duration + getHeaders := func(from uint64) { + request = time.Now() + + ttl = d.peers.rates.TargetTimeout() + timeout.Reset(ttl) + + if skeleton { + p.log.Trace("Fetching skeleton headers", "count", MaxHeaderFetch, "from", from) + go p.peer.RequestHeadersByNumber(from+uint64(MaxHeaderFetch)-1, MaxSkeletonSize, MaxHeaderFetch-1, false) + } else { + p.log.Trace("Fetching full headers", "count", MaxHeaderFetch, "from", from) + go p.peer.RequestHeadersByNumber(from, MaxHeaderFetch, 0, false) + } + } + getNextPivot := func() { + pivoting = true + request = time.Now() + + ttl = d.peers.rates.TargetTimeout() + timeout.Reset(ttl) + + d.pivotLock.RLock() + pivot := d.pivotHeader.Number.Uint64() + d.pivotLock.RUnlock() + + p.log.Trace("Fetching next pivot header", "number", pivot+uint64(fsMinFullBlocks)) + go p.peer.RequestHeadersByNumber(pivot+uint64(fsMinFullBlocks), 2, fsMinFullBlocks-9, false) // move +64 when it's 2x64-8 deep + } + // Start pulling the header chain skeleton until all is done + ancestor := from + getHeaders(from) + + mode := d.getMode() + for { + select { + case <-d.cancelCh: + return errCanceled + + case packet := <-d.headerCh: + // Make sure the active peer is giving us the skeleton headers + if packet.PeerId() != p.id { + log.Debug("Received skeleton from incorrect peer", "peer", packet.PeerId()) + break + } + headerReqTimer.UpdateSince(request) + timeout.Stop() + + // If the pivot is being checked, move if it became stale and run the real retrieval + var pivot uint64 + + d.pivotLock.RLock() + if d.pivotHeader != nil { + pivot = d.pivotHeader.Number.Uint64() + } + d.pivotLock.RUnlock() + + if pivoting { + if packet.Items() == 2 { + // Retrieve the headers and do some sanity checks, just in case + headers := packet.(*headerPack).headers + + if have, want := headers[0].Number.Uint64(), pivot+uint64(fsMinFullBlocks); have != want { + log.Warn("Peer sent invalid next pivot", "have", have, "want", want) + return fmt.Errorf("%w: next pivot number %d != requested %d", errInvalidChain, have, want) + } + if have, want := headers[1].Number.Uint64(), pivot+2*uint64(fsMinFullBlocks)-8; have != want { + log.Warn("Peer sent invalid pivot confirmer", "have", have, "want", want) + return fmt.Errorf("%w: next pivot confirmer number %d != requested %d", errInvalidChain, have, want) + } + log.Warn("Pivot seemingly stale, moving", "old", pivot, "new", headers[0].Number) + pivot = headers[0].Number.Uint64() + + d.pivotLock.Lock() + d.pivotHeader = headers[0] + d.pivotLock.Unlock() + + // Write out the pivot into the database so a rollback beyond + // it will reenable fast sync and update the state root that + // the state syncer will be downloading. + rawdb.WriteLastPivotNumber(d.stateDB, pivot) + } + pivoting = false + getHeaders(from) + continue + } + // If the skeleton's finished, pull any remaining head headers directly from the origin + if skeleton && packet.Items() == 0 { + skeleton = false + getHeaders(from) + continue + } + // If no more headers are inbound, notify the content fetchers and return + if packet.Items() == 0 { + // Don't abort header fetches while the pivot is downloading + if atomic.LoadInt32(&d.committed) == 0 && pivot <= from { + p.log.Debug("No headers, waiting for pivot commit") + select { + case <-time.After(fsHeaderContCheck): + getHeaders(from) + continue + case <-d.cancelCh: + return errCanceled + } + } + // Pivot done (or not in fast sync) and no more headers, terminate the process + p.log.Debug("No more headers available") + select { + case d.headerProcCh <- nil: + return nil + case <-d.cancelCh: + return errCanceled + } + } + headers := packet.(*headerPack).headers + + // If we received a skeleton batch, resolve internals concurrently + if skeleton { + filled, proced, err := d.fillHeaderSkeleton(from, headers) + if err != nil { + p.log.Debug("Skeleton chain invalid", "err", err) + return fmt.Errorf("%w: %v", errInvalidChain, err) + } + headers = filled[proced:] + from += uint64(proced) + } else { + // If we're closing in on the chain head, but haven't yet reached it, delay + // the last few headers so mini reorgs on the head don't cause invalid hash + // chain errors. + if n := len(headers); n > 0 { + // Retrieve the current head we're at + var head uint64 + if mode == LightSync { + head = d.lightchain.CurrentHeader().Number.Uint64() + } else { + head = d.blockchain.CurrentFastBlock().NumberU64() + if full := d.blockchain.CurrentBlock().NumberU64(); head < full { + head = full + } + } + // If the head is below the common ancestor, we're actually deduplicating + // already existing chain segments, so use the ancestor as the fake head. + // Otherwise we might end up delaying header deliveries pointlessly. + if head < ancestor { + head = ancestor + } + // If the head is way older than this batch, delay the last few headers + if head+uint64(reorgProtThreshold) < headers[n-1].Number.Uint64() { + delay := reorgProtHeaderDelay + if delay > n { + delay = n + } + headers = headers[:n-delay] + } + } + } + // Insert all the new headers and fetch the next batch + if len(headers) > 0 { + p.log.Trace("Scheduling new headers", "count", len(headers), "from", from) + select { + case d.headerProcCh <- headers: + case <-d.cancelCh: + return errCanceled + } + from += uint64(len(headers)) + + // If we're still skeleton filling fast sync, check pivot staleness + // before continuing to the next skeleton filling + if skeleton && pivot > 0 { + getNextPivot() + } else { + getHeaders(from) + } + } else { + // No headers delivered, or all of them being delayed, sleep a bit and retry + p.log.Trace("All headers delayed, waiting") + select { + case <-time.After(fsHeaderContCheck): + getHeaders(from) + continue + case <-d.cancelCh: + return errCanceled + } + } + + case <-timeout.C: + if d.dropPeer == nil { + // The dropPeer method is nil when `--copydb` is used for a local copy. + // Timeouts can occur if e.g. compaction hits at the wrong time, and can be ignored + p.log.Warn("Downloader wants to drop peer, but peerdrop-function is not set", "peer", p.id) + break + } + // Header retrieval timed out, consider the peer bad and drop + p.log.Debug("Header request timed out", "elapsed", ttl) + headerTimeoutMeter.Mark(1) + d.dropPeer(p.id) + + // Finish the sync gracefully instead of dumping the gathered data though + for _, ch := range []chan bool{d.bodyWakeCh, d.receiptWakeCh} { + select { + case ch <- false: + case <-d.cancelCh: + } + } + select { + case d.headerProcCh <- nil: + case <-d.cancelCh: + } + return fmt.Errorf("%w: header request timed out", errBadPeer) + } + } +} + +// fillHeaderSkeleton concurrently retrieves headers from all our available peers +// and maps them to the provided skeleton header chain. +// +// Any partial results from the beginning of the skeleton is (if possible) forwarded +// immediately to the header processor to keep the rest of the pipeline full even +// in the case of header stalls. +// +// The method returns the entire filled skeleton and also the number of headers +// already forwarded for processing. +func (d *Downloader) fillHeaderSkeleton(from uint64, skeleton []*types.Header) ([]*types.Header, int, error) { + log.Debug("Filling up skeleton", "from", from) + d.queue.ScheduleSkeleton(from, skeleton) + + var ( + deliver = func(packet dataPack) (int, error) { + pack := packet.(*headerPack) + return d.queue.DeliverHeaders(pack.peerID, pack.headers, d.headerProcCh) + } + expire = func() map[string]int { return d.queue.ExpireHeaders(d.peers.rates.TargetTimeout()) } + reserve = func(p *peerConnection, count int) (*fetchRequest, bool, bool) { + return d.queue.ReserveHeaders(p, count), false, false + } + fetch = func(p *peerConnection, req *fetchRequest) error { return p.FetchHeaders(req.From, MaxHeaderFetch) } + capacity = func(p *peerConnection) int { return p.HeaderCapacity(d.peers.rates.TargetRoundTrip()) } + setIdle = func(p *peerConnection, accepted int, deliveryTime time.Time) { + p.SetHeadersIdle(accepted, deliveryTime) + } + ) + err := d.fetchParts(d.headerCh, deliver, d.queue.headerContCh, expire, + d.queue.PendingHeaders, d.queue.InFlightHeaders, reserve, + nil, fetch, d.queue.CancelHeaders, capacity, d.peers.HeaderIdlePeers, setIdle, "headers") + + log.Debug("Skeleton fill terminated", "err", err) + + filled, proced := d.queue.RetrieveHeaders() + return filled, proced, err +} + +// fetchBodies iteratively downloads the scheduled block bodies, taking any +// available peers, reserving a chunk of blocks for each, waiting for delivery +// and also periodically checking for timeouts. +func (d *Downloader) fetchBodies(from uint64) error { + log.Debug("Downloading block bodies", "origin", from) + + var ( + deliver = func(packet dataPack) (int, error) { + pack := packet.(*bodyPack) + return d.queue.DeliverBodies(pack.peerID, pack.transactions, pack.uncles) + } + expire = func() map[string]int { return d.queue.ExpireBodies(d.peers.rates.TargetTimeout()) } + fetch = func(p *peerConnection, req *fetchRequest) error { return p.FetchBodies(req) } + capacity = func(p *peerConnection) int { return p.BlockCapacity(d.peers.rates.TargetRoundTrip()) } + setIdle = func(p *peerConnection, accepted int, deliveryTime time.Time) { p.SetBodiesIdle(accepted, deliveryTime) } + ) + err := d.fetchParts(d.bodyCh, deliver, d.bodyWakeCh, expire, + d.queue.PendingBlocks, d.queue.InFlightBlocks, d.queue.ReserveBodies, + d.bodyFetchHook, fetch, d.queue.CancelBodies, capacity, d.peers.BodyIdlePeers, setIdle, "bodies") + + log.Debug("Block body download terminated", "err", err) + return err +} + +// fetchReceipts iteratively downloads the scheduled block receipts, taking any +// available peers, reserving a chunk of receipts for each, waiting for delivery +// and also periodically checking for timeouts. +func (d *Downloader) fetchReceipts(from uint64) error { + log.Debug("Downloading transaction receipts", "origin", from) + + var ( + deliver = func(packet dataPack) (int, error) { + pack := packet.(*receiptPack) + return d.queue.DeliverReceipts(pack.peerID, pack.receipts) + } + expire = func() map[string]int { return d.queue.ExpireReceipts(d.peers.rates.TargetTimeout()) } + fetch = func(p *peerConnection, req *fetchRequest) error { return p.FetchReceipts(req) } + capacity = func(p *peerConnection) int { return p.ReceiptCapacity(d.peers.rates.TargetRoundTrip()) } + setIdle = func(p *peerConnection, accepted int, deliveryTime time.Time) { + p.SetReceiptsIdle(accepted, deliveryTime) + } + ) + err := d.fetchParts(d.receiptCh, deliver, d.receiptWakeCh, expire, + d.queue.PendingReceipts, d.queue.InFlightReceipts, d.queue.ReserveReceipts, + d.receiptFetchHook, fetch, d.queue.CancelReceipts, capacity, d.peers.ReceiptIdlePeers, setIdle, "receipts") + + log.Debug("Transaction receipt download terminated", "err", err) + return err +} + +// fetchParts iteratively downloads scheduled block parts, taking any available +// peers, reserving a chunk of fetch requests for each, waiting for delivery and +// also periodically checking for timeouts. +// +// As the scheduling/timeout logic mostly is the same for all downloaded data +// types, this method is used by each for data gathering and is instrumented with +// various callbacks to handle the slight differences between processing them. +// +// The instrumentation parameters: +// - errCancel: error type to return if the fetch operation is cancelled (mostly makes logging nicer) +// - deliveryCh: channel from which to retrieve downloaded data packets (merged from all concurrent peers) +// - deliver: processing callback to deliver data packets into type specific download queues (usually within `queue`) +// - wakeCh: notification channel for waking the fetcher when new tasks are available (or sync completed) +// - expire: task callback method to abort requests that took too long and return the faulty peers (traffic shaping) +// - pending: task callback for the number of requests still needing download (detect completion/non-completability) +// - inFlight: task callback for the number of in-progress requests (wait for all active downloads to finish) +// - throttle: task callback to check if the processing queue is full and activate throttling (bound memory use) +// - reserve: task callback to reserve new download tasks to a particular peer (also signals partial completions) +// - fetchHook: tester callback to notify of new tasks being initiated (allows testing the scheduling logic) +// - fetch: network callback to actually send a particular download request to a physical remote peer +// - cancel: task callback to abort an in-flight download request and allow rescheduling it (in case of lost peer) +// - capacity: network callback to retrieve the estimated type-specific bandwidth capacity of a peer (traffic shaping) +// - idle: network callback to retrieve the currently (type specific) idle peers that can be assigned tasks +// - setIdle: network callback to set a peer back to idle and update its estimated capacity (traffic shaping) +// - kind: textual label of the type being downloaded to display in log messages +func (d *Downloader) fetchParts(deliveryCh chan dataPack, deliver func(dataPack) (int, error), wakeCh chan bool, + expire func() map[string]int, pending func() int, inFlight func() bool, reserve func(*peerConnection, int) (*fetchRequest, bool, bool), + fetchHook func([]*types.Header), fetch func(*peerConnection, *fetchRequest) error, cancel func(*fetchRequest), capacity func(*peerConnection) int, + idle func() ([]*peerConnection, int), setIdle func(*peerConnection, int, time.Time), kind string) error { + + // Create a ticker to detect expired retrieval tasks + ticker := time.NewTicker(100 * time.Millisecond) + defer ticker.Stop() + + update := make(chan struct{}, 1) + + // Prepare the queue and fetch block parts until the block header fetcher's done + finished := false + for { + select { + case <-d.cancelCh: + return errCanceled + + case packet := <-deliveryCh: + deliveryTime := time.Now() + // If the peer was previously banned and failed to deliver its pack + // in a reasonable time frame, ignore its message. + if peer := d.peers.Peer(packet.PeerId()); peer != nil { + // Deliver the received chunk of data and check chain validity + accepted, err := deliver(packet) + if errors.Is(err, errInvalidChain) { + return err + } + // Unless a peer delivered something completely else than requested (usually + // caused by a timed out request which came through in the end), set it to + // idle. If the delivery's stale, the peer should have already been idled. + if !errors.Is(err, errStaleDelivery) { + setIdle(peer, accepted, deliveryTime) + } + // Issue a log to the user to see what's going on + switch { + case err == nil && packet.Items() == 0: + peer.log.Trace("Requested data not delivered", "type", kind) + case err == nil: + peer.log.Trace("Delivered new batch of data", "type", kind, "count", packet.Stats()) + default: + peer.log.Debug("Failed to deliver retrieved data", "type", kind, "err", err) + } + } + // Blocks assembled, try to update the progress + select { + case update <- struct{}{}: + default: + } + + case cont := <-wakeCh: + // The header fetcher sent a continuation flag, check if it's done + if !cont { + finished = true + } + // Headers arrive, try to update the progress + select { + case update <- struct{}{}: + default: + } + + case <-ticker.C: + // Sanity check update the progress + select { + case update <- struct{}{}: + default: + } + + case <-update: + // Short circuit if we lost all our peers + if d.peers.Len() == 0 { + return errNoPeers + } + // Check for fetch request timeouts and demote the responsible peers + for pid, fails := range expire() { + if peer := d.peers.Peer(pid); peer != nil { + // If a lot of retrieval elements expired, we might have overestimated the remote peer or perhaps + // ourselves. Only reset to minimal throughput but don't drop just yet. If even the minimal times + // out that sync wise we need to get rid of the peer. + // + // The reason the minimum threshold is 2 is because the downloader tries to estimate the bandwidth + // and latency of a peer separately, which requires pushing the measures capacity a bit and seeing + // how response times reacts, to it always requests one more than the minimum (i.e. min 2). + if fails > 2 { + peer.log.Trace("Data delivery timed out", "type", kind) + setIdle(peer, 0, time.Now()) + } else { + peer.log.Debug("Stalling delivery, dropping", "type", kind) + + if d.dropPeer == nil { + // The dropPeer method is nil when `--copydb` is used for a local copy. + // Timeouts can occur if e.g. compaction hits at the wrong time, and can be ignored + peer.log.Warn("Downloader wants to drop peer, but peerdrop-function is not set", "peer", pid) + } else { + d.dropPeer(pid) + + // If this peer was the master peer, abort sync immediately + d.cancelLock.RLock() + master := pid == d.cancelPeer + d.cancelLock.RUnlock() + + if master { + d.cancel() + return errTimeout + } + } + } + } + } + // If there's nothing more to fetch, wait or terminate + if pending() == 0 { + if !inFlight() && finished { + log.Debug("Data fetching completed", "type", kind) + return nil + } + break + } + // Send a download request to all idle peers, until throttled + progressed, throttled, running := false, false, inFlight() + idles, total := idle() + pendCount := pending() + for _, peer := range idles { + // Short circuit if throttling activated + if throttled { + break + } + // Short circuit if there is no more available task. + if pendCount = pending(); pendCount == 0 { + break + } + // Reserve a chunk of fetches for a peer. A nil can mean either that + // no more headers are available, or that the peer is known not to + // have them. + request, progress, throttle := reserve(peer, capacity(peer)) + if progress { + progressed = true + } + if throttle { + throttled = true + throttleCounter.Inc(1) + } + if request == nil { + continue + } + if request.From > 0 { + peer.log.Trace("Requesting new batch of data", "type", kind, "from", request.From) + } else { + peer.log.Trace("Requesting new batch of data", "type", kind, "count", len(request.Headers), "from", request.Headers[0].Number) + } + // Fetch the chunk and make sure any errors return the hashes to the queue + if fetchHook != nil { + fetchHook(request.Headers) + } + if err := fetch(peer, request); err != nil { + // Although we could try and make an attempt to fix this, this error really + // means that we've double allocated a fetch task to a peer. If that is the + // case, the internal state of the downloader and the queue is very wrong so + // better hard crash and note the error instead of silently accumulating into + // a much bigger issue. + panic(fmt.Sprintf("%v: %s fetch assignment failed", peer, kind)) + } + running = true + } + // Make sure that we have peers available for fetching. If all peers have been tried + // and all failed throw an error + if !progressed && !throttled && !running && len(idles) == total && pendCount > 0 { + return errPeersUnavailable + } + } + } +} + +// processHeaders takes batches of retrieved headers from an input channel and +// keeps processing and scheduling them into the header chain and downloader's +// queue until the stream ends or a failure occurs. +func (d *Downloader) processHeaders(origin uint64, td *big.Int) error { + // Keep a count of uncertain headers to roll back + var ( + rollback uint64 // Zero means no rollback (fine as you can't unroll the genesis) + rollbackErr error + mode = d.getMode() + ) + defer func() { + if rollback > 0 { + lastHeader, lastFastBlock, lastBlock := d.lightchain.CurrentHeader().Number, common.Big0, common.Big0 + if mode != LightSync { + lastFastBlock = d.blockchain.CurrentFastBlock().Number() + lastBlock = d.blockchain.CurrentBlock().Number() + } + if err := d.lightchain.SetHead(rollback - 1); err != nil { // -1 to target the parent of the first uncertain block + // We're already unwinding the stack, only print the error to make it more visible + log.Error("Failed to roll back chain segment", "head", rollback-1, "err", err) + } + curFastBlock, curBlock := common.Big0, common.Big0 + if mode != LightSync { + curFastBlock = d.blockchain.CurrentFastBlock().Number() + curBlock = d.blockchain.CurrentBlock().Number() + } + log.Warn("Rolled back chain segment", + "header", fmt.Sprintf("%d->%d", lastHeader, d.lightchain.CurrentHeader().Number), + "fast", fmt.Sprintf("%d->%d", lastFastBlock, curFastBlock), + "block", fmt.Sprintf("%d->%d", lastBlock, curBlock), "reason", rollbackErr) + } + }() + // Wait for batches of headers to process + gotHeaders := false + + for { + select { + case <-d.cancelCh: + rollbackErr = errCanceled + return errCanceled + + case headers := <-d.headerProcCh: + // Terminate header processing if we synced up + if len(headers) == 0 { + // Notify everyone that headers are fully processed + for _, ch := range []chan bool{d.bodyWakeCh, d.receiptWakeCh} { + select { + case ch <- false: + case <-d.cancelCh: + } + } + // If no headers were retrieved at all, the peer violated its TD promise that it had a + // better chain compared to ours. The only exception is if its promised blocks were + // already imported by other means (e.g. fetcher): + // + // R , L : Both at block 10 + // R: Mine block 11, and propagate it to L + // L: Queue block 11 for import + // L: Notice that R's head and TD increased compared to ours, start sync + // L: Import of block 11 finishes + // L: Sync begins, and finds common ancestor at 11 + // L: Request new headers up from 11 (R's TD was higher, it must have something) + // R: Nothing to give + if mode != LightSync { + head := d.blockchain.CurrentBlock() + if !gotHeaders && td.Cmp(d.blockchain.GetTd(head.Hash(), head.NumberU64())) > 0 { + return errStallingPeer + } + } + // If fast or light syncing, ensure promised headers are indeed delivered. This is + // needed to detect scenarios where an attacker feeds a bad pivot and then bails out + // of delivering the post-pivot blocks that would flag the invalid content. + // + // This check cannot be executed "as is" for full imports, since blocks may still be + // queued for processing when the header download completes. However, as long as the + // peer gave us something useful, we're already happy/progressed (above check). + if mode == FastSync || mode == LightSync { + head := d.lightchain.CurrentHeader() + if td.Cmp(d.lightchain.GetTd(head.Hash(), head.Number.Uint64())) > 0 { + return errStallingPeer + } + } + // Disable any rollback and return + rollback = 0 + return nil + } + // Otherwise split the chunk of headers into batches and process them + gotHeaders = true + for len(headers) > 0 { + // Terminate if something failed in between processing chunks + select { + case <-d.cancelCh: + rollbackErr = errCanceled + return errCanceled + default: + } + // Select the next chunk of headers to import + limit := maxHeadersProcess + if limit > len(headers) { + limit = len(headers) + } + chunk := headers[:limit] + + // In case of header only syncing, validate the chunk immediately + if mode == FastSync || mode == LightSync { + // If we're importing pure headers, verify based on their recentness + var pivot uint64 + + d.pivotLock.RLock() + if d.pivotHeader != nil { + pivot = d.pivotHeader.Number.Uint64() + } + d.pivotLock.RUnlock() + + frequency := fsHeaderCheckFrequency + if chunk[len(chunk)-1].Number.Uint64()+uint64(fsHeaderForceVerify) > pivot { + frequency = 1 + } + if n, err := d.lightchain.InsertHeaderChain(chunk, frequency); err != nil { + rollbackErr = err + + // If some headers were inserted, track them as uncertain + if (mode == FastSync || frequency > 1) && n > 0 && rollback == 0 { + rollback = chunk[0].Number.Uint64() + } + log.Warn("Invalid header encountered", "number", chunk[n].Number, "hash", chunk[n].Hash(), "parent", chunk[n].ParentHash, "err", err) + return fmt.Errorf("%w: %v", errInvalidChain, err) + } + // All verifications passed, track all headers within the alloted limits + if mode == FastSync { + head := chunk[len(chunk)-1].Number.Uint64() + if head-rollback > uint64(fsHeaderSafetyNet) { + rollback = head - uint64(fsHeaderSafetyNet) + } else { + rollback = 1 + } + } + } + // Unless we're doing light chains, schedule the headers for associated content retrieval + if mode == FullSync || mode == FastSync { + // If we've reached the allowed number of pending headers, stall a bit + for d.queue.PendingBlocks() >= maxQueuedHeaders || d.queue.PendingReceipts() >= maxQueuedHeaders { + select { + case <-d.cancelCh: + rollbackErr = errCanceled + return errCanceled + case <-time.After(time.Second): + } + } + // Otherwise insert the headers for content retrieval + inserts := d.queue.Schedule(chunk, origin) + if len(inserts) != len(chunk) { + rollbackErr = fmt.Errorf("stale headers: len inserts %v len(chunk) %v", len(inserts), len(chunk)) + return fmt.Errorf("%w: stale headers", errBadPeer) + } + } + headers = headers[limit:] + origin += uint64(limit) + } + // Update the highest block number we know if a higher one is found. + d.syncStatsLock.Lock() + if d.syncStatsChainHeight < origin { + d.syncStatsChainHeight = origin - 1 + } + d.syncStatsLock.Unlock() + + // Signal the content downloaders of the availablility of new tasks + for _, ch := range []chan bool{d.bodyWakeCh, d.receiptWakeCh} { + select { + case ch <- true: + default: + } + } + } + } +} + +// processFullSyncContent takes fetch results from the queue and imports them into the chain. +func (d *Downloader) processFullSyncContent() error { + for { + results := d.queue.Results(true) + if len(results) == 0 { + return nil + } + if d.chainInsertHook != nil { + d.chainInsertHook(results) + } + if err := d.importBlockResults(results); err != nil { + return err + } + } +} + +func (d *Downloader) importBlockResults(results []*fetchResult) error { + // Check for any early termination requests + if len(results) == 0 { + return nil + } + select { + case <-d.quitCh: + return errCancelContentProcessing + default: + } + // Retrieve the a batch of results to import + first, last := results[0].Header, results[len(results)-1].Header + log.Debug("Inserting downloaded chain", "items", len(results), + "firstnum", first.Number, "firsthash", first.Hash(), + "lastnum", last.Number, "lasthash", last.Hash(), + ) + blocks := make([]*types.Block, len(results)) + for i, result := range results { + blocks[i] = types.NewBlockWithHeader(result.Header).WithBody(result.Transactions, result.Uncles) + } + if index, err := d.blockchain.InsertChain(blocks); err != nil { + if index < len(results) { + log.Debug("Downloaded item processing failed", "number", results[index].Header.Number, "hash", results[index].Header.Hash(), "err", err) + } else { + // The InsertChain method in blockchain.go will sometimes return an out-of-bounds index, + // when it needs to preprocess blocks to import a sidechain. + // The importer will put together a new list of blocks to import, which is a superset + // of the blocks delivered from the downloader, and the indexing will be off. + log.Debug("Downloaded item processing failed on sidechain import", "index", index, "err", err) + } + return fmt.Errorf("%w: %v", errInvalidChain, err) + } + return nil +} + +// processFastSyncContent takes fetch results from the queue and writes them to the +// database. It also controls the synchronisation of state nodes of the pivot block. +func (d *Downloader) processFastSyncContent() error { + // Start syncing state of the reported head block. This should get us most of + // the state of the pivot block. + d.pivotLock.RLock() + sync := d.syncState(d.pivotHeader.Root) + d.pivotLock.RUnlock() + + defer func() { + // The `sync` object is replaced every time the pivot moves. We need to + // defer close the very last active one, hence the lazy evaluation vs. + // calling defer sync.Cancel() !!! + sync.Cancel() + }() + + closeOnErr := func(s *stateSync) { + if err := s.Wait(); err != nil && err != errCancelStateFetch && err != errCanceled && err != snap.ErrCancelled { + d.queue.Close() // wake up Results + } + } + go closeOnErr(sync) + + // To cater for moving pivot points, track the pivot block and subsequently + // accumulated download results separately. + var ( + oldPivot *fetchResult // Locked in pivot block, might change eventually + oldTail []*fetchResult // Downloaded content after the pivot + ) + for { + // Wait for the next batch of downloaded data to be available, and if the pivot + // block became stale, move the goalpost + results := d.queue.Results(oldPivot == nil) // Block if we're not monitoring pivot staleness + if len(results) == 0 { + // If pivot sync is done, stop + if oldPivot == nil { + return sync.Cancel() + } + // If sync failed, stop + select { + case <-d.cancelCh: + sync.Cancel() + return errCanceled + default: + } + } + if d.chainInsertHook != nil { + d.chainInsertHook(results) + } + // If we haven't downloaded the pivot block yet, check pivot staleness + // notifications from the header downloader + d.pivotLock.RLock() + pivot := d.pivotHeader + d.pivotLock.RUnlock() + + if oldPivot == nil { + if pivot.Root != sync.root { + sync.Cancel() + sync = d.syncState(pivot.Root) + + go closeOnErr(sync) + } + } else { + results = append(append([]*fetchResult{oldPivot}, oldTail...), results...) + } + // Split around the pivot block and process the two sides via fast/full sync + if atomic.LoadInt32(&d.committed) == 0 { + latest := results[len(results)-1].Header + // If the height is above the pivot block by 2 sets, it means the pivot + // become stale in the network and it was garbage collected, move to a + // new pivot. + // + // Note, we have `reorgProtHeaderDelay` number of blocks withheld, Those + // need to be taken into account, otherwise we're detecting the pivot move + // late and will drop peers due to unavailable state!!! + if height := latest.Number.Uint64(); height >= pivot.Number.Uint64()+2*uint64(fsMinFullBlocks)-uint64(reorgProtHeaderDelay) { + log.Warn("Pivot became stale, moving", "old", pivot.Number.Uint64(), "new", height-uint64(fsMinFullBlocks)+uint64(reorgProtHeaderDelay)) + pivot = results[len(results)-1-fsMinFullBlocks+reorgProtHeaderDelay].Header // must exist as lower old pivot is uncommitted + + d.pivotLock.Lock() + d.pivotHeader = pivot + d.pivotLock.Unlock() + + // Write out the pivot into the database so a rollback beyond it will + // reenable fast sync + rawdb.WriteLastPivotNumber(d.stateDB, pivot.Number.Uint64()) + } + } + P, beforeP, afterP := splitAroundPivot(pivot.Number.Uint64(), results) + if err := d.commitFastSyncData(beforeP, sync); err != nil { + return err + } + if P != nil { + // If new pivot block found, cancel old state retrieval and restart + if oldPivot != P { + sync.Cancel() + sync = d.syncState(P.Header.Root) + + go closeOnErr(sync) + oldPivot = P + } + // Wait for completion, occasionally checking for pivot staleness + select { + case <-sync.done: + if sync.err != nil { + return sync.err + } + if err := d.commitPivotBlock(P); err != nil { + return err + } + oldPivot = nil + + case <-time.After(time.Second): + oldTail = afterP + continue + } + } + // Fast sync done, pivot commit done, full import + if err := d.importBlockResults(afterP); err != nil { + return err + } + } +} + +func splitAroundPivot(pivot uint64, results []*fetchResult) (p *fetchResult, before, after []*fetchResult) { + if len(results) == 0 { + return nil, nil, nil + } + if lastNum := results[len(results)-1].Header.Number.Uint64(); lastNum < pivot { + // the pivot is somewhere in the future + return nil, results, nil + } + // This can also be optimized, but only happens very seldom + for _, result := range results { + num := result.Header.Number.Uint64() + switch { + case num < pivot: + before = append(before, result) + case num == pivot: + p = result + default: + after = append(after, result) + } + } + return p, before, after +} + +func (d *Downloader) commitFastSyncData(results []*fetchResult, stateSync *stateSync) error { + // Check for any early termination requests + if len(results) == 0 { + return nil + } + select { + case <-d.quitCh: + return errCancelContentProcessing + case <-stateSync.done: + if err := stateSync.Wait(); err != nil { + return err + } + default: + } + // Retrieve the a batch of results to import + first, last := results[0].Header, results[len(results)-1].Header + log.Debug("Inserting fast-sync blocks", "items", len(results), + "firstnum", first.Number, "firsthash", first.Hash(), + "lastnumn", last.Number, "lasthash", last.Hash(), + ) + blocks := make([]*types.Block, len(results)) + receipts := make([]types.Receipts, len(results)) + for i, result := range results { + blocks[i] = types.NewBlockWithHeader(result.Header).WithBody(result.Transactions, result.Uncles) + receipts[i] = result.Receipts + } + if index, err := d.blockchain.InsertReceiptChain(blocks, receipts, d.ancientLimit); err != nil { + log.Debug("Downloaded item processing failed", "number", results[index].Header.Number, "hash", results[index].Header.Hash(), "err", err) + return fmt.Errorf("%w: %v", errInvalidChain, err) + } + return nil +} + +func (d *Downloader) commitPivotBlock(result *fetchResult) error { + block := types.NewBlockWithHeader(result.Header).WithBody(result.Transactions, result.Uncles) + log.Debug("Committing fast sync pivot as new head", "number", block.Number(), "hash", block.Hash()) + + // Commit the pivot block as the new head, will require full sync from here on + if _, err := d.blockchain.InsertReceiptChain([]*types.Block{block}, []types.Receipts{result.Receipts}, d.ancientLimit); err != nil { + return err + } + if err := d.blockchain.FastSyncCommitHead(block.Hash()); err != nil { + return err + } + atomic.StoreInt32(&d.committed, 1) + + // If we had a bloom filter for the state sync, deallocate it now. Note, we only + // deallocate internally, but keep the empty wrapper. This ensures that if we do + // a rollback after committing the pivot and restarting fast sync, we don't end + // up using a nil bloom. Empty bloom is fine, it just returns that it does not + // have the info we need, so reach down to the database instead. + if d.stateBloom != nil { + d.stateBloom.Close() + } + return nil +} + +// DeliverHeaders injects a new batch of block headers received from a remote +// node into the download schedule. +func (d *Downloader) DeliverHeaders(id string, headers []*types.Header) error { + return d.deliver(d.headerCh, &headerPack{id, headers}, headerInMeter, headerDropMeter) +} + +// DeliverBodies injects a new batch of block bodies received from a remote node. +func (d *Downloader) DeliverBodies(id string, transactions [][]*types.Transaction, uncles [][]*types.Header) error { + return d.deliver(d.bodyCh, &bodyPack{id, transactions, uncles}, bodyInMeter, bodyDropMeter) +} + +// DeliverReceipts injects a new batch of receipts received from a remote node. +func (d *Downloader) DeliverReceipts(id string, receipts [][]*types.Receipt) error { + return d.deliver(d.receiptCh, &receiptPack{id, receipts}, receiptInMeter, receiptDropMeter) +} + +// DeliverNodeData injects a new batch of node state data received from a remote node. +func (d *Downloader) DeliverNodeData(id string, data [][]byte) error { + return d.deliver(d.stateCh, &statePack{id, data}, stateInMeter, stateDropMeter) +} + +// DeliverSnapPacket is invoked from a peer's message handler when it transmits a +// data packet for the local node to consume. +func (d *Downloader) DeliverSnapPacket(peer *snap.Peer, packet snap.Packet) error { + switch packet := packet.(type) { + case *snap.AccountRangePacket: + hashes, accounts, err := packet.Unpack() + if err != nil { + return err + } + return d.SnapSyncer.OnAccounts(peer, packet.ID, hashes, accounts, packet.Proof) + + case *snap.StorageRangesPacket: + hashset, slotset := packet.Unpack() + return d.SnapSyncer.OnStorage(peer, packet.ID, hashset, slotset, packet.Proof) + + case *snap.ByteCodesPacket: + return d.SnapSyncer.OnByteCodes(peer, packet.ID, packet.Codes) + + case *snap.TrieNodesPacket: + return d.SnapSyncer.OnTrieNodes(peer, packet.ID, packet.Nodes) + + default: + return fmt.Errorf("unexpected snap packet type: %T", packet) + } +} + +// deliver injects a new batch of data received from a remote node. +func (d *Downloader) deliver(destCh chan dataPack, packet dataPack, inMeter, dropMeter metrics.Meter) (err error) { + // Update the delivery metrics for both good and failed deliveries + inMeter.Mark(int64(packet.Items())) + defer func() { + if err != nil { + dropMeter.Mark(int64(packet.Items())) + } + }() + // Deliver or abort if the sync is canceled while queuing + d.cancelLock.RLock() + cancel := d.cancelCh + d.cancelLock.RUnlock() + if cancel == nil { + return errNoSyncActive + } + select { + case destCh <- packet: + return nil + case <-cancel: + return errNoSyncActive + } +} diff --git a/les/downloader/downloader_test.go b/les/downloader/downloader_test.go new file mode 100644 index 000000000..17cd3630c --- /dev/null +++ b/les/downloader/downloader_test.go @@ -0,0 +1,1622 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package downloader + +import ( + "errors" + "fmt" + "math/big" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/state/snapshot" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/eth/protocols/eth" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/trie" +) + +// Reduce some of the parameters to make the tester faster. +func init() { + fullMaxForkAncestry = 10000 + lightMaxForkAncestry = 10000 + blockCacheMaxItems = 1024 + fsHeaderContCheck = 500 * time.Millisecond +} + +// downloadTester is a test simulator for mocking out local block chain. +type downloadTester struct { + downloader *Downloader + + genesis *types.Block // Genesis blocks used by the tester and peers + stateDb ethdb.Database // Database used by the tester for syncing from peers + peerDb ethdb.Database // Database of the peers containing all data + peers map[string]*downloadTesterPeer + + ownHashes []common.Hash // Hash chain belonging to the tester + ownHeaders map[common.Hash]*types.Header // Headers belonging to the tester + ownBlocks map[common.Hash]*types.Block // Blocks belonging to the tester + ownReceipts map[common.Hash]types.Receipts // Receipts belonging to the tester + ownChainTd map[common.Hash]*big.Int // Total difficulties of the blocks in the local chain + + ancientHeaders map[common.Hash]*types.Header // Ancient headers belonging to the tester + ancientBlocks map[common.Hash]*types.Block // Ancient blocks belonging to the tester + ancientReceipts map[common.Hash]types.Receipts // Ancient receipts belonging to the tester + ancientChainTd map[common.Hash]*big.Int // Ancient total difficulties of the blocks in the local chain + + lock sync.RWMutex +} + +// newTester creates a new downloader test mocker. +func newTester() *downloadTester { + tester := &downloadTester{ + genesis: testGenesis, + peerDb: testDB, + peers: make(map[string]*downloadTesterPeer), + ownHashes: []common.Hash{testGenesis.Hash()}, + ownHeaders: map[common.Hash]*types.Header{testGenesis.Hash(): testGenesis.Header()}, + ownBlocks: map[common.Hash]*types.Block{testGenesis.Hash(): testGenesis}, + ownReceipts: map[common.Hash]types.Receipts{testGenesis.Hash(): nil}, + ownChainTd: map[common.Hash]*big.Int{testGenesis.Hash(): testGenesis.Difficulty()}, + + // Initialize ancient store with test genesis block + ancientHeaders: map[common.Hash]*types.Header{testGenesis.Hash(): testGenesis.Header()}, + ancientBlocks: map[common.Hash]*types.Block{testGenesis.Hash(): testGenesis}, + ancientReceipts: map[common.Hash]types.Receipts{testGenesis.Hash(): nil}, + ancientChainTd: map[common.Hash]*big.Int{testGenesis.Hash(): testGenesis.Difficulty()}, + } + tester.stateDb = rawdb.NewMemoryDatabase() + tester.stateDb.Put(testGenesis.Root().Bytes(), []byte{0x00}) + + tester.downloader = New(0, tester.stateDb, trie.NewSyncBloom(1, tester.stateDb), new(event.TypeMux), tester, nil, tester.dropPeer) + return tester +} + +// terminate aborts any operations on the embedded downloader and releases all +// held resources. +func (dl *downloadTester) terminate() { + dl.downloader.Terminate() +} + +// sync starts synchronizing with a remote peer, blocking until it completes. +func (dl *downloadTester) sync(id string, td *big.Int, mode SyncMode) error { + dl.lock.RLock() + hash := dl.peers[id].chain.headBlock().Hash() + // If no particular TD was requested, load from the peer's blockchain + if td == nil { + td = dl.peers[id].chain.td(hash) + } + dl.lock.RUnlock() + + // Synchronise with the chosen peer and ensure proper cleanup afterwards + err := dl.downloader.synchronise(id, hash, td, mode) + select { + case <-dl.downloader.cancelCh: + // Ok, downloader fully cancelled after sync cycle + default: + // Downloader is still accepting packets, can block a peer up + panic("downloader active post sync cycle") // panic will be caught by tester + } + return err +} + +// HasHeader checks if a header is present in the testers canonical chain. +func (dl *downloadTester) HasHeader(hash common.Hash, number uint64) bool { + return dl.GetHeaderByHash(hash) != nil +} + +// HasBlock checks if a block is present in the testers canonical chain. +func (dl *downloadTester) HasBlock(hash common.Hash, number uint64) bool { + return dl.GetBlockByHash(hash) != nil +} + +// HasFastBlock checks if a block is present in the testers canonical chain. +func (dl *downloadTester) HasFastBlock(hash common.Hash, number uint64) bool { + dl.lock.RLock() + defer dl.lock.RUnlock() + + if _, ok := dl.ancientReceipts[hash]; ok { + return true + } + _, ok := dl.ownReceipts[hash] + return ok +} + +// GetHeader retrieves a header from the testers canonical chain. +func (dl *downloadTester) GetHeaderByHash(hash common.Hash) *types.Header { + dl.lock.RLock() + defer dl.lock.RUnlock() + return dl.getHeaderByHash(hash) +} + +// getHeaderByHash returns the header if found either within ancients or own blocks) +// This method assumes that the caller holds at least the read-lock (dl.lock) +func (dl *downloadTester) getHeaderByHash(hash common.Hash) *types.Header { + header := dl.ancientHeaders[hash] + if header != nil { + return header + } + return dl.ownHeaders[hash] +} + +// GetBlock retrieves a block from the testers canonical chain. +func (dl *downloadTester) GetBlockByHash(hash common.Hash) *types.Block { + dl.lock.RLock() + defer dl.lock.RUnlock() + + block := dl.ancientBlocks[hash] + if block != nil { + return block + } + return dl.ownBlocks[hash] +} + +// CurrentHeader retrieves the current head header from the canonical chain. +func (dl *downloadTester) CurrentHeader() *types.Header { + dl.lock.RLock() + defer dl.lock.RUnlock() + + for i := len(dl.ownHashes) - 1; i >= 0; i-- { + if header := dl.ancientHeaders[dl.ownHashes[i]]; header != nil { + return header + } + if header := dl.ownHeaders[dl.ownHashes[i]]; header != nil { + return header + } + } + return dl.genesis.Header() +} + +// CurrentBlock retrieves the current head block from the canonical chain. +func (dl *downloadTester) CurrentBlock() *types.Block { + dl.lock.RLock() + defer dl.lock.RUnlock() + + for i := len(dl.ownHashes) - 1; i >= 0; i-- { + if block := dl.ancientBlocks[dl.ownHashes[i]]; block != nil { + if _, err := dl.stateDb.Get(block.Root().Bytes()); err == nil { + return block + } + return block + } + if block := dl.ownBlocks[dl.ownHashes[i]]; block != nil { + if _, err := dl.stateDb.Get(block.Root().Bytes()); err == nil { + return block + } + } + } + return dl.genesis +} + +// CurrentFastBlock retrieves the current head fast-sync block from the canonical chain. +func (dl *downloadTester) CurrentFastBlock() *types.Block { + dl.lock.RLock() + defer dl.lock.RUnlock() + + for i := len(dl.ownHashes) - 1; i >= 0; i-- { + if block := dl.ancientBlocks[dl.ownHashes[i]]; block != nil { + return block + } + if block := dl.ownBlocks[dl.ownHashes[i]]; block != nil { + return block + } + } + return dl.genesis +} + +// FastSyncCommitHead manually sets the head block to a given hash. +func (dl *downloadTester) FastSyncCommitHead(hash common.Hash) error { + // For now only check that the state trie is correct + if block := dl.GetBlockByHash(hash); block != nil { + _, err := trie.NewSecure(block.Root(), trie.NewDatabase(dl.stateDb)) + return err + } + return fmt.Errorf("non existent block: %x", hash[:4]) +} + +// GetTd retrieves the block's total difficulty from the canonical chain. +func (dl *downloadTester) GetTd(hash common.Hash, number uint64) *big.Int { + dl.lock.RLock() + defer dl.lock.RUnlock() + + return dl.getTd(hash) +} + +// getTd retrieves the block's total difficulty if found either within +// ancients or own blocks). +// This method assumes that the caller holds at least the read-lock (dl.lock) +func (dl *downloadTester) getTd(hash common.Hash) *big.Int { + if td := dl.ancientChainTd[hash]; td != nil { + return td + } + return dl.ownChainTd[hash] +} + +// InsertHeaderChain injects a new batch of headers into the simulated chain. +func (dl *downloadTester) InsertHeaderChain(headers []*types.Header, checkFreq int) (i int, err error) { + dl.lock.Lock() + defer dl.lock.Unlock() + // Do a quick check, as the blockchain.InsertHeaderChain doesn't insert anything in case of errors + if dl.getHeaderByHash(headers[0].ParentHash) == nil { + return 0, fmt.Errorf("InsertHeaderChain: unknown parent at first position, parent of number %d", headers[0].Number) + } + var hashes []common.Hash + for i := 1; i < len(headers); i++ { + hash := headers[i-1].Hash() + if headers[i].ParentHash != headers[i-1].Hash() { + return i, fmt.Errorf("non-contiguous import at position %d", i) + } + hashes = append(hashes, hash) + } + hashes = append(hashes, headers[len(headers)-1].Hash()) + // Do a full insert if pre-checks passed + for i, header := range headers { + hash := hashes[i] + if dl.getHeaderByHash(hash) != nil { + continue + } + if dl.getHeaderByHash(header.ParentHash) == nil { + // This _should_ be impossible, due to precheck and induction + return i, fmt.Errorf("InsertHeaderChain: unknown parent at position %d", i) + } + dl.ownHashes = append(dl.ownHashes, hash) + dl.ownHeaders[hash] = header + + td := dl.getTd(header.ParentHash) + dl.ownChainTd[hash] = new(big.Int).Add(td, header.Difficulty) + } + return len(headers), nil +} + +// InsertChain injects a new batch of blocks into the simulated chain. +func (dl *downloadTester) InsertChain(blocks types.Blocks) (i int, err error) { + dl.lock.Lock() + defer dl.lock.Unlock() + for i, block := range blocks { + if parent, ok := dl.ownBlocks[block.ParentHash()]; !ok { + return i, fmt.Errorf("InsertChain: unknown parent at position %d / %d", i, len(blocks)) + } else if _, err := dl.stateDb.Get(parent.Root().Bytes()); err != nil { + return i, fmt.Errorf("InsertChain: unknown parent state %x: %v", parent.Root(), err) + } + if hdr := dl.getHeaderByHash(block.Hash()); hdr == nil { + dl.ownHashes = append(dl.ownHashes, block.Hash()) + dl.ownHeaders[block.Hash()] = block.Header() + } + dl.ownBlocks[block.Hash()] = block + dl.ownReceipts[block.Hash()] = make(types.Receipts, 0) + dl.stateDb.Put(block.Root().Bytes(), []byte{0x00}) + td := dl.getTd(block.ParentHash()) + dl.ownChainTd[block.Hash()] = new(big.Int).Add(td, block.Difficulty()) + } + return len(blocks), nil +} + +// InsertReceiptChain injects a new batch of receipts into the simulated chain. +func (dl *downloadTester) InsertReceiptChain(blocks types.Blocks, receipts []types.Receipts, ancientLimit uint64) (i int, err error) { + dl.lock.Lock() + defer dl.lock.Unlock() + + for i := 0; i < len(blocks) && i < len(receipts); i++ { + if _, ok := dl.ownHeaders[blocks[i].Hash()]; !ok { + return i, errors.New("unknown owner") + } + if _, ok := dl.ancientBlocks[blocks[i].ParentHash()]; !ok { + if _, ok := dl.ownBlocks[blocks[i].ParentHash()]; !ok { + return i, errors.New("InsertReceiptChain: unknown parent") + } + } + if blocks[i].NumberU64() <= ancientLimit { + dl.ancientBlocks[blocks[i].Hash()] = blocks[i] + dl.ancientReceipts[blocks[i].Hash()] = receipts[i] + + // Migrate from active db to ancient db + dl.ancientHeaders[blocks[i].Hash()] = blocks[i].Header() + dl.ancientChainTd[blocks[i].Hash()] = new(big.Int).Add(dl.ancientChainTd[blocks[i].ParentHash()], blocks[i].Difficulty()) + delete(dl.ownHeaders, blocks[i].Hash()) + delete(dl.ownChainTd, blocks[i].Hash()) + } else { + dl.ownBlocks[blocks[i].Hash()] = blocks[i] + dl.ownReceipts[blocks[i].Hash()] = receipts[i] + } + } + return len(blocks), nil +} + +// SetHead rewinds the local chain to a new head. +func (dl *downloadTester) SetHead(head uint64) error { + dl.lock.Lock() + defer dl.lock.Unlock() + + // Find the hash of the head to reset to + var hash common.Hash + for h, header := range dl.ownHeaders { + if header.Number.Uint64() == head { + hash = h + } + } + for h, header := range dl.ancientHeaders { + if header.Number.Uint64() == head { + hash = h + } + } + if hash == (common.Hash{}) { + return fmt.Errorf("unknown head to set: %d", head) + } + // Find the offset in the header chain + var offset int + for o, h := range dl.ownHashes { + if h == hash { + offset = o + break + } + } + // Remove all the hashes and associated data afterwards + for i := offset + 1; i < len(dl.ownHashes); i++ { + delete(dl.ownChainTd, dl.ownHashes[i]) + delete(dl.ownHeaders, dl.ownHashes[i]) + delete(dl.ownReceipts, dl.ownHashes[i]) + delete(dl.ownBlocks, dl.ownHashes[i]) + + delete(dl.ancientChainTd, dl.ownHashes[i]) + delete(dl.ancientHeaders, dl.ownHashes[i]) + delete(dl.ancientReceipts, dl.ownHashes[i]) + delete(dl.ancientBlocks, dl.ownHashes[i]) + } + dl.ownHashes = dl.ownHashes[:offset+1] + return nil +} + +// Rollback removes some recently added elements from the chain. +func (dl *downloadTester) Rollback(hashes []common.Hash) { +} + +// newPeer registers a new block download source into the downloader. +func (dl *downloadTester) newPeer(id string, version uint, chain *testChain) error { + dl.lock.Lock() + defer dl.lock.Unlock() + + peer := &downloadTesterPeer{dl: dl, id: id, chain: chain} + dl.peers[id] = peer + return dl.downloader.RegisterPeer(id, version, peer) +} + +// dropPeer simulates a hard peer removal from the connection pool. +func (dl *downloadTester) dropPeer(id string) { + dl.lock.Lock() + defer dl.lock.Unlock() + + delete(dl.peers, id) + dl.downloader.UnregisterPeer(id) +} + +// Snapshots implements the BlockChain interface for the downloader, but is a noop. +func (dl *downloadTester) Snapshots() *snapshot.Tree { + return nil +} + +type downloadTesterPeer struct { + dl *downloadTester + id string + chain *testChain + missingStates map[common.Hash]bool // State entries that fast sync should not return +} + +// Head constructs a function to retrieve a peer's current head hash +// and total difficulty. +func (dlp *downloadTesterPeer) Head() (common.Hash, *big.Int) { + b := dlp.chain.headBlock() + return b.Hash(), dlp.chain.td(b.Hash()) +} + +// RequestHeadersByHash constructs a GetBlockHeaders function based on a hashed +// origin; associated with a particular peer in the download tester. The returned +// function can be used to retrieve batches of headers from the particular peer. +func (dlp *downloadTesterPeer) RequestHeadersByHash(origin common.Hash, amount int, skip int, reverse bool) error { + result := dlp.chain.headersByHash(origin, amount, skip, reverse) + go dlp.dl.downloader.DeliverHeaders(dlp.id, result) + return nil +} + +// RequestHeadersByNumber constructs a GetBlockHeaders function based on a numbered +// origin; associated with a particular peer in the download tester. The returned +// function can be used to retrieve batches of headers from the particular peer. +func (dlp *downloadTesterPeer) RequestHeadersByNumber(origin uint64, amount int, skip int, reverse bool) error { + result := dlp.chain.headersByNumber(origin, amount, skip, reverse) + go dlp.dl.downloader.DeliverHeaders(dlp.id, result) + return nil +} + +// RequestBodies constructs a getBlockBodies method associated with a particular +// peer in the download tester. The returned function can be used to retrieve +// batches of block bodies from the particularly requested peer. +func (dlp *downloadTesterPeer) RequestBodies(hashes []common.Hash) error { + txs, uncles := dlp.chain.bodies(hashes) + go dlp.dl.downloader.DeliverBodies(dlp.id, txs, uncles) + return nil +} + +// RequestReceipts constructs a getReceipts method associated with a particular +// peer in the download tester. The returned function can be used to retrieve +// batches of block receipts from the particularly requested peer. +func (dlp *downloadTesterPeer) RequestReceipts(hashes []common.Hash) error { + receipts := dlp.chain.receipts(hashes) + go dlp.dl.downloader.DeliverReceipts(dlp.id, receipts) + return nil +} + +// RequestNodeData constructs a getNodeData method associated with a particular +// peer in the download tester. The returned function can be used to retrieve +// batches of node state data from the particularly requested peer. +func (dlp *downloadTesterPeer) RequestNodeData(hashes []common.Hash) error { + dlp.dl.lock.RLock() + defer dlp.dl.lock.RUnlock() + + results := make([][]byte, 0, len(hashes)) + for _, hash := range hashes { + if data, err := dlp.dl.peerDb.Get(hash.Bytes()); err == nil { + if !dlp.missingStates[hash] { + results = append(results, data) + } + } + } + go dlp.dl.downloader.DeliverNodeData(dlp.id, results) + return nil +} + +// assertOwnChain checks if the local chain contains the correct number of items +// of the various chain components. +func assertOwnChain(t *testing.T, tester *downloadTester, length int) { + // Mark this method as a helper to report errors at callsite, not in here + t.Helper() + + assertOwnForkedChain(t, tester, 1, []int{length}) +} + +// assertOwnForkedChain checks if the local forked chain contains the correct +// number of items of the various chain components. +func assertOwnForkedChain(t *testing.T, tester *downloadTester, common int, lengths []int) { + // Mark this method as a helper to report errors at callsite, not in here + t.Helper() + + // Initialize the counters for the first fork + headers, blocks, receipts := lengths[0], lengths[0], lengths[0] + + // Update the counters for each subsequent fork + for _, length := range lengths[1:] { + headers += length - common + blocks += length - common + receipts += length - common + } + if tester.downloader.getMode() == LightSync { + blocks, receipts = 1, 1 + } + if hs := len(tester.ownHeaders) + len(tester.ancientHeaders) - 1; hs != headers { + t.Fatalf("synchronised headers mismatch: have %v, want %v", hs, headers) + } + if bs := len(tester.ownBlocks) + len(tester.ancientBlocks) - 1; bs != blocks { + t.Fatalf("synchronised blocks mismatch: have %v, want %v", bs, blocks) + } + if rs := len(tester.ownReceipts) + len(tester.ancientReceipts) - 1; rs != receipts { + t.Fatalf("synchronised receipts mismatch: have %v, want %v", rs, receipts) + } +} + +func TestCanonicalSynchronisation66Full(t *testing.T) { testCanonSync(t, eth.ETH66, FullSync) } +func TestCanonicalSynchronisation66Fast(t *testing.T) { testCanonSync(t, eth.ETH66, FastSync) } +func TestCanonicalSynchronisation66Light(t *testing.T) { testCanonSync(t, eth.ETH66, LightSync) } + +func testCanonSync(t *testing.T, protocol uint, mode SyncMode) { + t.Parallel() + + tester := newTester() + defer tester.terminate() + + // Create a small enough block chain to download + chain := testChainBase.shorten(blockCacheMaxItems - 15) + tester.newPeer("peer", protocol, chain) + + // Synchronise with the peer and make sure all relevant data was retrieved + if err := tester.sync("peer", nil, mode); err != nil { + t.Fatalf("failed to synchronise blocks: %v", err) + } + assertOwnChain(t, tester, chain.len()) +} + +// Tests that if a large batch of blocks are being downloaded, it is throttled +// until the cached blocks are retrieved. +func TestThrottling66Full(t *testing.T) { testThrottling(t, eth.ETH66, FullSync) } +func TestThrottling66Fast(t *testing.T) { testThrottling(t, eth.ETH66, FastSync) } + +func testThrottling(t *testing.T, protocol uint, mode SyncMode) { + t.Parallel() + tester := newTester() + + // Create a long block chain to download and the tester + targetBlocks := testChainBase.len() - 1 + tester.newPeer("peer", protocol, testChainBase) + + // Wrap the importer to allow stepping + blocked, proceed := uint32(0), make(chan struct{}) + tester.downloader.chainInsertHook = func(results []*fetchResult) { + atomic.StoreUint32(&blocked, uint32(len(results))) + <-proceed + } + // Start a synchronisation concurrently + errc := make(chan error, 1) + go func() { + errc <- tester.sync("peer", nil, mode) + }() + // Iteratively take some blocks, always checking the retrieval count + for { + // Check the retrieval count synchronously (! reason for this ugly block) + tester.lock.RLock() + retrieved := len(tester.ownBlocks) + tester.lock.RUnlock() + if retrieved >= targetBlocks+1 { + break + } + // Wait a bit for sync to throttle itself + var cached, frozen int + for start := time.Now(); time.Since(start) < 3*time.Second; { + time.Sleep(25 * time.Millisecond) + + tester.lock.Lock() + tester.downloader.queue.lock.Lock() + tester.downloader.queue.resultCache.lock.Lock() + { + cached = tester.downloader.queue.resultCache.countCompleted() + frozen = int(atomic.LoadUint32(&blocked)) + retrieved = len(tester.ownBlocks) + } + tester.downloader.queue.resultCache.lock.Unlock() + tester.downloader.queue.lock.Unlock() + tester.lock.Unlock() + + if cached == blockCacheMaxItems || + cached == blockCacheMaxItems-reorgProtHeaderDelay || + retrieved+cached+frozen == targetBlocks+1 || + retrieved+cached+frozen == targetBlocks+1-reorgProtHeaderDelay { + break + } + } + // Make sure we filled up the cache, then exhaust it + time.Sleep(25 * time.Millisecond) // give it a chance to screw up + tester.lock.RLock() + retrieved = len(tester.ownBlocks) + tester.lock.RUnlock() + if cached != blockCacheMaxItems && cached != blockCacheMaxItems-reorgProtHeaderDelay && retrieved+cached+frozen != targetBlocks+1 && retrieved+cached+frozen != targetBlocks+1-reorgProtHeaderDelay { + t.Fatalf("block count mismatch: have %v, want %v (owned %v, blocked %v, target %v)", cached, blockCacheMaxItems, retrieved, frozen, targetBlocks+1) + } + + // Permit the blocked blocks to import + if atomic.LoadUint32(&blocked) > 0 { + atomic.StoreUint32(&blocked, uint32(0)) + proceed <- struct{}{} + } + } + // Check that we haven't pulled more blocks than available + assertOwnChain(t, tester, targetBlocks+1) + if err := <-errc; err != nil { + t.Fatalf("block synchronization failed: %v", err) + } + tester.terminate() + +} + +// Tests that simple synchronization against a forked chain works correctly. In +// this test common ancestor lookup should *not* be short circuited, and a full +// binary search should be executed. +func TestForkedSync66Full(t *testing.T) { testForkedSync(t, eth.ETH66, FullSync) } +func TestForkedSync66Fast(t *testing.T) { testForkedSync(t, eth.ETH66, FastSync) } +func TestForkedSync66Light(t *testing.T) { testForkedSync(t, eth.ETH66, LightSync) } + +func testForkedSync(t *testing.T, protocol uint, mode SyncMode) { + t.Parallel() + + tester := newTester() + defer tester.terminate() + + chainA := testChainForkLightA.shorten(testChainBase.len() + 80) + chainB := testChainForkLightB.shorten(testChainBase.len() + 80) + tester.newPeer("fork A", protocol, chainA) + tester.newPeer("fork B", protocol, chainB) + // Synchronise with the peer and make sure all blocks were retrieved + if err := tester.sync("fork A", nil, mode); err != nil { + t.Fatalf("failed to synchronise blocks: %v", err) + } + assertOwnChain(t, tester, chainA.len()) + + // Synchronise with the second peer and make sure that fork is pulled too + if err := tester.sync("fork B", nil, mode); err != nil { + t.Fatalf("failed to synchronise blocks: %v", err) + } + assertOwnForkedChain(t, tester, testChainBase.len(), []int{chainA.len(), chainB.len()}) +} + +// Tests that synchronising against a much shorter but much heavyer fork works +// corrently and is not dropped. +func TestHeavyForkedSync66Full(t *testing.T) { testHeavyForkedSync(t, eth.ETH66, FullSync) } +func TestHeavyForkedSync66Fast(t *testing.T) { testHeavyForkedSync(t, eth.ETH66, FastSync) } +func TestHeavyForkedSync66Light(t *testing.T) { testHeavyForkedSync(t, eth.ETH66, LightSync) } + +func testHeavyForkedSync(t *testing.T, protocol uint, mode SyncMode) { + t.Parallel() + + tester := newTester() + defer tester.terminate() + + chainA := testChainForkLightA.shorten(testChainBase.len() + 80) + chainB := testChainForkHeavy.shorten(testChainBase.len() + 80) + tester.newPeer("light", protocol, chainA) + tester.newPeer("heavy", protocol, chainB) + + // Synchronise with the peer and make sure all blocks were retrieved + if err := tester.sync("light", nil, mode); err != nil { + t.Fatalf("failed to synchronise blocks: %v", err) + } + assertOwnChain(t, tester, chainA.len()) + + // Synchronise with the second peer and make sure that fork is pulled too + if err := tester.sync("heavy", nil, mode); err != nil { + t.Fatalf("failed to synchronise blocks: %v", err) + } + assertOwnForkedChain(t, tester, testChainBase.len(), []int{chainA.len(), chainB.len()}) +} + +// Tests that chain forks are contained within a certain interval of the current +// chain head, ensuring that malicious peers cannot waste resources by feeding +// long dead chains. +func TestBoundedForkedSync66Full(t *testing.T) { testBoundedForkedSync(t, eth.ETH66, FullSync) } +func TestBoundedForkedSync66Fast(t *testing.T) { testBoundedForkedSync(t, eth.ETH66, FastSync) } +func TestBoundedForkedSync66Light(t *testing.T) { testBoundedForkedSync(t, eth.ETH66, LightSync) } + +func testBoundedForkedSync(t *testing.T, protocol uint, mode SyncMode) { + t.Parallel() + + tester := newTester() + defer tester.terminate() + + chainA := testChainForkLightA + chainB := testChainForkLightB + tester.newPeer("original", protocol, chainA) + tester.newPeer("rewriter", protocol, chainB) + + // Synchronise with the peer and make sure all blocks were retrieved + if err := tester.sync("original", nil, mode); err != nil { + t.Fatalf("failed to synchronise blocks: %v", err) + } + assertOwnChain(t, tester, chainA.len()) + + // Synchronise with the second peer and ensure that the fork is rejected to being too old + if err := tester.sync("rewriter", nil, mode); err != errInvalidAncestor { + t.Fatalf("sync failure mismatch: have %v, want %v", err, errInvalidAncestor) + } +} + +// Tests that chain forks are contained within a certain interval of the current +// chain head for short but heavy forks too. These are a bit special because they +// take different ancestor lookup paths. +func TestBoundedHeavyForkedSync66Full(t *testing.T) { + testBoundedHeavyForkedSync(t, eth.ETH66, FullSync) +} +func TestBoundedHeavyForkedSync66Fast(t *testing.T) { + testBoundedHeavyForkedSync(t, eth.ETH66, FastSync) +} +func TestBoundedHeavyForkedSync66Light(t *testing.T) { + testBoundedHeavyForkedSync(t, eth.ETH66, LightSync) +} + +func testBoundedHeavyForkedSync(t *testing.T, protocol uint, mode SyncMode) { + t.Parallel() + tester := newTester() + + // Create a long enough forked chain + chainA := testChainForkLightA + chainB := testChainForkHeavy + tester.newPeer("original", protocol, chainA) + + // Synchronise with the peer and make sure all blocks were retrieved + if err := tester.sync("original", nil, mode); err != nil { + t.Fatalf("failed to synchronise blocks: %v", err) + } + assertOwnChain(t, tester, chainA.len()) + + tester.newPeer("heavy-rewriter", protocol, chainB) + // Synchronise with the second peer and ensure that the fork is rejected to being too old + if err := tester.sync("heavy-rewriter", nil, mode); err != errInvalidAncestor { + t.Fatalf("sync failure mismatch: have %v, want %v", err, errInvalidAncestor) + } + tester.terminate() +} + +// Tests that an inactive downloader will not accept incoming block headers, +// bodies and receipts. +func TestInactiveDownloader63(t *testing.T) { + t.Parallel() + + tester := newTester() + defer tester.terminate() + + // Check that neither block headers nor bodies are accepted + if err := tester.downloader.DeliverHeaders("bad peer", []*types.Header{}); err != errNoSyncActive { + t.Errorf("error mismatch: have %v, want %v", err, errNoSyncActive) + } + if err := tester.downloader.DeliverBodies("bad peer", [][]*types.Transaction{}, [][]*types.Header{}); err != errNoSyncActive { + t.Errorf("error mismatch: have %v, want %v", err, errNoSyncActive) + } + if err := tester.downloader.DeliverReceipts("bad peer", [][]*types.Receipt{}); err != errNoSyncActive { + t.Errorf("error mismatch: have %v, want %v", err, errNoSyncActive) + } +} + +// Tests that a canceled download wipes all previously accumulated state. +func TestCancel66Full(t *testing.T) { testCancel(t, eth.ETH66, FullSync) } +func TestCancel66Fast(t *testing.T) { testCancel(t, eth.ETH66, FastSync) } +func TestCancel66Light(t *testing.T) { testCancel(t, eth.ETH66, LightSync) } + +func testCancel(t *testing.T, protocol uint, mode SyncMode) { + t.Parallel() + + tester := newTester() + defer tester.terminate() + + chain := testChainBase.shorten(MaxHeaderFetch) + tester.newPeer("peer", protocol, chain) + + // Make sure canceling works with a pristine downloader + tester.downloader.Cancel() + if !tester.downloader.queue.Idle() { + t.Errorf("download queue not idle") + } + // Synchronise with the peer, but cancel afterwards + if err := tester.sync("peer", nil, mode); err != nil { + t.Fatalf("failed to synchronise blocks: %v", err) + } + tester.downloader.Cancel() + if !tester.downloader.queue.Idle() { + t.Errorf("download queue not idle") + } +} + +// Tests that synchronisation from multiple peers works as intended (multi thread sanity test). +func TestMultiSynchronisation66Full(t *testing.T) { testMultiSynchronisation(t, eth.ETH66, FullSync) } +func TestMultiSynchronisation66Fast(t *testing.T) { testMultiSynchronisation(t, eth.ETH66, FastSync) } +func TestMultiSynchronisation66Light(t *testing.T) { testMultiSynchronisation(t, eth.ETH66, LightSync) } + +func testMultiSynchronisation(t *testing.T, protocol uint, mode SyncMode) { + t.Parallel() + + tester := newTester() + defer tester.terminate() + + // Create various peers with various parts of the chain + targetPeers := 8 + chain := testChainBase.shorten(targetPeers * 100) + + for i := 0; i < targetPeers; i++ { + id := fmt.Sprintf("peer #%d", i) + tester.newPeer(id, protocol, chain.shorten(chain.len()/(i+1))) + } + if err := tester.sync("peer #0", nil, mode); err != nil { + t.Fatalf("failed to synchronise blocks: %v", err) + } + assertOwnChain(t, tester, chain.len()) +} + +// Tests that synchronisations behave well in multi-version protocol environments +// and not wreak havoc on other nodes in the network. +func TestMultiProtoSynchronisation66Full(t *testing.T) { testMultiProtoSync(t, eth.ETH66, FullSync) } +func TestMultiProtoSynchronisation66Fast(t *testing.T) { testMultiProtoSync(t, eth.ETH66, FastSync) } +func TestMultiProtoSynchronisation66Light(t *testing.T) { testMultiProtoSync(t, eth.ETH66, LightSync) } + +func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { + t.Parallel() + + tester := newTester() + defer tester.terminate() + + // Create a small enough block chain to download + chain := testChainBase.shorten(blockCacheMaxItems - 15) + + // Create peers of every type + tester.newPeer("peer 66", eth.ETH66, chain) + //tester.newPeer("peer 65", eth.ETH67, chain) + + // Synchronise with the requested peer and make sure all blocks were retrieved + if err := tester.sync(fmt.Sprintf("peer %d", protocol), nil, mode); err != nil { + t.Fatalf("failed to synchronise blocks: %v", err) + } + assertOwnChain(t, tester, chain.len()) + + // Check that no peers have been dropped off + for _, version := range []int{66} { + peer := fmt.Sprintf("peer %d", version) + if _, ok := tester.peers[peer]; !ok { + t.Errorf("%s dropped", peer) + } + } +} + +// Tests that if a block is empty (e.g. header only), no body request should be +// made, and instead the header should be assembled into a whole block in itself. +func TestEmptyShortCircuit66Full(t *testing.T) { testEmptyShortCircuit(t, eth.ETH66, FullSync) } +func TestEmptyShortCircuit66Fast(t *testing.T) { testEmptyShortCircuit(t, eth.ETH66, FastSync) } +func TestEmptyShortCircuit66Light(t *testing.T) { testEmptyShortCircuit(t, eth.ETH66, LightSync) } + +func testEmptyShortCircuit(t *testing.T, protocol uint, mode SyncMode) { + t.Parallel() + + tester := newTester() + defer tester.terminate() + + // Create a block chain to download + chain := testChainBase + tester.newPeer("peer", protocol, chain) + + // Instrument the downloader to signal body requests + bodiesHave, receiptsHave := int32(0), int32(0) + tester.downloader.bodyFetchHook = func(headers []*types.Header) { + atomic.AddInt32(&bodiesHave, int32(len(headers))) + } + tester.downloader.receiptFetchHook = func(headers []*types.Header) { + atomic.AddInt32(&receiptsHave, int32(len(headers))) + } + // Synchronise with the peer and make sure all blocks were retrieved + if err := tester.sync("peer", nil, mode); err != nil { + t.Fatalf("failed to synchronise blocks: %v", err) + } + assertOwnChain(t, tester, chain.len()) + + // Validate the number of block bodies that should have been requested + bodiesNeeded, receiptsNeeded := 0, 0 + for _, block := range chain.blockm { + if mode != LightSync && block != tester.genesis && (len(block.Transactions()) > 0 || len(block.Uncles()) > 0) { + bodiesNeeded++ + } + } + for _, receipt := range chain.receiptm { + if mode == FastSync && len(receipt) > 0 { + receiptsNeeded++ + } + } + if int(bodiesHave) != bodiesNeeded { + t.Errorf("body retrieval count mismatch: have %v, want %v", bodiesHave, bodiesNeeded) + } + if int(receiptsHave) != receiptsNeeded { + t.Errorf("receipt retrieval count mismatch: have %v, want %v", receiptsHave, receiptsNeeded) + } +} + +// Tests that headers are enqueued continuously, preventing malicious nodes from +// stalling the downloader by feeding gapped header chains. +func TestMissingHeaderAttack66Full(t *testing.T) { testMissingHeaderAttack(t, eth.ETH66, FullSync) } +func TestMissingHeaderAttack66Fast(t *testing.T) { testMissingHeaderAttack(t, eth.ETH66, FastSync) } +func TestMissingHeaderAttack66Light(t *testing.T) { testMissingHeaderAttack(t, eth.ETH66, LightSync) } + +func testMissingHeaderAttack(t *testing.T, protocol uint, mode SyncMode) { + t.Parallel() + + tester := newTester() + defer tester.terminate() + + chain := testChainBase.shorten(blockCacheMaxItems - 15) + brokenChain := chain.shorten(chain.len()) + delete(brokenChain.headerm, brokenChain.chain[brokenChain.len()/2]) + tester.newPeer("attack", protocol, brokenChain) + + if err := tester.sync("attack", nil, mode); err == nil { + t.Fatalf("succeeded attacker synchronisation") + } + // Synchronise with the valid peer and make sure sync succeeds + tester.newPeer("valid", protocol, chain) + if err := tester.sync("valid", nil, mode); err != nil { + t.Fatalf("failed to synchronise blocks: %v", err) + } + assertOwnChain(t, tester, chain.len()) +} + +// Tests that if requested headers are shifted (i.e. first is missing), the queue +// detects the invalid numbering. +func TestShiftedHeaderAttack66Full(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH66, FullSync) } +func TestShiftedHeaderAttack66Fast(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH66, FastSync) } +func TestShiftedHeaderAttack66Light(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH66, LightSync) } + +func testShiftedHeaderAttack(t *testing.T, protocol uint, mode SyncMode) { + t.Parallel() + + tester := newTester() + defer tester.terminate() + + chain := testChainBase.shorten(blockCacheMaxItems - 15) + + // Attempt a full sync with an attacker feeding shifted headers + brokenChain := chain.shorten(chain.len()) + delete(brokenChain.headerm, brokenChain.chain[1]) + delete(brokenChain.blockm, brokenChain.chain[1]) + delete(brokenChain.receiptm, brokenChain.chain[1]) + tester.newPeer("attack", protocol, brokenChain) + if err := tester.sync("attack", nil, mode); err == nil { + t.Fatalf("succeeded attacker synchronisation") + } + + // Synchronise with the valid peer and make sure sync succeeds + tester.newPeer("valid", protocol, chain) + if err := tester.sync("valid", nil, mode); err != nil { + t.Fatalf("failed to synchronise blocks: %v", err) + } + assertOwnChain(t, tester, chain.len()) +} + +// Tests that upon detecting an invalid header, the recent ones are rolled back +// for various failure scenarios. Afterwards a full sync is attempted to make +// sure no state was corrupted. +func TestInvalidHeaderRollback66Fast(t *testing.T) { testInvalidHeaderRollback(t, eth.ETH66, FastSync) } + +func testInvalidHeaderRollback(t *testing.T, protocol uint, mode SyncMode) { + t.Parallel() + + tester := newTester() + + // Create a small enough block chain to download + targetBlocks := 3*fsHeaderSafetyNet + 256 + fsMinFullBlocks + chain := testChainBase.shorten(targetBlocks) + + // Attempt to sync with an attacker that feeds junk during the fast sync phase. + // This should result in the last fsHeaderSafetyNet headers being rolled back. + missing := fsHeaderSafetyNet + MaxHeaderFetch + 1 + fastAttackChain := chain.shorten(chain.len()) + delete(fastAttackChain.headerm, fastAttackChain.chain[missing]) + tester.newPeer("fast-attack", protocol, fastAttackChain) + + if err := tester.sync("fast-attack", nil, mode); err == nil { + t.Fatalf("succeeded fast attacker synchronisation") + } + if head := tester.CurrentHeader().Number.Int64(); int(head) > MaxHeaderFetch { + t.Errorf("rollback head mismatch: have %v, want at most %v", head, MaxHeaderFetch) + } + + // Attempt to sync with an attacker that feeds junk during the block import phase. + // This should result in both the last fsHeaderSafetyNet number of headers being + // rolled back, and also the pivot point being reverted to a non-block status. + missing = 3*fsHeaderSafetyNet + MaxHeaderFetch + 1 + blockAttackChain := chain.shorten(chain.len()) + delete(fastAttackChain.headerm, fastAttackChain.chain[missing]) // Make sure the fast-attacker doesn't fill in + delete(blockAttackChain.headerm, blockAttackChain.chain[missing]) + tester.newPeer("block-attack", protocol, blockAttackChain) + + if err := tester.sync("block-attack", nil, mode); err == nil { + t.Fatalf("succeeded block attacker synchronisation") + } + if head := tester.CurrentHeader().Number.Int64(); int(head) > 2*fsHeaderSafetyNet+MaxHeaderFetch { + t.Errorf("rollback head mismatch: have %v, want at most %v", head, 2*fsHeaderSafetyNet+MaxHeaderFetch) + } + if mode == FastSync { + if head := tester.CurrentBlock().NumberU64(); head != 0 { + t.Errorf("fast sync pivot block #%d not rolled back", head) + } + } + + // Attempt to sync with an attacker that withholds promised blocks after the + // fast sync pivot point. This could be a trial to leave the node with a bad + // but already imported pivot block. + withholdAttackChain := chain.shorten(chain.len()) + tester.newPeer("withhold-attack", protocol, withholdAttackChain) + tester.downloader.syncInitHook = func(uint64, uint64) { + for i := missing; i < withholdAttackChain.len(); i++ { + delete(withholdAttackChain.headerm, withholdAttackChain.chain[i]) + } + tester.downloader.syncInitHook = nil + } + if err := tester.sync("withhold-attack", nil, mode); err == nil { + t.Fatalf("succeeded withholding attacker synchronisation") + } + if head := tester.CurrentHeader().Number.Int64(); int(head) > 2*fsHeaderSafetyNet+MaxHeaderFetch { + t.Errorf("rollback head mismatch: have %v, want at most %v", head, 2*fsHeaderSafetyNet+MaxHeaderFetch) + } + if mode == FastSync { + if head := tester.CurrentBlock().NumberU64(); head != 0 { + t.Errorf("fast sync pivot block #%d not rolled back", head) + } + } + + // synchronise with the valid peer and make sure sync succeeds. Since the last rollback + // should also disable fast syncing for this process, verify that we did a fresh full + // sync. Note, we can't assert anything about the receipts since we won't purge the + // database of them, hence we can't use assertOwnChain. + tester.newPeer("valid", protocol, chain) + if err := tester.sync("valid", nil, mode); err != nil { + t.Fatalf("failed to synchronise blocks: %v", err) + } + if hs := len(tester.ownHeaders); hs != chain.len() { + t.Fatalf("synchronised headers mismatch: have %v, want %v", hs, chain.len()) + } + if mode != LightSync { + if bs := len(tester.ownBlocks); bs != chain.len() { + t.Fatalf("synchronised blocks mismatch: have %v, want %v", bs, chain.len()) + } + } + tester.terminate() +} + +// Tests that a peer advertising a high TD doesn't get to stall the downloader +// afterwards by not sending any useful hashes. +func TestHighTDStarvationAttack66Full(t *testing.T) { + testHighTDStarvationAttack(t, eth.ETH66, FullSync) +} +func TestHighTDStarvationAttack66Fast(t *testing.T) { + testHighTDStarvationAttack(t, eth.ETH66, FastSync) +} +func TestHighTDStarvationAttack66Light(t *testing.T) { + testHighTDStarvationAttack(t, eth.ETH66, LightSync) +} + +func testHighTDStarvationAttack(t *testing.T, protocol uint, mode SyncMode) { + t.Parallel() + + tester := newTester() + + chain := testChainBase.shorten(1) + tester.newPeer("attack", protocol, chain) + if err := tester.sync("attack", big.NewInt(1000000), mode); err != errStallingPeer { + t.Fatalf("synchronisation error mismatch: have %v, want %v", err, errStallingPeer) + } + tester.terminate() +} + +// Tests that misbehaving peers are disconnected, whilst behaving ones are not. +func TestBlockHeaderAttackerDropping66(t *testing.T) { testBlockHeaderAttackerDropping(t, eth.ETH66) } + +func testBlockHeaderAttackerDropping(t *testing.T, protocol uint) { + t.Parallel() + + // Define the disconnection requirement for individual hash fetch errors + tests := []struct { + result error + drop bool + }{ + {nil, false}, // Sync succeeded, all is well + {errBusy, false}, // Sync is already in progress, no problem + {errUnknownPeer, false}, // Peer is unknown, was already dropped, don't double drop + {errBadPeer, true}, // Peer was deemed bad for some reason, drop it + {errStallingPeer, true}, // Peer was detected to be stalling, drop it + {errUnsyncedPeer, true}, // Peer was detected to be unsynced, drop it + {errNoPeers, false}, // No peers to download from, soft race, no issue + {errTimeout, true}, // No hashes received in due time, drop the peer + {errEmptyHeaderSet, true}, // No headers were returned as a response, drop as it's a dead end + {errPeersUnavailable, true}, // Nobody had the advertised blocks, drop the advertiser + {errInvalidAncestor, true}, // Agreed upon ancestor is not acceptable, drop the chain rewriter + {errInvalidChain, true}, // Hash chain was detected as invalid, definitely drop + {errInvalidBody, false}, // A bad peer was detected, but not the sync origin + {errInvalidReceipt, false}, // A bad peer was detected, but not the sync origin + {errCancelContentProcessing, false}, // Synchronisation was canceled, origin may be innocent, don't drop + } + // Run the tests and check disconnection status + tester := newTester() + defer tester.terminate() + chain := testChainBase.shorten(1) + + for i, tt := range tests { + // Register a new peer and ensure its presence + id := fmt.Sprintf("test %d", i) + if err := tester.newPeer(id, protocol, chain); err != nil { + t.Fatalf("test %d: failed to register new peer: %v", i, err) + } + if _, ok := tester.peers[id]; !ok { + t.Fatalf("test %d: registered peer not found", i) + } + // Simulate a synchronisation and check the required result + tester.downloader.synchroniseMock = func(string, common.Hash) error { return tt.result } + + tester.downloader.Synchronise(id, tester.genesis.Hash(), big.NewInt(1000), FullSync) + if _, ok := tester.peers[id]; !ok != tt.drop { + t.Errorf("test %d: peer drop mismatch for %v: have %v, want %v", i, tt.result, !ok, tt.drop) + } + } +} + +// Tests that synchronisation progress (origin block number, current block number +// and highest block number) is tracked and updated correctly. +func TestSyncProgress66Full(t *testing.T) { testSyncProgress(t, eth.ETH66, FullSync) } +func TestSyncProgress66Fast(t *testing.T) { testSyncProgress(t, eth.ETH66, FastSync) } +func TestSyncProgress66Light(t *testing.T) { testSyncProgress(t, eth.ETH66, LightSync) } + +func testSyncProgress(t *testing.T, protocol uint, mode SyncMode) { + t.Parallel() + + tester := newTester() + defer tester.terminate() + chain := testChainBase.shorten(blockCacheMaxItems - 15) + + // Set a sync init hook to catch progress changes + starting := make(chan struct{}) + progress := make(chan struct{}) + + tester.downloader.syncInitHook = func(origin, latest uint64) { + starting <- struct{}{} + <-progress + } + checkProgress(t, tester.downloader, "pristine", ethereum.SyncProgress{}) + + // Synchronise half the blocks and check initial progress + tester.newPeer("peer-half", protocol, chain.shorten(chain.len()/2)) + pending := new(sync.WaitGroup) + pending.Add(1) + + go func() { + defer pending.Done() + if err := tester.sync("peer-half", nil, mode); err != nil { + panic(fmt.Sprintf("failed to synchronise blocks: %v", err)) + } + }() + <-starting + checkProgress(t, tester.downloader, "initial", ethereum.SyncProgress{ + HighestBlock: uint64(chain.len()/2 - 1), + }) + progress <- struct{}{} + pending.Wait() + + // Synchronise all the blocks and check continuation progress + tester.newPeer("peer-full", protocol, chain) + pending.Add(1) + go func() { + defer pending.Done() + if err := tester.sync("peer-full", nil, mode); err != nil { + panic(fmt.Sprintf("failed to synchronise blocks: %v", err)) + } + }() + <-starting + checkProgress(t, tester.downloader, "completing", ethereum.SyncProgress{ + StartingBlock: uint64(chain.len()/2 - 1), + CurrentBlock: uint64(chain.len()/2 - 1), + HighestBlock: uint64(chain.len() - 1), + }) + + // Check final progress after successful sync + progress <- struct{}{} + pending.Wait() + checkProgress(t, tester.downloader, "final", ethereum.SyncProgress{ + StartingBlock: uint64(chain.len()/2 - 1), + CurrentBlock: uint64(chain.len() - 1), + HighestBlock: uint64(chain.len() - 1), + }) +} + +func checkProgress(t *testing.T, d *Downloader, stage string, want ethereum.SyncProgress) { + // Mark this method as a helper to report errors at callsite, not in here + t.Helper() + + p := d.Progress() + p.KnownStates, p.PulledStates = 0, 0 + want.KnownStates, want.PulledStates = 0, 0 + if p != want { + t.Fatalf("%s progress mismatch:\nhave %+v\nwant %+v", stage, p, want) + } +} + +// Tests that synchronisation progress (origin block number and highest block +// number) is tracked and updated correctly in case of a fork (or manual head +// revertal). +func TestForkedSyncProgress66Full(t *testing.T) { testForkedSyncProgress(t, eth.ETH66, FullSync) } +func TestForkedSyncProgress66Fast(t *testing.T) { testForkedSyncProgress(t, eth.ETH66, FastSync) } +func TestForkedSyncProgress66Light(t *testing.T) { testForkedSyncProgress(t, eth.ETH66, LightSync) } + +func testForkedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { + t.Parallel() + + tester := newTester() + defer tester.terminate() + chainA := testChainForkLightA.shorten(testChainBase.len() + MaxHeaderFetch) + chainB := testChainForkLightB.shorten(testChainBase.len() + MaxHeaderFetch) + + // Set a sync init hook to catch progress changes + starting := make(chan struct{}) + progress := make(chan struct{}) + + tester.downloader.syncInitHook = func(origin, latest uint64) { + starting <- struct{}{} + <-progress + } + checkProgress(t, tester.downloader, "pristine", ethereum.SyncProgress{}) + + // Synchronise with one of the forks and check progress + tester.newPeer("fork A", protocol, chainA) + pending := new(sync.WaitGroup) + pending.Add(1) + go func() { + defer pending.Done() + if err := tester.sync("fork A", nil, mode); err != nil { + panic(fmt.Sprintf("failed to synchronise blocks: %v", err)) + } + }() + <-starting + + checkProgress(t, tester.downloader, "initial", ethereum.SyncProgress{ + HighestBlock: uint64(chainA.len() - 1), + }) + progress <- struct{}{} + pending.Wait() + + // Simulate a successful sync above the fork + tester.downloader.syncStatsChainOrigin = tester.downloader.syncStatsChainHeight + + // Synchronise with the second fork and check progress resets + tester.newPeer("fork B", protocol, chainB) + pending.Add(1) + go func() { + defer pending.Done() + if err := tester.sync("fork B", nil, mode); err != nil { + panic(fmt.Sprintf("failed to synchronise blocks: %v", err)) + } + }() + <-starting + checkProgress(t, tester.downloader, "forking", ethereum.SyncProgress{ + StartingBlock: uint64(testChainBase.len()) - 1, + CurrentBlock: uint64(chainA.len() - 1), + HighestBlock: uint64(chainB.len() - 1), + }) + + // Check final progress after successful sync + progress <- struct{}{} + pending.Wait() + checkProgress(t, tester.downloader, "final", ethereum.SyncProgress{ + StartingBlock: uint64(testChainBase.len()) - 1, + CurrentBlock: uint64(chainB.len() - 1), + HighestBlock: uint64(chainB.len() - 1), + }) +} + +// Tests that if synchronisation is aborted due to some failure, then the progress +// origin is not updated in the next sync cycle, as it should be considered the +// continuation of the previous sync and not a new instance. +func TestFailedSyncProgress66Full(t *testing.T) { testFailedSyncProgress(t, eth.ETH66, FullSync) } +func TestFailedSyncProgress66Fast(t *testing.T) { testFailedSyncProgress(t, eth.ETH66, FastSync) } +func TestFailedSyncProgress66Light(t *testing.T) { testFailedSyncProgress(t, eth.ETH66, LightSync) } + +func testFailedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { + t.Parallel() + + tester := newTester() + defer tester.terminate() + chain := testChainBase.shorten(blockCacheMaxItems - 15) + + // Set a sync init hook to catch progress changes + starting := make(chan struct{}) + progress := make(chan struct{}) + + tester.downloader.syncInitHook = func(origin, latest uint64) { + starting <- struct{}{} + <-progress + } + checkProgress(t, tester.downloader, "pristine", ethereum.SyncProgress{}) + + // Attempt a full sync with a faulty peer + brokenChain := chain.shorten(chain.len()) + missing := brokenChain.len() / 2 + delete(brokenChain.headerm, brokenChain.chain[missing]) + delete(brokenChain.blockm, brokenChain.chain[missing]) + delete(brokenChain.receiptm, brokenChain.chain[missing]) + tester.newPeer("faulty", protocol, brokenChain) + + pending := new(sync.WaitGroup) + pending.Add(1) + go func() { + defer pending.Done() + if err := tester.sync("faulty", nil, mode); err == nil { + panic("succeeded faulty synchronisation") + } + }() + <-starting + checkProgress(t, tester.downloader, "initial", ethereum.SyncProgress{ + HighestBlock: uint64(brokenChain.len() - 1), + }) + progress <- struct{}{} + pending.Wait() + afterFailedSync := tester.downloader.Progress() + + // Synchronise with a good peer and check that the progress origin remind the same + // after a failure + tester.newPeer("valid", protocol, chain) + pending.Add(1) + go func() { + defer pending.Done() + if err := tester.sync("valid", nil, mode); err != nil { + panic(fmt.Sprintf("failed to synchronise blocks: %v", err)) + } + }() + <-starting + checkProgress(t, tester.downloader, "completing", afterFailedSync) + + // Check final progress after successful sync + progress <- struct{}{} + pending.Wait() + checkProgress(t, tester.downloader, "final", ethereum.SyncProgress{ + CurrentBlock: uint64(chain.len() - 1), + HighestBlock: uint64(chain.len() - 1), + }) +} + +// Tests that if an attacker fakes a chain height, after the attack is detected, +// the progress height is successfully reduced at the next sync invocation. +func TestFakedSyncProgress66Full(t *testing.T) { testFakedSyncProgress(t, eth.ETH66, FullSync) } +func TestFakedSyncProgress66Fast(t *testing.T) { testFakedSyncProgress(t, eth.ETH66, FastSync) } +func TestFakedSyncProgress66Light(t *testing.T) { testFakedSyncProgress(t, eth.ETH66, LightSync) } + +func testFakedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { + t.Parallel() + + tester := newTester() + defer tester.terminate() + chain := testChainBase.shorten(blockCacheMaxItems - 15) + + // Set a sync init hook to catch progress changes + starting := make(chan struct{}) + progress := make(chan struct{}) + tester.downloader.syncInitHook = func(origin, latest uint64) { + starting <- struct{}{} + <-progress + } + checkProgress(t, tester.downloader, "pristine", ethereum.SyncProgress{}) + + // Create and sync with an attacker that promises a higher chain than available. + brokenChain := chain.shorten(chain.len()) + numMissing := 5 + for i := brokenChain.len() - 2; i > brokenChain.len()-numMissing; i-- { + delete(brokenChain.headerm, brokenChain.chain[i]) + } + tester.newPeer("attack", protocol, brokenChain) + + pending := new(sync.WaitGroup) + pending.Add(1) + go func() { + defer pending.Done() + if err := tester.sync("attack", nil, mode); err == nil { + panic("succeeded attacker synchronisation") + } + }() + <-starting + checkProgress(t, tester.downloader, "initial", ethereum.SyncProgress{ + HighestBlock: uint64(brokenChain.len() - 1), + }) + progress <- struct{}{} + pending.Wait() + afterFailedSync := tester.downloader.Progress() + + // Synchronise with a good peer and check that the progress height has been reduced to + // the true value. + validChain := chain.shorten(chain.len() - numMissing) + tester.newPeer("valid", protocol, validChain) + pending.Add(1) + + go func() { + defer pending.Done() + if err := tester.sync("valid", nil, mode); err != nil { + panic(fmt.Sprintf("failed to synchronise blocks: %v", err)) + } + }() + <-starting + checkProgress(t, tester.downloader, "completing", ethereum.SyncProgress{ + CurrentBlock: afterFailedSync.CurrentBlock, + HighestBlock: uint64(validChain.len() - 1), + }) + + // Check final progress after successful sync. + progress <- struct{}{} + pending.Wait() + checkProgress(t, tester.downloader, "final", ethereum.SyncProgress{ + CurrentBlock: uint64(validChain.len() - 1), + HighestBlock: uint64(validChain.len() - 1), + }) +} + +// This test reproduces an issue where unexpected deliveries would +// block indefinitely if they arrived at the right time. +func TestDeliverHeadersHang66Full(t *testing.T) { testDeliverHeadersHang(t, eth.ETH66, FullSync) } +func TestDeliverHeadersHang66Fast(t *testing.T) { testDeliverHeadersHang(t, eth.ETH66, FastSync) } +func TestDeliverHeadersHang66Light(t *testing.T) { testDeliverHeadersHang(t, eth.ETH66, LightSync) } + +func testDeliverHeadersHang(t *testing.T, protocol uint, mode SyncMode) { + t.Parallel() + + master := newTester() + defer master.terminate() + chain := testChainBase.shorten(15) + + for i := 0; i < 200; i++ { + tester := newTester() + tester.peerDb = master.peerDb + tester.newPeer("peer", protocol, chain) + + // Whenever the downloader requests headers, flood it with + // a lot of unrequested header deliveries. + tester.downloader.peers.peers["peer"].peer = &floodingTestPeer{ + peer: tester.downloader.peers.peers["peer"].peer, + tester: tester, + } + if err := tester.sync("peer", nil, mode); err != nil { + t.Errorf("test %d: sync failed: %v", i, err) + } + tester.terminate() + } +} + +type floodingTestPeer struct { + peer Peer + tester *downloadTester +} + +func (ftp *floodingTestPeer) Head() (common.Hash, *big.Int) { return ftp.peer.Head() } +func (ftp *floodingTestPeer) RequestHeadersByHash(hash common.Hash, count int, skip int, reverse bool) error { + return ftp.peer.RequestHeadersByHash(hash, count, skip, reverse) +} +func (ftp *floodingTestPeer) RequestBodies(hashes []common.Hash) error { + return ftp.peer.RequestBodies(hashes) +} +func (ftp *floodingTestPeer) RequestReceipts(hashes []common.Hash) error { + return ftp.peer.RequestReceipts(hashes) +} +func (ftp *floodingTestPeer) RequestNodeData(hashes []common.Hash) error { + return ftp.peer.RequestNodeData(hashes) +} + +func (ftp *floodingTestPeer) RequestHeadersByNumber(from uint64, count, skip int, reverse bool) error { + deliveriesDone := make(chan struct{}, 500) + for i := 0; i < cap(deliveriesDone)-1; i++ { + peer := fmt.Sprintf("fake-peer%d", i) + go func() { + ftp.tester.downloader.DeliverHeaders(peer, []*types.Header{{}, {}, {}, {}}) + deliveriesDone <- struct{}{} + }() + } + + // None of the extra deliveries should block. + timeout := time.After(60 * time.Second) + launched := false + for i := 0; i < cap(deliveriesDone); i++ { + select { + case <-deliveriesDone: + if !launched { + // Start delivering the requested headers + // after one of the flooding responses has arrived. + go func() { + ftp.peer.RequestHeadersByNumber(from, count, skip, reverse) + deliveriesDone <- struct{}{} + }() + launched = true + } + case <-timeout: + panic("blocked") + } + } + return nil +} + +func TestRemoteHeaderRequestSpan(t *testing.T) { + testCases := []struct { + remoteHeight uint64 + localHeight uint64 + expected []int + }{ + // Remote is way higher. We should ask for the remote head and go backwards + {1500, 1000, + []int{1323, 1339, 1355, 1371, 1387, 1403, 1419, 1435, 1451, 1467, 1483, 1499}, + }, + {15000, 13006, + []int{14823, 14839, 14855, 14871, 14887, 14903, 14919, 14935, 14951, 14967, 14983, 14999}, + }, + // Remote is pretty close to us. We don't have to fetch as many + {1200, 1150, + []int{1149, 1154, 1159, 1164, 1169, 1174, 1179, 1184, 1189, 1194, 1199}, + }, + // Remote is equal to us (so on a fork with higher td) + // We should get the closest couple of ancestors + {1500, 1500, + []int{1497, 1499}, + }, + // We're higher than the remote! Odd + {1000, 1500, + []int{997, 999}, + }, + // Check some weird edgecases that it behaves somewhat rationally + {0, 1500, + []int{0, 2}, + }, + {6000000, 0, + []int{5999823, 5999839, 5999855, 5999871, 5999887, 5999903, 5999919, 5999935, 5999951, 5999967, 5999983, 5999999}, + }, + {0, 0, + []int{0, 2}, + }, + } + reqs := func(from, count, span int) []int { + var r []int + num := from + for len(r) < count { + r = append(r, num) + num += span + 1 + } + return r + } + for i, tt := range testCases { + from, count, span, max := calculateRequestSpan(tt.remoteHeight, tt.localHeight) + data := reqs(int(from), count, span) + + if max != uint64(data[len(data)-1]) { + t.Errorf("test %d: wrong last value %d != %d", i, data[len(data)-1], max) + } + failed := false + if len(data) != len(tt.expected) { + failed = true + t.Errorf("test %d: length wrong, expected %d got %d", i, len(tt.expected), len(data)) + } else { + for j, n := range data { + if n != tt.expected[j] { + failed = true + break + } + } + } + if failed { + res := strings.Replace(fmt.Sprint(data), " ", ",", -1) + exp := strings.Replace(fmt.Sprint(tt.expected), " ", ",", -1) + t.Logf("got: %v\n", res) + t.Logf("exp: %v\n", exp) + t.Errorf("test %d: wrong values", i) + } + } +} + +// Tests that peers below a pre-configured checkpoint block are prevented from +// being fast-synced from, avoiding potential cheap eclipse attacks. +func TestCheckpointEnforcement66Full(t *testing.T) { testCheckpointEnforcement(t, eth.ETH66, FullSync) } +func TestCheckpointEnforcement66Fast(t *testing.T) { testCheckpointEnforcement(t, eth.ETH66, FastSync) } +func TestCheckpointEnforcement66Light(t *testing.T) { + testCheckpointEnforcement(t, eth.ETH66, LightSync) +} + +func testCheckpointEnforcement(t *testing.T, protocol uint, mode SyncMode) { + t.Parallel() + + // Create a new tester with a particular hard coded checkpoint block + tester := newTester() + defer tester.terminate() + + tester.downloader.checkpoint = uint64(fsMinFullBlocks) + 256 + chain := testChainBase.shorten(int(tester.downloader.checkpoint) - 1) + + // Attempt to sync with the peer and validate the result + tester.newPeer("peer", protocol, chain) + + var expect error + if mode == FastSync || mode == LightSync { + expect = errUnsyncedPeer + } + if err := tester.sync("peer", nil, mode); !errors.Is(err, expect) { + t.Fatalf("block sync error mismatch: have %v, want %v", err, expect) + } + if mode == FastSync || mode == LightSync { + assertOwnChain(t, tester, 1) + } else { + assertOwnChain(t, tester, chain.len()) + } +} diff --git a/les/downloader/events.go b/les/downloader/events.go new file mode 100644 index 000000000..25255a3a7 --- /dev/null +++ b/les/downloader/events.go @@ -0,0 +1,25 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package downloader + +import "github.com/ethereum/go-ethereum/core/types" + +type DoneEvent struct { + Latest *types.Header +} +type StartEvent struct{} +type FailedEvent struct{ Err error } diff --git a/les/downloader/metrics.go b/les/downloader/metrics.go new file mode 100644 index 000000000..c38732043 --- /dev/null +++ b/les/downloader/metrics.go @@ -0,0 +1,45 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains the metrics collected by the downloader. + +package downloader + +import ( + "github.com/ethereum/go-ethereum/metrics" +) + +var ( + headerInMeter = metrics.NewRegisteredMeter("eth/downloader/headers/in", nil) + headerReqTimer = metrics.NewRegisteredTimer("eth/downloader/headers/req", nil) + headerDropMeter = metrics.NewRegisteredMeter("eth/downloader/headers/drop", nil) + headerTimeoutMeter = metrics.NewRegisteredMeter("eth/downloader/headers/timeout", nil) + + bodyInMeter = metrics.NewRegisteredMeter("eth/downloader/bodies/in", nil) + bodyReqTimer = metrics.NewRegisteredTimer("eth/downloader/bodies/req", nil) + bodyDropMeter = metrics.NewRegisteredMeter("eth/downloader/bodies/drop", nil) + bodyTimeoutMeter = metrics.NewRegisteredMeter("eth/downloader/bodies/timeout", nil) + + receiptInMeter = metrics.NewRegisteredMeter("eth/downloader/receipts/in", nil) + receiptReqTimer = metrics.NewRegisteredTimer("eth/downloader/receipts/req", nil) + receiptDropMeter = metrics.NewRegisteredMeter("eth/downloader/receipts/drop", nil) + receiptTimeoutMeter = metrics.NewRegisteredMeter("eth/downloader/receipts/timeout", nil) + + stateInMeter = metrics.NewRegisteredMeter("eth/downloader/states/in", nil) + stateDropMeter = metrics.NewRegisteredMeter("eth/downloader/states/drop", nil) + + throttleCounter = metrics.NewRegisteredCounter("eth/downloader/throttle", nil) +) diff --git a/les/downloader/modes.go b/les/downloader/modes.go new file mode 100644 index 000000000..3ea14d22d --- /dev/null +++ b/les/downloader/modes.go @@ -0,0 +1,81 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package downloader + +import "fmt" + +// SyncMode represents the synchronisation mode of the downloader. +// It is a uint32 as it is used with atomic operations. +type SyncMode uint32 + +const ( + FullSync SyncMode = iota // Synchronise the entire blockchain history from full blocks + FastSync // Quickly download the headers, full sync only at the chain + SnapSync // Download the chain and the state via compact snapshots + LightSync // Download only the headers and terminate afterwards +) + +func (mode SyncMode) IsValid() bool { + return mode >= FullSync && mode <= LightSync +} + +// String implements the stringer interface. +func (mode SyncMode) String() string { + switch mode { + case FullSync: + return "full" + case FastSync: + return "fast" + case SnapSync: + return "snap" + case LightSync: + return "light" + default: + return "unknown" + } +} + +func (mode SyncMode) MarshalText() ([]byte, error) { + switch mode { + case FullSync: + return []byte("full"), nil + case FastSync: + return []byte("fast"), nil + case SnapSync: + return []byte("snap"), nil + case LightSync: + return []byte("light"), nil + default: + return nil, fmt.Errorf("unknown sync mode %d", mode) + } +} + +func (mode *SyncMode) UnmarshalText(text []byte) error { + switch string(text) { + case "full": + *mode = FullSync + case "fast": + *mode = FastSync + case "snap": + *mode = SnapSync + case "light": + *mode = LightSync + default: + return fmt.Errorf(`unknown sync mode %q, want "full", "fast" or "light"`, text) + } + return nil +} diff --git a/les/downloader/peer.go b/les/downloader/peer.go new file mode 100644 index 000000000..863294832 --- /dev/null +++ b/les/downloader/peer.go @@ -0,0 +1,501 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains the active peer-set of the downloader, maintaining both failures +// as well as reputation metrics to prioritize the block retrievals. + +package downloader + +import ( + "errors" + "math/big" + "sort" + "sync" + "sync/atomic" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/eth/protocols/eth" + "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p/msgrate" +) + +const ( + maxLackingHashes = 4096 // Maximum number of entries allowed on the list or lacking items +) + +var ( + errAlreadyFetching = errors.New("already fetching blocks from peer") + errAlreadyRegistered = errors.New("peer is already registered") + errNotRegistered = errors.New("peer is not registered") +) + +// peerConnection represents an active peer from which hashes and blocks are retrieved. +type peerConnection struct { + id string // Unique identifier of the peer + + headerIdle int32 // Current header activity state of the peer (idle = 0, active = 1) + blockIdle int32 // Current block activity state of the peer (idle = 0, active = 1) + receiptIdle int32 // Current receipt activity state of the peer (idle = 0, active = 1) + stateIdle int32 // Current node data activity state of the peer (idle = 0, active = 1) + + headerStarted time.Time // Time instance when the last header fetch was started + blockStarted time.Time // Time instance when the last block (body) fetch was started + receiptStarted time.Time // Time instance when the last receipt fetch was started + stateStarted time.Time // Time instance when the last node data fetch was started + + rates *msgrate.Tracker // Tracker to hone in on the number of items retrievable per second + lacking map[common.Hash]struct{} // Set of hashes not to request (didn't have previously) + + peer Peer + + version uint // Eth protocol version number to switch strategies + log log.Logger // Contextual logger to add extra infos to peer logs + lock sync.RWMutex +} + +// LightPeer encapsulates the methods required to synchronise with a remote light peer. +type LightPeer interface { + Head() (common.Hash, *big.Int) + RequestHeadersByHash(common.Hash, int, int, bool) error + RequestHeadersByNumber(uint64, int, int, bool) error +} + +// Peer encapsulates the methods required to synchronise with a remote full peer. +type Peer interface { + LightPeer + RequestBodies([]common.Hash) error + RequestReceipts([]common.Hash) error + RequestNodeData([]common.Hash) error +} + +// lightPeerWrapper wraps a LightPeer struct, stubbing out the Peer-only methods. +type lightPeerWrapper struct { + peer LightPeer +} + +func (w *lightPeerWrapper) Head() (common.Hash, *big.Int) { return w.peer.Head() } +func (w *lightPeerWrapper) RequestHeadersByHash(h common.Hash, amount int, skip int, reverse bool) error { + return w.peer.RequestHeadersByHash(h, amount, skip, reverse) +} +func (w *lightPeerWrapper) RequestHeadersByNumber(i uint64, amount int, skip int, reverse bool) error { + return w.peer.RequestHeadersByNumber(i, amount, skip, reverse) +} +func (w *lightPeerWrapper) RequestBodies([]common.Hash) error { + panic("RequestBodies not supported in light client mode sync") +} +func (w *lightPeerWrapper) RequestReceipts([]common.Hash) error { + panic("RequestReceipts not supported in light client mode sync") +} +func (w *lightPeerWrapper) RequestNodeData([]common.Hash) error { + panic("RequestNodeData not supported in light client mode sync") +} + +// newPeerConnection creates a new downloader peer. +func newPeerConnection(id string, version uint, peer Peer, logger log.Logger) *peerConnection { + return &peerConnection{ + id: id, + lacking: make(map[common.Hash]struct{}), + peer: peer, + version: version, + log: logger, + } +} + +// Reset clears the internal state of a peer entity. +func (p *peerConnection) Reset() { + p.lock.Lock() + defer p.lock.Unlock() + + atomic.StoreInt32(&p.headerIdle, 0) + atomic.StoreInt32(&p.blockIdle, 0) + atomic.StoreInt32(&p.receiptIdle, 0) + atomic.StoreInt32(&p.stateIdle, 0) + + p.lacking = make(map[common.Hash]struct{}) +} + +// FetchHeaders sends a header retrieval request to the remote peer. +func (p *peerConnection) FetchHeaders(from uint64, count int) error { + // Short circuit if the peer is already fetching + if !atomic.CompareAndSwapInt32(&p.headerIdle, 0, 1) { + return errAlreadyFetching + } + p.headerStarted = time.Now() + + // Issue the header retrieval request (absolute upwards without gaps) + go p.peer.RequestHeadersByNumber(from, count, 0, false) + + return nil +} + +// FetchBodies sends a block body retrieval request to the remote peer. +func (p *peerConnection) FetchBodies(request *fetchRequest) error { + // Short circuit if the peer is already fetching + if !atomic.CompareAndSwapInt32(&p.blockIdle, 0, 1) { + return errAlreadyFetching + } + p.blockStarted = time.Now() + + go func() { + // Convert the header set to a retrievable slice + hashes := make([]common.Hash, 0, len(request.Headers)) + for _, header := range request.Headers { + hashes = append(hashes, header.Hash()) + } + p.peer.RequestBodies(hashes) + }() + + return nil +} + +// FetchReceipts sends a receipt retrieval request to the remote peer. +func (p *peerConnection) FetchReceipts(request *fetchRequest) error { + // Short circuit if the peer is already fetching + if !atomic.CompareAndSwapInt32(&p.receiptIdle, 0, 1) { + return errAlreadyFetching + } + p.receiptStarted = time.Now() + + go func() { + // Convert the header set to a retrievable slice + hashes := make([]common.Hash, 0, len(request.Headers)) + for _, header := range request.Headers { + hashes = append(hashes, header.Hash()) + } + p.peer.RequestReceipts(hashes) + }() + + return nil +} + +// FetchNodeData sends a node state data retrieval request to the remote peer. +func (p *peerConnection) FetchNodeData(hashes []common.Hash) error { + // Short circuit if the peer is already fetching + if !atomic.CompareAndSwapInt32(&p.stateIdle, 0, 1) { + return errAlreadyFetching + } + p.stateStarted = time.Now() + + go p.peer.RequestNodeData(hashes) + + return nil +} + +// SetHeadersIdle sets the peer to idle, allowing it to execute new header retrieval +// requests. Its estimated header retrieval throughput is updated with that measured +// just now. +func (p *peerConnection) SetHeadersIdle(delivered int, deliveryTime time.Time) { + p.rates.Update(eth.BlockHeadersMsg, deliveryTime.Sub(p.headerStarted), delivered) + atomic.StoreInt32(&p.headerIdle, 0) +} + +// SetBodiesIdle sets the peer to idle, allowing it to execute block body retrieval +// requests. Its estimated body retrieval throughput is updated with that measured +// just now. +func (p *peerConnection) SetBodiesIdle(delivered int, deliveryTime time.Time) { + p.rates.Update(eth.BlockBodiesMsg, deliveryTime.Sub(p.blockStarted), delivered) + atomic.StoreInt32(&p.blockIdle, 0) +} + +// SetReceiptsIdle sets the peer to idle, allowing it to execute new receipt +// retrieval requests. Its estimated receipt retrieval throughput is updated +// with that measured just now. +func (p *peerConnection) SetReceiptsIdle(delivered int, deliveryTime time.Time) { + p.rates.Update(eth.ReceiptsMsg, deliveryTime.Sub(p.receiptStarted), delivered) + atomic.StoreInt32(&p.receiptIdle, 0) +} + +// SetNodeDataIdle sets the peer to idle, allowing it to execute new state trie +// data retrieval requests. Its estimated state retrieval throughput is updated +// with that measured just now. +func (p *peerConnection) SetNodeDataIdle(delivered int, deliveryTime time.Time) { + p.rates.Update(eth.NodeDataMsg, deliveryTime.Sub(p.stateStarted), delivered) + atomic.StoreInt32(&p.stateIdle, 0) +} + +// HeaderCapacity retrieves the peers header download allowance based on its +// previously discovered throughput. +func (p *peerConnection) HeaderCapacity(targetRTT time.Duration) int { + cap := p.rates.Capacity(eth.BlockHeadersMsg, targetRTT) + if cap > MaxHeaderFetch { + cap = MaxHeaderFetch + } + return cap +} + +// BlockCapacity retrieves the peers block download allowance based on its +// previously discovered throughput. +func (p *peerConnection) BlockCapacity(targetRTT time.Duration) int { + cap := p.rates.Capacity(eth.BlockBodiesMsg, targetRTT) + if cap > MaxBlockFetch { + cap = MaxBlockFetch + } + return cap +} + +// ReceiptCapacity retrieves the peers receipt download allowance based on its +// previously discovered throughput. +func (p *peerConnection) ReceiptCapacity(targetRTT time.Duration) int { + cap := p.rates.Capacity(eth.ReceiptsMsg, targetRTT) + if cap > MaxReceiptFetch { + cap = MaxReceiptFetch + } + return cap +} + +// NodeDataCapacity retrieves the peers state download allowance based on its +// previously discovered throughput. +func (p *peerConnection) NodeDataCapacity(targetRTT time.Duration) int { + cap := p.rates.Capacity(eth.NodeDataMsg, targetRTT) + if cap > MaxStateFetch { + cap = MaxStateFetch + } + return cap +} + +// MarkLacking appends a new entity to the set of items (blocks, receipts, states) +// that a peer is known not to have (i.e. have been requested before). If the +// set reaches its maximum allowed capacity, items are randomly dropped off. +func (p *peerConnection) MarkLacking(hash common.Hash) { + p.lock.Lock() + defer p.lock.Unlock() + + for len(p.lacking) >= maxLackingHashes { + for drop := range p.lacking { + delete(p.lacking, drop) + break + } + } + p.lacking[hash] = struct{}{} +} + +// Lacks retrieves whether the hash of a blockchain item is on the peers lacking +// list (i.e. whether we know that the peer does not have it). +func (p *peerConnection) Lacks(hash common.Hash) bool { + p.lock.RLock() + defer p.lock.RUnlock() + + _, ok := p.lacking[hash] + return ok +} + +// peerSet represents the collection of active peer participating in the chain +// download procedure. +type peerSet struct { + peers map[string]*peerConnection + rates *msgrate.Trackers // Set of rate trackers to give the sync a common beat + + newPeerFeed event.Feed + peerDropFeed event.Feed + + lock sync.RWMutex +} + +// newPeerSet creates a new peer set top track the active download sources. +func newPeerSet() *peerSet { + return &peerSet{ + peers: make(map[string]*peerConnection), + rates: msgrate.NewTrackers(log.New("proto", "eth")), + } +} + +// SubscribeNewPeers subscribes to peer arrival events. +func (ps *peerSet) SubscribeNewPeers(ch chan<- *peerConnection) event.Subscription { + return ps.newPeerFeed.Subscribe(ch) +} + +// SubscribePeerDrops subscribes to peer departure events. +func (ps *peerSet) SubscribePeerDrops(ch chan<- *peerConnection) event.Subscription { + return ps.peerDropFeed.Subscribe(ch) +} + +// Reset iterates over the current peer set, and resets each of the known peers +// to prepare for a next batch of block retrieval. +func (ps *peerSet) Reset() { + ps.lock.RLock() + defer ps.lock.RUnlock() + + for _, peer := range ps.peers { + peer.Reset() + } +} + +// Register injects a new peer into the working set, or returns an error if the +// peer is already known. +// +// The method also sets the starting throughput values of the new peer to the +// average of all existing peers, to give it a realistic chance of being used +// for data retrievals. +func (ps *peerSet) Register(p *peerConnection) error { + // Register the new peer with some meaningful defaults + ps.lock.Lock() + if _, ok := ps.peers[p.id]; ok { + ps.lock.Unlock() + return errAlreadyRegistered + } + p.rates = msgrate.NewTracker(ps.rates.MeanCapacities(), ps.rates.MedianRoundTrip()) + if err := ps.rates.Track(p.id, p.rates); err != nil { + return err + } + ps.peers[p.id] = p + ps.lock.Unlock() + + ps.newPeerFeed.Send(p) + return nil +} + +// Unregister removes a remote peer from the active set, disabling any further +// actions to/from that particular entity. +func (ps *peerSet) Unregister(id string) error { + ps.lock.Lock() + p, ok := ps.peers[id] + if !ok { + ps.lock.Unlock() + return errNotRegistered + } + delete(ps.peers, id) + ps.rates.Untrack(id) + ps.lock.Unlock() + + ps.peerDropFeed.Send(p) + return nil +} + +// Peer retrieves the registered peer with the given id. +func (ps *peerSet) Peer(id string) *peerConnection { + ps.lock.RLock() + defer ps.lock.RUnlock() + + return ps.peers[id] +} + +// Len returns if the current number of peers in the set. +func (ps *peerSet) Len() int { + ps.lock.RLock() + defer ps.lock.RUnlock() + + return len(ps.peers) +} + +// AllPeers retrieves a flat list of all the peers within the set. +func (ps *peerSet) AllPeers() []*peerConnection { + ps.lock.RLock() + defer ps.lock.RUnlock() + + list := make([]*peerConnection, 0, len(ps.peers)) + for _, p := range ps.peers { + list = append(list, p) + } + return list +} + +// HeaderIdlePeers retrieves a flat list of all the currently header-idle peers +// within the active peer set, ordered by their reputation. +func (ps *peerSet) HeaderIdlePeers() ([]*peerConnection, int) { + idle := func(p *peerConnection) bool { + return atomic.LoadInt32(&p.headerIdle) == 0 + } + throughput := func(p *peerConnection) int { + return p.rates.Capacity(eth.BlockHeadersMsg, time.Second) + } + return ps.idlePeers(eth.ETH66, eth.ETH66, idle, throughput) +} + +// BodyIdlePeers retrieves a flat list of all the currently body-idle peers within +// the active peer set, ordered by their reputation. +func (ps *peerSet) BodyIdlePeers() ([]*peerConnection, int) { + idle := func(p *peerConnection) bool { + return atomic.LoadInt32(&p.blockIdle) == 0 + } + throughput := func(p *peerConnection) int { + return p.rates.Capacity(eth.BlockBodiesMsg, time.Second) + } + return ps.idlePeers(eth.ETH66, eth.ETH66, idle, throughput) +} + +// ReceiptIdlePeers retrieves a flat list of all the currently receipt-idle peers +// within the active peer set, ordered by their reputation. +func (ps *peerSet) ReceiptIdlePeers() ([]*peerConnection, int) { + idle := func(p *peerConnection) bool { + return atomic.LoadInt32(&p.receiptIdle) == 0 + } + throughput := func(p *peerConnection) int { + return p.rates.Capacity(eth.ReceiptsMsg, time.Second) + } + return ps.idlePeers(eth.ETH66, eth.ETH66, idle, throughput) +} + +// NodeDataIdlePeers retrieves a flat list of all the currently node-data-idle +// peers within the active peer set, ordered by their reputation. +func (ps *peerSet) NodeDataIdlePeers() ([]*peerConnection, int) { + idle := func(p *peerConnection) bool { + return atomic.LoadInt32(&p.stateIdle) == 0 + } + throughput := func(p *peerConnection) int { + return p.rates.Capacity(eth.NodeDataMsg, time.Second) + } + return ps.idlePeers(eth.ETH66, eth.ETH66, idle, throughput) +} + +// idlePeers retrieves a flat list of all currently idle peers satisfying the +// protocol version constraints, using the provided function to check idleness. +// The resulting set of peers are sorted by their capacity. +func (ps *peerSet) idlePeers(minProtocol, maxProtocol uint, idleCheck func(*peerConnection) bool, capacity func(*peerConnection) int) ([]*peerConnection, int) { + ps.lock.RLock() + defer ps.lock.RUnlock() + + var ( + total = 0 + idle = make([]*peerConnection, 0, len(ps.peers)) + tps = make([]int, 0, len(ps.peers)) + ) + for _, p := range ps.peers { + if p.version >= minProtocol && p.version <= maxProtocol { + if idleCheck(p) { + idle = append(idle, p) + tps = append(tps, capacity(p)) + } + total++ + } + } + + // And sort them + sortPeers := &peerCapacitySort{idle, tps} + sort.Sort(sortPeers) + return sortPeers.p, total +} + +// peerCapacitySort implements sort.Interface. +// It sorts peer connections by capacity (descending). +type peerCapacitySort struct { + p []*peerConnection + tp []int +} + +func (ps *peerCapacitySort) Len() int { + return len(ps.p) +} + +func (ps *peerCapacitySort) Less(i, j int) bool { + return ps.tp[i] > ps.tp[j] +} + +func (ps *peerCapacitySort) Swap(i, j int) { + ps.p[i], ps.p[j] = ps.p[j], ps.p[i] + ps.tp[i], ps.tp[j] = ps.tp[j], ps.tp[i] +} diff --git a/les/downloader/queue.go b/les/downloader/queue.go new file mode 100644 index 000000000..04ec12cfa --- /dev/null +++ b/les/downloader/queue.go @@ -0,0 +1,913 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// Contains the block download scheduler to collect download tasks and schedule +// them in an ordered, and throttled way. + +package downloader + +import ( + "errors" + "fmt" + "sync" + "sync/atomic" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/prque" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/trie" +) + +const ( + bodyType = uint(0) + receiptType = uint(1) +) + +var ( + blockCacheMaxItems = 8192 // Maximum number of blocks to cache before throttling the download + blockCacheInitialItems = 2048 // Initial number of blocks to start fetching, before we know the sizes of the blocks + blockCacheMemory = 256 * 1024 * 1024 // Maximum amount of memory to use for block caching + blockCacheSizeWeight = 0.1 // Multiplier to approximate the average block size based on past ones +) + +var ( + errNoFetchesPending = errors.New("no fetches pending") + errStaleDelivery = errors.New("stale delivery") +) + +// fetchRequest is a currently running data retrieval operation. +type fetchRequest struct { + Peer *peerConnection // Peer to which the request was sent + From uint64 // [eth/62] Requested chain element index (used for skeleton fills only) + Headers []*types.Header // [eth/62] Requested headers, sorted by request order + Time time.Time // Time when the request was made +} + +// fetchResult is a struct collecting partial results from data fetchers until +// all outstanding pieces complete and the result as a whole can be processed. +type fetchResult struct { + pending int32 // Flag telling what deliveries are outstanding + + Header *types.Header + Uncles []*types.Header + Transactions types.Transactions + Receipts types.Receipts +} + +func newFetchResult(header *types.Header, fastSync bool) *fetchResult { + item := &fetchResult{ + Header: header, + } + if !header.EmptyBody() { + item.pending |= (1 << bodyType) + } + if fastSync && !header.EmptyReceipts() { + item.pending |= (1 << receiptType) + } + return item +} + +// SetBodyDone flags the body as finished. +func (f *fetchResult) SetBodyDone() { + if v := atomic.LoadInt32(&f.pending); (v & (1 << bodyType)) != 0 { + atomic.AddInt32(&f.pending, -1) + } +} + +// AllDone checks if item is done. +func (f *fetchResult) AllDone() bool { + return atomic.LoadInt32(&f.pending) == 0 +} + +// SetReceiptsDone flags the receipts as finished. +func (f *fetchResult) SetReceiptsDone() { + if v := atomic.LoadInt32(&f.pending); (v & (1 << receiptType)) != 0 { + atomic.AddInt32(&f.pending, -2) + } +} + +// Done checks if the given type is done already +func (f *fetchResult) Done(kind uint) bool { + v := atomic.LoadInt32(&f.pending) + return v&(1< 0 +} + +// InFlightBlocks retrieves whether there are block fetch requests currently in +// flight. +func (q *queue) InFlightBlocks() bool { + q.lock.Lock() + defer q.lock.Unlock() + + return len(q.blockPendPool) > 0 +} + +// InFlightReceipts retrieves whether there are receipt fetch requests currently +// in flight. +func (q *queue) InFlightReceipts() bool { + q.lock.Lock() + defer q.lock.Unlock() + + return len(q.receiptPendPool) > 0 +} + +// Idle returns if the queue is fully idle or has some data still inside. +func (q *queue) Idle() bool { + q.lock.Lock() + defer q.lock.Unlock() + + queued := q.blockTaskQueue.Size() + q.receiptTaskQueue.Size() + pending := len(q.blockPendPool) + len(q.receiptPendPool) + + return (queued + pending) == 0 +} + +// ScheduleSkeleton adds a batch of header retrieval tasks to the queue to fill +// up an already retrieved header skeleton. +func (q *queue) ScheduleSkeleton(from uint64, skeleton []*types.Header) { + q.lock.Lock() + defer q.lock.Unlock() + + // No skeleton retrieval can be in progress, fail hard if so (huge implementation bug) + if q.headerResults != nil { + panic("skeleton assembly already in progress") + } + // Schedule all the header retrieval tasks for the skeleton assembly + q.headerTaskPool = make(map[uint64]*types.Header) + q.headerTaskQueue = prque.New(nil) + q.headerPeerMiss = make(map[string]map[uint64]struct{}) // Reset availability to correct invalid chains + q.headerResults = make([]*types.Header, len(skeleton)*MaxHeaderFetch) + q.headerProced = 0 + q.headerOffset = from + q.headerContCh = make(chan bool, 1) + + for i, header := range skeleton { + index := from + uint64(i*MaxHeaderFetch) + + q.headerTaskPool[index] = header + q.headerTaskQueue.Push(index, -int64(index)) + } +} + +// RetrieveHeaders retrieves the header chain assemble based on the scheduled +// skeleton. +func (q *queue) RetrieveHeaders() ([]*types.Header, int) { + q.lock.Lock() + defer q.lock.Unlock() + + headers, proced := q.headerResults, q.headerProced + q.headerResults, q.headerProced = nil, 0 + + return headers, proced +} + +// Schedule adds a set of headers for the download queue for scheduling, returning +// the new headers encountered. +func (q *queue) Schedule(headers []*types.Header, from uint64) []*types.Header { + q.lock.Lock() + defer q.lock.Unlock() + + // Insert all the headers prioritised by the contained block number + inserts := make([]*types.Header, 0, len(headers)) + for _, header := range headers { + // Make sure chain order is honoured and preserved throughout + hash := header.Hash() + if header.Number == nil || header.Number.Uint64() != from { + log.Warn("Header broke chain ordering", "number", header.Number, "hash", hash, "expected", from) + break + } + if q.headerHead != (common.Hash{}) && q.headerHead != header.ParentHash { + log.Warn("Header broke chain ancestry", "number", header.Number, "hash", hash) + break + } + // Make sure no duplicate requests are executed + // We cannot skip this, even if the block is empty, since this is + // what triggers the fetchResult creation. + if _, ok := q.blockTaskPool[hash]; ok { + log.Warn("Header already scheduled for block fetch", "number", header.Number, "hash", hash) + } else { + q.blockTaskPool[hash] = header + q.blockTaskQueue.Push(header, -int64(header.Number.Uint64())) + } + // Queue for receipt retrieval + if q.mode == FastSync && !header.EmptyReceipts() { + if _, ok := q.receiptTaskPool[hash]; ok { + log.Warn("Header already scheduled for receipt fetch", "number", header.Number, "hash", hash) + } else { + q.receiptTaskPool[hash] = header + q.receiptTaskQueue.Push(header, -int64(header.Number.Uint64())) + } + } + inserts = append(inserts, header) + q.headerHead = hash + from++ + } + return inserts +} + +// Results retrieves and permanently removes a batch of fetch results from +// the cache. the result slice will be empty if the queue has been closed. +// Results can be called concurrently with Deliver and Schedule, +// but assumes that there are not two simultaneous callers to Results +func (q *queue) Results(block bool) []*fetchResult { + // Abort early if there are no items and non-blocking requested + if !block && !q.resultCache.HasCompletedItems() { + return nil + } + closed := false + for !closed && !q.resultCache.HasCompletedItems() { + // In order to wait on 'active', we need to obtain the lock. + // That may take a while, if someone is delivering at the same + // time, so after obtaining the lock, we check again if there + // are any results to fetch. + // Also, in-between we ask for the lock and the lock is obtained, + // someone can have closed the queue. In that case, we should + // return the available results and stop blocking + q.lock.Lock() + if q.resultCache.HasCompletedItems() || q.closed { + q.lock.Unlock() + break + } + // No items available, and not closed + q.active.Wait() + closed = q.closed + q.lock.Unlock() + } + // Regardless if closed or not, we can still deliver whatever we have + results := q.resultCache.GetCompleted(maxResultsProcess) + for _, result := range results { + // Recalculate the result item weights to prevent memory exhaustion + size := result.Header.Size() + for _, uncle := range result.Uncles { + size += uncle.Size() + } + for _, receipt := range result.Receipts { + size += receipt.Size() + } + for _, tx := range result.Transactions { + size += tx.Size() + } + q.resultSize = common.StorageSize(blockCacheSizeWeight)*size + + (1-common.StorageSize(blockCacheSizeWeight))*q.resultSize + } + // Using the newly calibrated resultsize, figure out the new throttle limit + // on the result cache + throttleThreshold := uint64((common.StorageSize(blockCacheMemory) + q.resultSize - 1) / q.resultSize) + throttleThreshold = q.resultCache.SetThrottleThreshold(throttleThreshold) + + // Log some info at certain times + if time.Since(q.lastStatLog) > 60*time.Second { + q.lastStatLog = time.Now() + info := q.Stats() + info = append(info, "throttle", throttleThreshold) + log.Info("Downloader queue stats", info...) + } + return results +} + +func (q *queue) Stats() []interface{} { + q.lock.RLock() + defer q.lock.RUnlock() + + return q.stats() +} + +func (q *queue) stats() []interface{} { + return []interface{}{ + "receiptTasks", q.receiptTaskQueue.Size(), + "blockTasks", q.blockTaskQueue.Size(), + "itemSize", q.resultSize, + } +} + +// ReserveHeaders reserves a set of headers for the given peer, skipping any +// previously failed batches. +func (q *queue) ReserveHeaders(p *peerConnection, count int) *fetchRequest { + q.lock.Lock() + defer q.lock.Unlock() + + // Short circuit if the peer's already downloading something (sanity check to + // not corrupt state) + if _, ok := q.headerPendPool[p.id]; ok { + return nil + } + // Retrieve a batch of hashes, skipping previously failed ones + send, skip := uint64(0), []uint64{} + for send == 0 && !q.headerTaskQueue.Empty() { + from, _ := q.headerTaskQueue.Pop() + if q.headerPeerMiss[p.id] != nil { + if _, ok := q.headerPeerMiss[p.id][from.(uint64)]; ok { + skip = append(skip, from.(uint64)) + continue + } + } + send = from.(uint64) + } + // Merge all the skipped batches back + for _, from := range skip { + q.headerTaskQueue.Push(from, -int64(from)) + } + // Assemble and return the block download request + if send == 0 { + return nil + } + request := &fetchRequest{ + Peer: p, + From: send, + Time: time.Now(), + } + q.headerPendPool[p.id] = request + return request +} + +// ReserveBodies reserves a set of body fetches for the given peer, skipping any +// previously failed downloads. Beside the next batch of needed fetches, it also +// returns a flag whether empty blocks were queued requiring processing. +func (q *queue) ReserveBodies(p *peerConnection, count int) (*fetchRequest, bool, bool) { + q.lock.Lock() + defer q.lock.Unlock() + + return q.reserveHeaders(p, count, q.blockTaskPool, q.blockTaskQueue, q.blockPendPool, bodyType) +} + +// ReserveReceipts reserves a set of receipt fetches for the given peer, skipping +// any previously failed downloads. Beside the next batch of needed fetches, it +// also returns a flag whether empty receipts were queued requiring importing. +func (q *queue) ReserveReceipts(p *peerConnection, count int) (*fetchRequest, bool, bool) { + q.lock.Lock() + defer q.lock.Unlock() + + return q.reserveHeaders(p, count, q.receiptTaskPool, q.receiptTaskQueue, q.receiptPendPool, receiptType) +} + +// reserveHeaders reserves a set of data download operations for a given peer, +// skipping any previously failed ones. This method is a generic version used +// by the individual special reservation functions. +// +// Note, this method expects the queue lock to be already held for writing. The +// reason the lock is not obtained in here is because the parameters already need +// to access the queue, so they already need a lock anyway. +// +// Returns: +// item - the fetchRequest +// progress - whether any progress was made +// throttle - if the caller should throttle for a while +func (q *queue) reserveHeaders(p *peerConnection, count int, taskPool map[common.Hash]*types.Header, taskQueue *prque.Prque, + pendPool map[string]*fetchRequest, kind uint) (*fetchRequest, bool, bool) { + // Short circuit if the pool has been depleted, or if the peer's already + // downloading something (sanity check not to corrupt state) + if taskQueue.Empty() { + return nil, false, true + } + if _, ok := pendPool[p.id]; ok { + return nil, false, false + } + // Retrieve a batch of tasks, skipping previously failed ones + send := make([]*types.Header, 0, count) + skip := make([]*types.Header, 0) + progress := false + throttled := false + for proc := 0; len(send) < count && !taskQueue.Empty(); proc++ { + // the task queue will pop items in order, so the highest prio block + // is also the lowest block number. + h, _ := taskQueue.Peek() + header := h.(*types.Header) + // we can ask the resultcache if this header is within the + // "prioritized" segment of blocks. If it is not, we need to throttle + + stale, throttle, item, err := q.resultCache.AddFetch(header, q.mode == FastSync) + if stale { + // Don't put back in the task queue, this item has already been + // delivered upstream + taskQueue.PopItem() + progress = true + delete(taskPool, header.Hash()) + proc = proc - 1 + log.Error("Fetch reservation already delivered", "number", header.Number.Uint64()) + continue + } + if throttle { + // There are no resultslots available. Leave it in the task queue + // However, if there are any left as 'skipped', we should not tell + // the caller to throttle, since we still want some other + // peer to fetch those for us + throttled = len(skip) == 0 + break + } + if err != nil { + // this most definitely should _not_ happen + log.Warn("Failed to reserve headers", "err", err) + // There are no resultslots available. Leave it in the task queue + break + } + if item.Done(kind) { + // If it's a noop, we can skip this task + delete(taskPool, header.Hash()) + taskQueue.PopItem() + proc = proc - 1 + progress = true + continue + } + // Remove it from the task queue + taskQueue.PopItem() + // Otherwise unless the peer is known not to have the data, add to the retrieve list + if p.Lacks(header.Hash()) { + skip = append(skip, header) + } else { + send = append(send, header) + } + } + // Merge all the skipped headers back + for _, header := range skip { + taskQueue.Push(header, -int64(header.Number.Uint64())) + } + if q.resultCache.HasCompletedItems() { + // Wake Results, resultCache was modified + q.active.Signal() + } + // Assemble and return the block download request + if len(send) == 0 { + return nil, progress, throttled + } + request := &fetchRequest{ + Peer: p, + Headers: send, + Time: time.Now(), + } + pendPool[p.id] = request + return request, progress, throttled +} + +// CancelHeaders aborts a fetch request, returning all pending skeleton indexes to the queue. +func (q *queue) CancelHeaders(request *fetchRequest) { + q.lock.Lock() + defer q.lock.Unlock() + q.cancel(request, q.headerTaskQueue, q.headerPendPool) +} + +// CancelBodies aborts a body fetch request, returning all pending headers to the +// task queue. +func (q *queue) CancelBodies(request *fetchRequest) { + q.lock.Lock() + defer q.lock.Unlock() + q.cancel(request, q.blockTaskQueue, q.blockPendPool) +} + +// CancelReceipts aborts a body fetch request, returning all pending headers to +// the task queue. +func (q *queue) CancelReceipts(request *fetchRequest) { + q.lock.Lock() + defer q.lock.Unlock() + q.cancel(request, q.receiptTaskQueue, q.receiptPendPool) +} + +// Cancel aborts a fetch request, returning all pending hashes to the task queue. +func (q *queue) cancel(request *fetchRequest, taskQueue *prque.Prque, pendPool map[string]*fetchRequest) { + if request.From > 0 { + taskQueue.Push(request.From, -int64(request.From)) + } + for _, header := range request.Headers { + taskQueue.Push(header, -int64(header.Number.Uint64())) + } + delete(pendPool, request.Peer.id) +} + +// Revoke cancels all pending requests belonging to a given peer. This method is +// meant to be called during a peer drop to quickly reassign owned data fetches +// to remaining nodes. +func (q *queue) Revoke(peerID string) { + q.lock.Lock() + defer q.lock.Unlock() + + if request, ok := q.blockPendPool[peerID]; ok { + for _, header := range request.Headers { + q.blockTaskQueue.Push(header, -int64(header.Number.Uint64())) + } + delete(q.blockPendPool, peerID) + } + if request, ok := q.receiptPendPool[peerID]; ok { + for _, header := range request.Headers { + q.receiptTaskQueue.Push(header, -int64(header.Number.Uint64())) + } + delete(q.receiptPendPool, peerID) + } +} + +// ExpireHeaders checks for in flight requests that exceeded a timeout allowance, +// canceling them and returning the responsible peers for penalisation. +func (q *queue) ExpireHeaders(timeout time.Duration) map[string]int { + q.lock.Lock() + defer q.lock.Unlock() + + return q.expire(timeout, q.headerPendPool, q.headerTaskQueue, headerTimeoutMeter) +} + +// ExpireBodies checks for in flight block body requests that exceeded a timeout +// allowance, canceling them and returning the responsible peers for penalisation. +func (q *queue) ExpireBodies(timeout time.Duration) map[string]int { + q.lock.Lock() + defer q.lock.Unlock() + + return q.expire(timeout, q.blockPendPool, q.blockTaskQueue, bodyTimeoutMeter) +} + +// ExpireReceipts checks for in flight receipt requests that exceeded a timeout +// allowance, canceling them and returning the responsible peers for penalisation. +func (q *queue) ExpireReceipts(timeout time.Duration) map[string]int { + q.lock.Lock() + defer q.lock.Unlock() + + return q.expire(timeout, q.receiptPendPool, q.receiptTaskQueue, receiptTimeoutMeter) +} + +// expire is the generic check that move expired tasks from a pending pool back +// into a task pool, returning all entities caught with expired tasks. +// +// Note, this method expects the queue lock to be already held. The +// reason the lock is not obtained in here is because the parameters already need +// to access the queue, so they already need a lock anyway. +func (q *queue) expire(timeout time.Duration, pendPool map[string]*fetchRequest, taskQueue *prque.Prque, timeoutMeter metrics.Meter) map[string]int { + // Iterate over the expired requests and return each to the queue + expiries := make(map[string]int) + for id, request := range pendPool { + if time.Since(request.Time) > timeout { + // Update the metrics with the timeout + timeoutMeter.Mark(1) + + // Return any non satisfied requests to the pool + if request.From > 0 { + taskQueue.Push(request.From, -int64(request.From)) + } + for _, header := range request.Headers { + taskQueue.Push(header, -int64(header.Number.Uint64())) + } + // Add the peer to the expiry report along the number of failed requests + expiries[id] = len(request.Headers) + + // Remove the expired requests from the pending pool directly + delete(pendPool, id) + } + } + return expiries +} + +// DeliverHeaders injects a header retrieval response into the header results +// cache. This method either accepts all headers it received, or none of them +// if they do not map correctly to the skeleton. +// +// If the headers are accepted, the method makes an attempt to deliver the set +// of ready headers to the processor to keep the pipeline full. However it will +// not block to prevent stalling other pending deliveries. +func (q *queue) DeliverHeaders(id string, headers []*types.Header, headerProcCh chan []*types.Header) (int, error) { + q.lock.Lock() + defer q.lock.Unlock() + + var logger log.Logger + if len(id) < 16 { + // Tests use short IDs, don't choke on them + logger = log.New("peer", id) + } else { + logger = log.New("peer", id[:16]) + } + // Short circuit if the data was never requested + request := q.headerPendPool[id] + if request == nil { + return 0, errNoFetchesPending + } + headerReqTimer.UpdateSince(request.Time) + delete(q.headerPendPool, id) + + // Ensure headers can be mapped onto the skeleton chain + target := q.headerTaskPool[request.From].Hash() + + accepted := len(headers) == MaxHeaderFetch + if accepted { + if headers[0].Number.Uint64() != request.From { + logger.Trace("First header broke chain ordering", "number", headers[0].Number, "hash", headers[0].Hash(), "expected", request.From) + accepted = false + } else if headers[len(headers)-1].Hash() != target { + logger.Trace("Last header broke skeleton structure ", "number", headers[len(headers)-1].Number, "hash", headers[len(headers)-1].Hash(), "expected", target) + accepted = false + } + } + if accepted { + parentHash := headers[0].Hash() + for i, header := range headers[1:] { + hash := header.Hash() + if want := request.From + 1 + uint64(i); header.Number.Uint64() != want { + logger.Warn("Header broke chain ordering", "number", header.Number, "hash", hash, "expected", want) + accepted = false + break + } + if parentHash != header.ParentHash { + logger.Warn("Header broke chain ancestry", "number", header.Number, "hash", hash) + accepted = false + break + } + // Set-up parent hash for next round + parentHash = hash + } + } + // If the batch of headers wasn't accepted, mark as unavailable + if !accepted { + logger.Trace("Skeleton filling not accepted", "from", request.From) + + miss := q.headerPeerMiss[id] + if miss == nil { + q.headerPeerMiss[id] = make(map[uint64]struct{}) + miss = q.headerPeerMiss[id] + } + miss[request.From] = struct{}{} + + q.headerTaskQueue.Push(request.From, -int64(request.From)) + return 0, errors.New("delivery not accepted") + } + // Clean up a successful fetch and try to deliver any sub-results + copy(q.headerResults[request.From-q.headerOffset:], headers) + delete(q.headerTaskPool, request.From) + + ready := 0 + for q.headerProced+ready < len(q.headerResults) && q.headerResults[q.headerProced+ready] != nil { + ready += MaxHeaderFetch + } + if ready > 0 { + // Headers are ready for delivery, gather them and push forward (non blocking) + process := make([]*types.Header, ready) + copy(process, q.headerResults[q.headerProced:q.headerProced+ready]) + + select { + case headerProcCh <- process: + logger.Trace("Pre-scheduled new headers", "count", len(process), "from", process[0].Number) + q.headerProced += len(process) + default: + } + } + // Check for termination and return + if len(q.headerTaskPool) == 0 { + q.headerContCh <- false + } + return len(headers), nil +} + +// DeliverBodies injects a block body retrieval response into the results queue. +// The method returns the number of blocks bodies accepted from the delivery and +// also wakes any threads waiting for data delivery. +func (q *queue) DeliverBodies(id string, txLists [][]*types.Transaction, uncleLists [][]*types.Header) (int, error) { + q.lock.Lock() + defer q.lock.Unlock() + trieHasher := trie.NewStackTrie(nil) + validate := func(index int, header *types.Header) error { + if types.DeriveSha(types.Transactions(txLists[index]), trieHasher) != header.TxHash { + return errInvalidBody + } + if types.CalcUncleHash(uncleLists[index]) != header.UncleHash { + return errInvalidBody + } + return nil + } + + reconstruct := func(index int, result *fetchResult) { + result.Transactions = txLists[index] + result.Uncles = uncleLists[index] + result.SetBodyDone() + } + return q.deliver(id, q.blockTaskPool, q.blockTaskQueue, q.blockPendPool, + bodyReqTimer, len(txLists), validate, reconstruct) +} + +// DeliverReceipts injects a receipt retrieval response into the results queue. +// The method returns the number of transaction receipts accepted from the delivery +// and also wakes any threads waiting for data delivery. +func (q *queue) DeliverReceipts(id string, receiptList [][]*types.Receipt) (int, error) { + q.lock.Lock() + defer q.lock.Unlock() + trieHasher := trie.NewStackTrie(nil) + validate := func(index int, header *types.Header) error { + if types.DeriveSha(types.Receipts(receiptList[index]), trieHasher) != header.ReceiptHash { + return errInvalidReceipt + } + return nil + } + reconstruct := func(index int, result *fetchResult) { + result.Receipts = receiptList[index] + result.SetReceiptsDone() + } + return q.deliver(id, q.receiptTaskPool, q.receiptTaskQueue, q.receiptPendPool, + receiptReqTimer, len(receiptList), validate, reconstruct) +} + +// deliver injects a data retrieval response into the results queue. +// +// Note, this method expects the queue lock to be already held for writing. The +// reason this lock is not obtained in here is because the parameters already need +// to access the queue, so they already need a lock anyway. +func (q *queue) deliver(id string, taskPool map[common.Hash]*types.Header, + taskQueue *prque.Prque, pendPool map[string]*fetchRequest, reqTimer metrics.Timer, + results int, validate func(index int, header *types.Header) error, + reconstruct func(index int, result *fetchResult)) (int, error) { + + // Short circuit if the data was never requested + request := pendPool[id] + if request == nil { + return 0, errNoFetchesPending + } + reqTimer.UpdateSince(request.Time) + delete(pendPool, id) + + // If no data items were retrieved, mark them as unavailable for the origin peer + if results == 0 { + for _, header := range request.Headers { + request.Peer.MarkLacking(header.Hash()) + } + } + // Assemble each of the results with their headers and retrieved data parts + var ( + accepted int + failure error + i int + hashes []common.Hash + ) + for _, header := range request.Headers { + // Short circuit assembly if no more fetch results are found + if i >= results { + break + } + // Validate the fields + if err := validate(i, header); err != nil { + failure = err + break + } + hashes = append(hashes, header.Hash()) + i++ + } + + for _, header := range request.Headers[:i] { + if res, stale, err := q.resultCache.GetDeliverySlot(header.Number.Uint64()); err == nil { + reconstruct(accepted, res) + } else { + // else: betweeen here and above, some other peer filled this result, + // or it was indeed a no-op. This should not happen, but if it does it's + // not something to panic about + log.Error("Delivery stale", "stale", stale, "number", header.Number.Uint64(), "err", err) + failure = errStaleDelivery + } + // Clean up a successful fetch + delete(taskPool, hashes[accepted]) + accepted++ + } + // Return all failed or missing fetches to the queue + for _, header := range request.Headers[accepted:] { + taskQueue.Push(header, -int64(header.Number.Uint64())) + } + // Wake up Results + if accepted > 0 { + q.active.Signal() + } + if failure == nil { + return accepted, nil + } + // If none of the data was good, it's a stale delivery + if accepted > 0 { + return accepted, fmt.Errorf("partial failure: %v", failure) + } + return accepted, fmt.Errorf("%w: %v", failure, errStaleDelivery) +} + +// Prepare configures the result cache to allow accepting and caching inbound +// fetch results. +func (q *queue) Prepare(offset uint64, mode SyncMode) { + q.lock.Lock() + defer q.lock.Unlock() + + // Prepare the queue for sync results + q.resultCache.Prepare(offset) + q.mode = mode +} diff --git a/les/downloader/queue_test.go b/les/downloader/queue_test.go new file mode 100644 index 000000000..cde5f306a --- /dev/null +++ b/les/downloader/queue_test.go @@ -0,0 +1,452 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package downloader + +import ( + "fmt" + "math/big" + "math/rand" + "sync" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/params" +) + +var ( + testdb = rawdb.NewMemoryDatabase() + genesis = core.GenesisBlockForTesting(testdb, testAddress, big.NewInt(1000000000000000)) +) + +// makeChain creates a chain of n blocks starting at and including parent. +// the returned hash chain is ordered head->parent. In addition, every 3rd block +// contains a transaction and every 5th an uncle to allow testing correct block +// reassembly. +func makeChain(n int, seed byte, parent *types.Block, empty bool) ([]*types.Block, []types.Receipts) { + blocks, receipts := core.GenerateChain(params.TestChainConfig, parent, ethash.NewFaker(), testdb, n, func(i int, block *core.BlockGen) { + block.SetCoinbase(common.Address{seed}) + // Add one tx to every secondblock + if !empty && i%2 == 0 { + signer := types.MakeSigner(params.TestChainConfig, block.Number()) + tx, err := types.SignTx(types.NewTransaction(block.TxNonce(testAddress), common.Address{seed}, big.NewInt(1000), params.TxGas, block.BaseFee(), nil), signer, testKey) + if err != nil { + panic(err) + } + block.AddTx(tx) + } + }) + return blocks, receipts +} + +type chainData struct { + blocks []*types.Block + offset int +} + +var chain *chainData +var emptyChain *chainData + +func init() { + // Create a chain of blocks to import + targetBlocks := 128 + blocks, _ := makeChain(targetBlocks, 0, genesis, false) + chain = &chainData{blocks, 0} + + blocks, _ = makeChain(targetBlocks, 0, genesis, true) + emptyChain = &chainData{blocks, 0} +} + +func (chain *chainData) headers() []*types.Header { + hdrs := make([]*types.Header, len(chain.blocks)) + for i, b := range chain.blocks { + hdrs[i] = b.Header() + } + return hdrs +} + +func (chain *chainData) Len() int { + return len(chain.blocks) +} + +func dummyPeer(id string) *peerConnection { + p := &peerConnection{ + id: id, + lacking: make(map[common.Hash]struct{}), + } + return p +} + +func TestBasics(t *testing.T) { + numOfBlocks := len(emptyChain.blocks) + numOfReceipts := len(emptyChain.blocks) / 2 + + q := newQueue(10, 10) + if !q.Idle() { + t.Errorf("new queue should be idle") + } + q.Prepare(1, FastSync) + if res := q.Results(false); len(res) != 0 { + t.Fatal("new queue should have 0 results") + } + + // Schedule a batch of headers + q.Schedule(chain.headers(), 1) + if q.Idle() { + t.Errorf("queue should not be idle") + } + if got, exp := q.PendingBlocks(), chain.Len(); got != exp { + t.Errorf("wrong pending block count, got %d, exp %d", got, exp) + } + // Only non-empty receipts get added to task-queue + if got, exp := q.PendingReceipts(), 64; got != exp { + t.Errorf("wrong pending receipt count, got %d, exp %d", got, exp) + } + // Items are now queued for downloading, next step is that we tell the + // queue that a certain peer will deliver them for us + { + peer := dummyPeer("peer-1") + fetchReq, _, throttle := q.ReserveBodies(peer, 50) + if !throttle { + // queue size is only 10, so throttling should occur + t.Fatal("should throttle") + } + // But we should still get the first things to fetch + if got, exp := len(fetchReq.Headers), 5; got != exp { + t.Fatalf("expected %d requests, got %d", exp, got) + } + if got, exp := fetchReq.Headers[0].Number.Uint64(), uint64(1); got != exp { + t.Fatalf("expected header %d, got %d", exp, got) + } + } + if exp, got := q.blockTaskQueue.Size(), numOfBlocks-10; exp != got { + t.Errorf("expected block task queue to be %d, got %d", exp, got) + } + if exp, got := q.receiptTaskQueue.Size(), numOfReceipts; exp != got { + t.Errorf("expected receipt task queue to be %d, got %d", exp, got) + } + { + peer := dummyPeer("peer-2") + fetchReq, _, throttle := q.ReserveBodies(peer, 50) + + // The second peer should hit throttling + if !throttle { + t.Fatalf("should not throttle") + } + // And not get any fetches at all, since it was throttled to begin with + if fetchReq != nil { + t.Fatalf("should have no fetches, got %d", len(fetchReq.Headers)) + } + } + if exp, got := q.blockTaskQueue.Size(), numOfBlocks-10; exp != got { + t.Errorf("expected block task queue to be %d, got %d", exp, got) + } + if exp, got := q.receiptTaskQueue.Size(), numOfReceipts; exp != got { + t.Errorf("expected receipt task queue to be %d, got %d", exp, got) + } + { + // The receipt delivering peer should not be affected + // by the throttling of body deliveries + peer := dummyPeer("peer-3") + fetchReq, _, throttle := q.ReserveReceipts(peer, 50) + if !throttle { + // queue size is only 10, so throttling should occur + t.Fatal("should throttle") + } + // But we should still get the first things to fetch + if got, exp := len(fetchReq.Headers), 5; got != exp { + t.Fatalf("expected %d requests, got %d", exp, got) + } + if got, exp := fetchReq.Headers[0].Number.Uint64(), uint64(1); got != exp { + t.Fatalf("expected header %d, got %d", exp, got) + } + + } + if exp, got := q.blockTaskQueue.Size(), numOfBlocks-10; exp != got { + t.Errorf("expected block task queue to be %d, got %d", exp, got) + } + if exp, got := q.receiptTaskQueue.Size(), numOfReceipts-5; exp != got { + t.Errorf("expected receipt task queue to be %d, got %d", exp, got) + } + if got, exp := q.resultCache.countCompleted(), 0; got != exp { + t.Errorf("wrong processable count, got %d, exp %d", got, exp) + } +} + +func TestEmptyBlocks(t *testing.T) { + numOfBlocks := len(emptyChain.blocks) + + q := newQueue(10, 10) + + q.Prepare(1, FastSync) + // Schedule a batch of headers + q.Schedule(emptyChain.headers(), 1) + if q.Idle() { + t.Errorf("queue should not be idle") + } + if got, exp := q.PendingBlocks(), len(emptyChain.blocks); got != exp { + t.Errorf("wrong pending block count, got %d, exp %d", got, exp) + } + if got, exp := q.PendingReceipts(), 0; got != exp { + t.Errorf("wrong pending receipt count, got %d, exp %d", got, exp) + } + // They won't be processable, because the fetchresults haven't been + // created yet + if got, exp := q.resultCache.countCompleted(), 0; got != exp { + t.Errorf("wrong processable count, got %d, exp %d", got, exp) + } + + // Items are now queued for downloading, next step is that we tell the + // queue that a certain peer will deliver them for us + // That should trigger all of them to suddenly become 'done' + { + // Reserve blocks + peer := dummyPeer("peer-1") + fetchReq, _, _ := q.ReserveBodies(peer, 50) + + // there should be nothing to fetch, blocks are empty + if fetchReq != nil { + t.Fatal("there should be no body fetch tasks remaining") + } + + } + if q.blockTaskQueue.Size() != numOfBlocks-10 { + t.Errorf("expected block task queue to be %d, got %d", numOfBlocks-10, q.blockTaskQueue.Size()) + } + if q.receiptTaskQueue.Size() != 0 { + t.Errorf("expected receipt task queue to be %d, got %d", 0, q.receiptTaskQueue.Size()) + } + { + peer := dummyPeer("peer-3") + fetchReq, _, _ := q.ReserveReceipts(peer, 50) + + // there should be nothing to fetch, blocks are empty + if fetchReq != nil { + t.Fatal("there should be no body fetch tasks remaining") + } + } + if q.blockTaskQueue.Size() != numOfBlocks-10 { + t.Errorf("expected block task queue to be %d, got %d", numOfBlocks-10, q.blockTaskQueue.Size()) + } + if q.receiptTaskQueue.Size() != 0 { + t.Errorf("expected receipt task queue to be %d, got %d", 0, q.receiptTaskQueue.Size()) + } + if got, exp := q.resultCache.countCompleted(), 10; got != exp { + t.Errorf("wrong processable count, got %d, exp %d", got, exp) + } +} + +// XTestDelivery does some more extensive testing of events that happen, +// blocks that become known and peers that make reservations and deliveries. +// disabled since it's not really a unit-test, but can be executed to test +// some more advanced scenarios +func XTestDelivery(t *testing.T) { + // the outside network, holding blocks + blo, rec := makeChain(128, 0, genesis, false) + world := newNetwork() + world.receipts = rec + world.chain = blo + world.progress(10) + if false { + log.Root().SetHandler(log.StdoutHandler) + + } + q := newQueue(10, 10) + var wg sync.WaitGroup + q.Prepare(1, FastSync) + wg.Add(1) + go func() { + // deliver headers + defer wg.Done() + c := 1 + for { + //fmt.Printf("getting headers from %d\n", c) + hdrs := world.headers(c) + l := len(hdrs) + //fmt.Printf("scheduling %d headers, first %d last %d\n", + // l, hdrs[0].Number.Uint64(), hdrs[len(hdrs)-1].Number.Uint64()) + q.Schedule(hdrs, uint64(c)) + c += l + } + }() + wg.Add(1) + go func() { + // collect results + defer wg.Done() + tot := 0 + for { + res := q.Results(true) + tot += len(res) + fmt.Printf("got %d results, %d tot\n", len(res), tot) + // Now we can forget about these + world.forget(res[len(res)-1].Header.Number.Uint64()) + + } + }() + wg.Add(1) + go func() { + defer wg.Done() + // reserve body fetch + i := 4 + for { + peer := dummyPeer(fmt.Sprintf("peer-%d", i)) + f, _, _ := q.ReserveBodies(peer, rand.Intn(30)) + if f != nil { + var emptyList []*types.Header + var txs [][]*types.Transaction + var uncles [][]*types.Header + numToSkip := rand.Intn(len(f.Headers)) + for _, hdr := range f.Headers[0 : len(f.Headers)-numToSkip] { + txs = append(txs, world.getTransactions(hdr.Number.Uint64())) + uncles = append(uncles, emptyList) + } + time.Sleep(100 * time.Millisecond) + _, err := q.DeliverBodies(peer.id, txs, uncles) + if err != nil { + fmt.Printf("delivered %d bodies %v\n", len(txs), err) + } + } else { + i++ + time.Sleep(200 * time.Millisecond) + } + } + }() + go func() { + defer wg.Done() + // reserve receiptfetch + peer := dummyPeer("peer-3") + for { + f, _, _ := q.ReserveReceipts(peer, rand.Intn(50)) + if f != nil { + var rcs [][]*types.Receipt + for _, hdr := range f.Headers { + rcs = append(rcs, world.getReceipts(hdr.Number.Uint64())) + } + _, err := q.DeliverReceipts(peer.id, rcs) + if err != nil { + fmt.Printf("delivered %d receipts %v\n", len(rcs), err) + } + time.Sleep(100 * time.Millisecond) + } else { + time.Sleep(200 * time.Millisecond) + } + } + }() + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; i < 50; i++ { + time.Sleep(300 * time.Millisecond) + //world.tick() + //fmt.Printf("trying to progress\n") + world.progress(rand.Intn(100)) + } + for i := 0; i < 50; i++ { + time.Sleep(2990 * time.Millisecond) + + } + }() + wg.Add(1) + go func() { + defer wg.Done() + for { + time.Sleep(990 * time.Millisecond) + fmt.Printf("world block tip is %d\n", + world.chain[len(world.chain)-1].Header().Number.Uint64()) + fmt.Println(q.Stats()) + } + }() + wg.Wait() +} + +func newNetwork() *network { + var l sync.RWMutex + return &network{ + cond: sync.NewCond(&l), + offset: 1, // block 1 is at blocks[0] + } +} + +// represents the network +type network struct { + offset int + chain []*types.Block + receipts []types.Receipts + lock sync.RWMutex + cond *sync.Cond +} + +func (n *network) getTransactions(blocknum uint64) types.Transactions { + index := blocknum - uint64(n.offset) + return n.chain[index].Transactions() +} +func (n *network) getReceipts(blocknum uint64) types.Receipts { + index := blocknum - uint64(n.offset) + if got := n.chain[index].Header().Number.Uint64(); got != blocknum { + fmt.Printf("Err, got %d exp %d\n", got, blocknum) + panic("sd") + } + return n.receipts[index] +} + +func (n *network) forget(blocknum uint64) { + index := blocknum - uint64(n.offset) + n.chain = n.chain[index:] + n.receipts = n.receipts[index:] + n.offset = int(blocknum) + +} +func (n *network) progress(numBlocks int) { + + n.lock.Lock() + defer n.lock.Unlock() + //fmt.Printf("progressing...\n") + newBlocks, newR := makeChain(numBlocks, 0, n.chain[len(n.chain)-1], false) + n.chain = append(n.chain, newBlocks...) + n.receipts = append(n.receipts, newR...) + n.cond.Broadcast() + +} + +func (n *network) headers(from int) []*types.Header { + numHeaders := 128 + var hdrs []*types.Header + index := from - n.offset + + for index >= len(n.chain) { + // wait for progress + n.cond.L.Lock() + //fmt.Printf("header going into wait\n") + n.cond.Wait() + index = from - n.offset + n.cond.L.Unlock() + } + n.lock.RLock() + defer n.lock.RUnlock() + for i, b := range n.chain[index:] { + hdrs = append(hdrs, b.Header()) + if i >= numHeaders { + break + } + } + return hdrs +} diff --git a/les/downloader/resultstore.go b/les/downloader/resultstore.go new file mode 100644 index 000000000..21928c2a0 --- /dev/null +++ b/les/downloader/resultstore.go @@ -0,0 +1,194 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package downloader + +import ( + "fmt" + "sync" + "sync/atomic" + + "github.com/ethereum/go-ethereum/core/types" +) + +// resultStore implements a structure for maintaining fetchResults, tracking their +// download-progress and delivering (finished) results. +type resultStore struct { + items []*fetchResult // Downloaded but not yet delivered fetch results + resultOffset uint64 // Offset of the first cached fetch result in the block chain + + // Internal index of first non-completed entry, updated atomically when needed. + // If all items are complete, this will equal length(items), so + // *important* : is not safe to use for indexing without checking against length + indexIncomplete int32 // atomic access + + // throttleThreshold is the limit up to which we _want_ to fill the + // results. If blocks are large, we want to limit the results to less + // than the number of available slots, and maybe only fill 1024 out of + // 8192 possible places. The queue will, at certain times, recalibrate + // this index. + throttleThreshold uint64 + + lock sync.RWMutex +} + +func newResultStore(size int) *resultStore { + return &resultStore{ + resultOffset: 0, + items: make([]*fetchResult, size), + throttleThreshold: uint64(size), + } +} + +// SetThrottleThreshold updates the throttling threshold based on the requested +// limit and the total queue capacity. It returns the (possibly capped) threshold +func (r *resultStore) SetThrottleThreshold(threshold uint64) uint64 { + r.lock.Lock() + defer r.lock.Unlock() + + limit := uint64(len(r.items)) + if threshold >= limit { + threshold = limit + } + r.throttleThreshold = threshold + return r.throttleThreshold +} + +// AddFetch adds a header for body/receipt fetching. This is used when the queue +// wants to reserve headers for fetching. +// +// It returns the following: +// stale - if true, this item is already passed, and should not be requested again +// throttled - if true, the store is at capacity, this particular header is not prio now +// item - the result to store data into +// err - any error that occurred +func (r *resultStore) AddFetch(header *types.Header, fastSync bool) (stale, throttled bool, item *fetchResult, err error) { + r.lock.Lock() + defer r.lock.Unlock() + + var index int + item, index, stale, throttled, err = r.getFetchResult(header.Number.Uint64()) + if err != nil || stale || throttled { + return stale, throttled, item, err + } + if item == nil { + item = newFetchResult(header, fastSync) + r.items[index] = item + } + return stale, throttled, item, err +} + +// GetDeliverySlot returns the fetchResult for the given header. If the 'stale' flag +// is true, that means the header has already been delivered 'upstream'. This method +// does not bubble up the 'throttle' flag, since it's moot at the point in time when +// the item is downloaded and ready for delivery +func (r *resultStore) GetDeliverySlot(headerNumber uint64) (*fetchResult, bool, error) { + r.lock.RLock() + defer r.lock.RUnlock() + + res, _, stale, _, err := r.getFetchResult(headerNumber) + return res, stale, err +} + +// getFetchResult returns the fetchResult corresponding to the given item, and +// the index where the result is stored. +func (r *resultStore) getFetchResult(headerNumber uint64) (item *fetchResult, index int, stale, throttle bool, err error) { + index = int(int64(headerNumber) - int64(r.resultOffset)) + throttle = index >= int(r.throttleThreshold) + stale = index < 0 + + if index >= len(r.items) { + err = fmt.Errorf("%w: index allocation went beyond available resultStore space "+ + "(index [%d] = header [%d] - resultOffset [%d], len(resultStore) = %d", errInvalidChain, + index, headerNumber, r.resultOffset, len(r.items)) + return nil, index, stale, throttle, err + } + if stale { + return nil, index, stale, throttle, nil + } + item = r.items[index] + return item, index, stale, throttle, nil +} + +// hasCompletedItems returns true if there are processable items available +// this method is cheaper than countCompleted +func (r *resultStore) HasCompletedItems() bool { + r.lock.RLock() + defer r.lock.RUnlock() + + if len(r.items) == 0 { + return false + } + if item := r.items[0]; item != nil && item.AllDone() { + return true + } + return false +} + +// countCompleted returns the number of items ready for delivery, stopping at +// the first non-complete item. +// +// The mthod assumes (at least) rlock is held. +func (r *resultStore) countCompleted() int { + // We iterate from the already known complete point, and see + // if any more has completed since last count + index := atomic.LoadInt32(&r.indexIncomplete) + for ; ; index++ { + if index >= int32(len(r.items)) { + break + } + result := r.items[index] + if result == nil || !result.AllDone() { + break + } + } + atomic.StoreInt32(&r.indexIncomplete, index) + return int(index) +} + +// GetCompleted returns the next batch of completed fetchResults +func (r *resultStore) GetCompleted(limit int) []*fetchResult { + r.lock.Lock() + defer r.lock.Unlock() + + completed := r.countCompleted() + if limit > completed { + limit = completed + } + results := make([]*fetchResult, limit) + copy(results, r.items[:limit]) + + // Delete the results from the cache and clear the tail. + copy(r.items, r.items[limit:]) + for i := len(r.items) - limit; i < len(r.items); i++ { + r.items[i] = nil + } + // Advance the expected block number of the first cache entry + r.resultOffset += uint64(limit) + atomic.AddInt32(&r.indexIncomplete, int32(-limit)) + + return results +} + +// Prepare initialises the offset with the given block number +func (r *resultStore) Prepare(offset uint64) { + r.lock.Lock() + defer r.lock.Unlock() + + if r.resultOffset < offset { + r.resultOffset = offset + } +} diff --git a/les/downloader/statesync.go b/les/downloader/statesync.go new file mode 100644 index 000000000..6c53e5577 --- /dev/null +++ b/les/downloader/statesync.go @@ -0,0 +1,615 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package downloader + +import ( + "fmt" + "sync" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/trie" + "golang.org/x/crypto/sha3" +) + +// stateReq represents a batch of state fetch requests grouped together into +// a single data retrieval network packet. +type stateReq struct { + nItems uint16 // Number of items requested for download (max is 384, so uint16 is sufficient) + trieTasks map[common.Hash]*trieTask // Trie node download tasks to track previous attempts + codeTasks map[common.Hash]*codeTask // Byte code download tasks to track previous attempts + timeout time.Duration // Maximum round trip time for this to complete + timer *time.Timer // Timer to fire when the RTT timeout expires + peer *peerConnection // Peer that we're requesting from + delivered time.Time // Time when the packet was delivered (independent when we process it) + response [][]byte // Response data of the peer (nil for timeouts) + dropped bool // Flag whether the peer dropped off early +} + +// timedOut returns if this request timed out. +func (req *stateReq) timedOut() bool { + return req.response == nil +} + +// stateSyncStats is a collection of progress stats to report during a state trie +// sync to RPC requests as well as to display in user logs. +type stateSyncStats struct { + processed uint64 // Number of state entries processed + duplicate uint64 // Number of state entries downloaded twice + unexpected uint64 // Number of non-requested state entries received + pending uint64 // Number of still pending state entries +} + +// syncState starts downloading state with the given root hash. +func (d *Downloader) syncState(root common.Hash) *stateSync { + // Create the state sync + s := newStateSync(d, root) + select { + case d.stateSyncStart <- s: + // If we tell the statesync to restart with a new root, we also need + // to wait for it to actually also start -- when old requests have timed + // out or been delivered + <-s.started + case <-d.quitCh: + s.err = errCancelStateFetch + close(s.done) + } + return s +} + +// stateFetcher manages the active state sync and accepts requests +// on its behalf. +func (d *Downloader) stateFetcher() { + for { + select { + case s := <-d.stateSyncStart: + for next := s; next != nil; { + next = d.runStateSync(next) + } + case <-d.stateCh: + // Ignore state responses while no sync is running. + case <-d.quitCh: + return + } + } +} + +// runStateSync runs a state synchronisation until it completes or another root +// hash is requested to be switched over to. +func (d *Downloader) runStateSync(s *stateSync) *stateSync { + var ( + active = make(map[string]*stateReq) // Currently in-flight requests + finished []*stateReq // Completed or failed requests + timeout = make(chan *stateReq) // Timed out active requests + ) + log.Trace("State sync starting", "root", s.root) + + defer func() { + // Cancel active request timers on exit. Also set peers to idle so they're + // available for the next sync. + for _, req := range active { + req.timer.Stop() + req.peer.SetNodeDataIdle(int(req.nItems), time.Now()) + } + }() + go s.run() + defer s.Cancel() + + // Listen for peer departure events to cancel assigned tasks + peerDrop := make(chan *peerConnection, 1024) + peerSub := s.d.peers.SubscribePeerDrops(peerDrop) + defer peerSub.Unsubscribe() + + for { + // Enable sending of the first buffered element if there is one. + var ( + deliverReq *stateReq + deliverReqCh chan *stateReq + ) + if len(finished) > 0 { + deliverReq = finished[0] + deliverReqCh = s.deliver + } + + select { + // The stateSync lifecycle: + case next := <-d.stateSyncStart: + d.spindownStateSync(active, finished, timeout, peerDrop) + return next + + case <-s.done: + d.spindownStateSync(active, finished, timeout, peerDrop) + return nil + + // Send the next finished request to the current sync: + case deliverReqCh <- deliverReq: + // Shift out the first request, but also set the emptied slot to nil for GC + copy(finished, finished[1:]) + finished[len(finished)-1] = nil + finished = finished[:len(finished)-1] + + // Handle incoming state packs: + case pack := <-d.stateCh: + // Discard any data not requested (or previously timed out) + req := active[pack.PeerId()] + if req == nil { + log.Debug("Unrequested node data", "peer", pack.PeerId(), "len", pack.Items()) + continue + } + // Finalize the request and queue up for processing + req.timer.Stop() + req.response = pack.(*statePack).states + req.delivered = time.Now() + + finished = append(finished, req) + delete(active, pack.PeerId()) + + // Handle dropped peer connections: + case p := <-peerDrop: + // Skip if no request is currently pending + req := active[p.id] + if req == nil { + continue + } + // Finalize the request and queue up for processing + req.timer.Stop() + req.dropped = true + req.delivered = time.Now() + + finished = append(finished, req) + delete(active, p.id) + + // Handle timed-out requests: + case req := <-timeout: + // If the peer is already requesting something else, ignore the stale timeout. + // This can happen when the timeout and the delivery happens simultaneously, + // causing both pathways to trigger. + if active[req.peer.id] != req { + continue + } + req.delivered = time.Now() + // Move the timed out data back into the download queue + finished = append(finished, req) + delete(active, req.peer.id) + + // Track outgoing state requests: + case req := <-d.trackStateReq: + // If an active request already exists for this peer, we have a problem. In + // theory the trie node schedule must never assign two requests to the same + // peer. In practice however, a peer might receive a request, disconnect and + // immediately reconnect before the previous times out. In this case the first + // request is never honored, alas we must not silently overwrite it, as that + // causes valid requests to go missing and sync to get stuck. + if old := active[req.peer.id]; old != nil { + log.Warn("Busy peer assigned new state fetch", "peer", old.peer.id) + // Move the previous request to the finished set + old.timer.Stop() + old.dropped = true + old.delivered = time.Now() + finished = append(finished, old) + } + // Start a timer to notify the sync loop if the peer stalled. + req.timer = time.AfterFunc(req.timeout, func() { + timeout <- req + }) + active[req.peer.id] = req + } + } +} + +// spindownStateSync 'drains' the outstanding requests; some will be delivered and other +// will time out. This is to ensure that when the next stateSync starts working, all peers +// are marked as idle and de facto _are_ idle. +func (d *Downloader) spindownStateSync(active map[string]*stateReq, finished []*stateReq, timeout chan *stateReq, peerDrop chan *peerConnection) { + log.Trace("State sync spinning down", "active", len(active), "finished", len(finished)) + for len(active) > 0 { + var ( + req *stateReq + reason string + ) + select { + // Handle (drop) incoming state packs: + case pack := <-d.stateCh: + req = active[pack.PeerId()] + reason = "delivered" + // Handle dropped peer connections: + case p := <-peerDrop: + req = active[p.id] + reason = "peerdrop" + // Handle timed-out requests: + case req = <-timeout: + reason = "timeout" + } + if req == nil { + continue + } + req.peer.log.Trace("State peer marked idle (spindown)", "req.items", int(req.nItems), "reason", reason) + req.timer.Stop() + delete(active, req.peer.id) + req.peer.SetNodeDataIdle(int(req.nItems), time.Now()) + } + // The 'finished' set contains deliveries that we were going to pass to processing. + // Those are now moot, but we still need to set those peers as idle, which would + // otherwise have been done after processing + for _, req := range finished { + req.peer.SetNodeDataIdle(int(req.nItems), time.Now()) + } +} + +// stateSync schedules requests for downloading a particular state trie defined +// by a given state root. +type stateSync struct { + d *Downloader // Downloader instance to access and manage current peerset + + root common.Hash // State root currently being synced + sched *trie.Sync // State trie sync scheduler defining the tasks + keccak crypto.KeccakState // Keccak256 hasher to verify deliveries with + + trieTasks map[common.Hash]*trieTask // Set of trie node tasks currently queued for retrieval + codeTasks map[common.Hash]*codeTask // Set of byte code tasks currently queued for retrieval + + numUncommitted int + bytesUncommitted int + + started chan struct{} // Started is signalled once the sync loop starts + + deliver chan *stateReq // Delivery channel multiplexing peer responses + cancel chan struct{} // Channel to signal a termination request + cancelOnce sync.Once // Ensures cancel only ever gets called once + done chan struct{} // Channel to signal termination completion + err error // Any error hit during sync (set before completion) +} + +// trieTask represents a single trie node download task, containing a set of +// peers already attempted retrieval from to detect stalled syncs and abort. +type trieTask struct { + path [][]byte + attempts map[string]struct{} +} + +// codeTask represents a single byte code download task, containing a set of +// peers already attempted retrieval from to detect stalled syncs and abort. +type codeTask struct { + attempts map[string]struct{} +} + +// newStateSync creates a new state trie download scheduler. This method does not +// yet start the sync. The user needs to call run to initiate. +func newStateSync(d *Downloader, root common.Hash) *stateSync { + return &stateSync{ + d: d, + root: root, + sched: state.NewStateSync(root, d.stateDB, d.stateBloom, nil), + keccak: sha3.NewLegacyKeccak256().(crypto.KeccakState), + trieTasks: make(map[common.Hash]*trieTask), + codeTasks: make(map[common.Hash]*codeTask), + deliver: make(chan *stateReq), + cancel: make(chan struct{}), + done: make(chan struct{}), + started: make(chan struct{}), + } +} + +// run starts the task assignment and response processing loop, blocking until +// it finishes, and finally notifying any goroutines waiting for the loop to +// finish. +func (s *stateSync) run() { + close(s.started) + if s.d.snapSync { + s.err = s.d.SnapSyncer.Sync(s.root, s.cancel) + } else { + s.err = s.loop() + } + close(s.done) +} + +// Wait blocks until the sync is done or canceled. +func (s *stateSync) Wait() error { + <-s.done + return s.err +} + +// Cancel cancels the sync and waits until it has shut down. +func (s *stateSync) Cancel() error { + s.cancelOnce.Do(func() { + close(s.cancel) + }) + return s.Wait() +} + +// loop is the main event loop of a state trie sync. It it responsible for the +// assignment of new tasks to peers (including sending it to them) as well as +// for the processing of inbound data. Note, that the loop does not directly +// receive data from peers, rather those are buffered up in the downloader and +// pushed here async. The reason is to decouple processing from data receipt +// and timeouts. +func (s *stateSync) loop() (err error) { + // Listen for new peer events to assign tasks to them + newPeer := make(chan *peerConnection, 1024) + peerSub := s.d.peers.SubscribeNewPeers(newPeer) + defer peerSub.Unsubscribe() + defer func() { + cerr := s.commit(true) + if err == nil { + err = cerr + } + }() + + // Keep assigning new tasks until the sync completes or aborts + for s.sched.Pending() > 0 { + if err = s.commit(false); err != nil { + return err + } + s.assignTasks() + // Tasks assigned, wait for something to happen + select { + case <-newPeer: + // New peer arrived, try to assign it download tasks + + case <-s.cancel: + return errCancelStateFetch + + case <-s.d.cancelCh: + return errCanceled + + case req := <-s.deliver: + // Response, disconnect or timeout triggered, drop the peer if stalling + log.Trace("Received node data response", "peer", req.peer.id, "count", len(req.response), "dropped", req.dropped, "timeout", !req.dropped && req.timedOut()) + if req.nItems <= 2 && !req.dropped && req.timedOut() { + // 2 items are the minimum requested, if even that times out, we've no use of + // this peer at the moment. + log.Warn("Stalling state sync, dropping peer", "peer", req.peer.id) + if s.d.dropPeer == nil { + // The dropPeer method is nil when `--copydb` is used for a local copy. + // Timeouts can occur if e.g. compaction hits at the wrong time, and can be ignored + req.peer.log.Warn("Downloader wants to drop peer, but peerdrop-function is not set", "peer", req.peer.id) + } else { + s.d.dropPeer(req.peer.id) + + // If this peer was the master peer, abort sync immediately + s.d.cancelLock.RLock() + master := req.peer.id == s.d.cancelPeer + s.d.cancelLock.RUnlock() + + if master { + s.d.cancel() + return errTimeout + } + } + } + // Process all the received blobs and check for stale delivery + delivered, err := s.process(req) + req.peer.SetNodeDataIdle(delivered, req.delivered) + if err != nil { + log.Warn("Node data write error", "err", err) + return err + } + } + } + return nil +} + +func (s *stateSync) commit(force bool) error { + if !force && s.bytesUncommitted < ethdb.IdealBatchSize { + return nil + } + start := time.Now() + b := s.d.stateDB.NewBatch() + if err := s.sched.Commit(b); err != nil { + return err + } + if err := b.Write(); err != nil { + return fmt.Errorf("DB write error: %v", err) + } + s.updateStats(s.numUncommitted, 0, 0, time.Since(start)) + s.numUncommitted = 0 + s.bytesUncommitted = 0 + return nil +} + +// assignTasks attempts to assign new tasks to all idle peers, either from the +// batch currently being retried, or fetching new data from the trie sync itself. +func (s *stateSync) assignTasks() { + // Iterate over all idle peers and try to assign them state fetches + peers, _ := s.d.peers.NodeDataIdlePeers() + for _, p := range peers { + // Assign a batch of fetches proportional to the estimated latency/bandwidth + cap := p.NodeDataCapacity(s.d.peers.rates.TargetRoundTrip()) + req := &stateReq{peer: p, timeout: s.d.peers.rates.TargetTimeout()} + + nodes, _, codes := s.fillTasks(cap, req) + + // If the peer was assigned tasks to fetch, send the network request + if len(nodes)+len(codes) > 0 { + req.peer.log.Trace("Requesting batch of state data", "nodes", len(nodes), "codes", len(codes), "root", s.root) + select { + case s.d.trackStateReq <- req: + req.peer.FetchNodeData(append(nodes, codes...)) // Unified retrieval under eth/6x + case <-s.cancel: + case <-s.d.cancelCh: + } + } + } +} + +// fillTasks fills the given request object with a maximum of n state download +// tasks to send to the remote peer. +func (s *stateSync) fillTasks(n int, req *stateReq) (nodes []common.Hash, paths []trie.SyncPath, codes []common.Hash) { + // Refill available tasks from the scheduler. + if fill := n - (len(s.trieTasks) + len(s.codeTasks)); fill > 0 { + nodes, paths, codes := s.sched.Missing(fill) + for i, hash := range nodes { + s.trieTasks[hash] = &trieTask{ + path: paths[i], + attempts: make(map[string]struct{}), + } + } + for _, hash := range codes { + s.codeTasks[hash] = &codeTask{ + attempts: make(map[string]struct{}), + } + } + } + // Find tasks that haven't been tried with the request's peer. Prefer code + // over trie nodes as those can be written to disk and forgotten about. + nodes = make([]common.Hash, 0, n) + paths = make([]trie.SyncPath, 0, n) + codes = make([]common.Hash, 0, n) + + req.trieTasks = make(map[common.Hash]*trieTask, n) + req.codeTasks = make(map[common.Hash]*codeTask, n) + + for hash, t := range s.codeTasks { + // Stop when we've gathered enough requests + if len(nodes)+len(codes) == n { + break + } + // Skip any requests we've already tried from this peer + if _, ok := t.attempts[req.peer.id]; ok { + continue + } + // Assign the request to this peer + t.attempts[req.peer.id] = struct{}{} + codes = append(codes, hash) + req.codeTasks[hash] = t + delete(s.codeTasks, hash) + } + for hash, t := range s.trieTasks { + // Stop when we've gathered enough requests + if len(nodes)+len(codes) == n { + break + } + // Skip any requests we've already tried from this peer + if _, ok := t.attempts[req.peer.id]; ok { + continue + } + // Assign the request to this peer + t.attempts[req.peer.id] = struct{}{} + + nodes = append(nodes, hash) + paths = append(paths, t.path) + + req.trieTasks[hash] = t + delete(s.trieTasks, hash) + } + req.nItems = uint16(len(nodes) + len(codes)) + return nodes, paths, codes +} + +// process iterates over a batch of delivered state data, injecting each item +// into a running state sync, re-queuing any items that were requested but not +// delivered. Returns whether the peer actually managed to deliver anything of +// value, and any error that occurred. +func (s *stateSync) process(req *stateReq) (int, error) { + // Collect processing stats and update progress if valid data was received + duplicate, unexpected, successful := 0, 0, 0 + + defer func(start time.Time) { + if duplicate > 0 || unexpected > 0 { + s.updateStats(0, duplicate, unexpected, time.Since(start)) + } + }(time.Now()) + + // Iterate over all the delivered data and inject one-by-one into the trie + for _, blob := range req.response { + hash, err := s.processNodeData(blob) + switch err { + case nil: + s.numUncommitted++ + s.bytesUncommitted += len(blob) + successful++ + case trie.ErrNotRequested: + unexpected++ + case trie.ErrAlreadyProcessed: + duplicate++ + default: + return successful, fmt.Errorf("invalid state node %s: %v", hash.TerminalString(), err) + } + // Delete from both queues (one delivery is enough for the syncer) + delete(req.trieTasks, hash) + delete(req.codeTasks, hash) + } + // Put unfulfilled tasks back into the retry queue + npeers := s.d.peers.Len() + for hash, task := range req.trieTasks { + // If the node did deliver something, missing items may be due to a protocol + // limit or a previous timeout + delayed delivery. Both cases should permit + // the node to retry the missing items (to avoid single-peer stalls). + if len(req.response) > 0 || req.timedOut() { + delete(task.attempts, req.peer.id) + } + // If we've requested the node too many times already, it may be a malicious + // sync where nobody has the right data. Abort. + if len(task.attempts) >= npeers { + return successful, fmt.Errorf("trie node %s failed with all peers (%d tries, %d peers)", hash.TerminalString(), len(task.attempts), npeers) + } + // Missing item, place into the retry queue. + s.trieTasks[hash] = task + } + for hash, task := range req.codeTasks { + // If the node did deliver something, missing items may be due to a protocol + // limit or a previous timeout + delayed delivery. Both cases should permit + // the node to retry the missing items (to avoid single-peer stalls). + if len(req.response) > 0 || req.timedOut() { + delete(task.attempts, req.peer.id) + } + // If we've requested the node too many times already, it may be a malicious + // sync where nobody has the right data. Abort. + if len(task.attempts) >= npeers { + return successful, fmt.Errorf("byte code %s failed with all peers (%d tries, %d peers)", hash.TerminalString(), len(task.attempts), npeers) + } + // Missing item, place into the retry queue. + s.codeTasks[hash] = task + } + return successful, nil +} + +// processNodeData tries to inject a trie node data blob delivered from a remote +// peer into the state trie, returning whether anything useful was written or any +// error occurred. +func (s *stateSync) processNodeData(blob []byte) (common.Hash, error) { + res := trie.SyncResult{Data: blob} + s.keccak.Reset() + s.keccak.Write(blob) + s.keccak.Read(res.Hash[:]) + err := s.sched.Process(res) + return res.Hash, err +} + +// updateStats bumps the various state sync progress counters and displays a log +// message for the user to see. +func (s *stateSync) updateStats(written, duplicate, unexpected int, duration time.Duration) { + s.d.syncStatsLock.Lock() + defer s.d.syncStatsLock.Unlock() + + s.d.syncStatsState.pending = uint64(s.sched.Pending()) + s.d.syncStatsState.processed += uint64(written) + s.d.syncStatsState.duplicate += uint64(duplicate) + s.d.syncStatsState.unexpected += uint64(unexpected) + + if written > 0 || duplicate > 0 || unexpected > 0 { + log.Info("Imported new state entries", "count", written, "elapsed", common.PrettyDuration(duration), "processed", s.d.syncStatsState.processed, "pending", s.d.syncStatsState.pending, "trieretry", len(s.trieTasks), "coderetry", len(s.codeTasks), "duplicate", s.d.syncStatsState.duplicate, "unexpected", s.d.syncStatsState.unexpected) + } + if written > 0 { + rawdb.WriteFastTrieProgress(s.d.stateDB, s.d.syncStatsState.processed) + } +} diff --git a/les/downloader/testchain_test.go b/les/downloader/testchain_test.go new file mode 100644 index 000000000..b9865f7e0 --- /dev/null +++ b/les/downloader/testchain_test.go @@ -0,0 +1,230 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package downloader + +import ( + "fmt" + "math/big" + "sync" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/params" +) + +// Test chain parameters. +var ( + testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + testAddress = crypto.PubkeyToAddress(testKey.PublicKey) + testDB = rawdb.NewMemoryDatabase() + testGenesis = core.GenesisBlockForTesting(testDB, testAddress, big.NewInt(1000000000000000)) +) + +// The common prefix of all test chains: +var testChainBase = newTestChain(blockCacheMaxItems+200, testGenesis) + +// Different forks on top of the base chain: +var testChainForkLightA, testChainForkLightB, testChainForkHeavy *testChain + +func init() { + var forkLen = int(fullMaxForkAncestry + 50) + var wg sync.WaitGroup + wg.Add(3) + go func() { testChainForkLightA = testChainBase.makeFork(forkLen, false, 1); wg.Done() }() + go func() { testChainForkLightB = testChainBase.makeFork(forkLen, false, 2); wg.Done() }() + go func() { testChainForkHeavy = testChainBase.makeFork(forkLen, true, 3); wg.Done() }() + wg.Wait() +} + +type testChain struct { + genesis *types.Block + chain []common.Hash + headerm map[common.Hash]*types.Header + blockm map[common.Hash]*types.Block + receiptm map[common.Hash][]*types.Receipt + tdm map[common.Hash]*big.Int +} + +// newTestChain creates a blockchain of the given length. +func newTestChain(length int, genesis *types.Block) *testChain { + tc := new(testChain).copy(length) + tc.genesis = genesis + tc.chain = append(tc.chain, genesis.Hash()) + tc.headerm[tc.genesis.Hash()] = tc.genesis.Header() + tc.tdm[tc.genesis.Hash()] = tc.genesis.Difficulty() + tc.blockm[tc.genesis.Hash()] = tc.genesis + tc.generate(length-1, 0, genesis, false) + return tc +} + +// makeFork creates a fork on top of the test chain. +func (tc *testChain) makeFork(length int, heavy bool, seed byte) *testChain { + fork := tc.copy(tc.len() + length) + fork.generate(length, seed, tc.headBlock(), heavy) + return fork +} + +// shorten creates a copy of the chain with the given length. It panics if the +// length is longer than the number of available blocks. +func (tc *testChain) shorten(length int) *testChain { + if length > tc.len() { + panic(fmt.Errorf("can't shorten test chain to %d blocks, it's only %d blocks long", length, tc.len())) + } + return tc.copy(length) +} + +func (tc *testChain) copy(newlen int) *testChain { + cpy := &testChain{ + genesis: tc.genesis, + headerm: make(map[common.Hash]*types.Header, newlen), + blockm: make(map[common.Hash]*types.Block, newlen), + receiptm: make(map[common.Hash][]*types.Receipt, newlen), + tdm: make(map[common.Hash]*big.Int, newlen), + } + for i := 0; i < len(tc.chain) && i < newlen; i++ { + hash := tc.chain[i] + cpy.chain = append(cpy.chain, tc.chain[i]) + cpy.tdm[hash] = tc.tdm[hash] + cpy.blockm[hash] = tc.blockm[hash] + cpy.headerm[hash] = tc.headerm[hash] + cpy.receiptm[hash] = tc.receiptm[hash] + } + return cpy +} + +// generate creates a chain of n blocks starting at and including parent. +// the returned hash chain is ordered head->parent. In addition, every 22th block +// contains a transaction and every 5th an uncle to allow testing correct block +// reassembly. +func (tc *testChain) generate(n int, seed byte, parent *types.Block, heavy bool) { + // start := time.Now() + // defer func() { fmt.Printf("test chain generated in %v\n", time.Since(start)) }() + + blocks, receipts := core.GenerateChain(params.TestChainConfig, parent, ethash.NewFaker(), testDB, n, func(i int, block *core.BlockGen) { + block.SetCoinbase(common.Address{seed}) + // If a heavy chain is requested, delay blocks to raise difficulty + if heavy { + block.OffsetTime(-1) + } + // Include transactions to the miner to make blocks more interesting. + if parent == tc.genesis && i%22 == 0 { + signer := types.MakeSigner(params.TestChainConfig, block.Number()) + tx, err := types.SignTx(types.NewTransaction(block.TxNonce(testAddress), common.Address{seed}, big.NewInt(1000), params.TxGas, block.BaseFee(), nil), signer, testKey) + if err != nil { + panic(err) + } + block.AddTx(tx) + } + // if the block number is a multiple of 5, add a bonus uncle to the block + if i > 0 && i%5 == 0 { + block.AddUncle(&types.Header{ + ParentHash: block.PrevBlock(i - 1).Hash(), + Number: big.NewInt(block.Number().Int64() - 1), + }) + } + }) + + // Convert the block-chain into a hash-chain and header/block maps + td := new(big.Int).Set(tc.td(parent.Hash())) + for i, b := range blocks { + td := td.Add(td, b.Difficulty()) + hash := b.Hash() + tc.chain = append(tc.chain, hash) + tc.blockm[hash] = b + tc.headerm[hash] = b.Header() + tc.receiptm[hash] = receipts[i] + tc.tdm[hash] = new(big.Int).Set(td) + } +} + +// len returns the total number of blocks in the chain. +func (tc *testChain) len() int { + return len(tc.chain) +} + +// headBlock returns the head of the chain. +func (tc *testChain) headBlock() *types.Block { + return tc.blockm[tc.chain[len(tc.chain)-1]] +} + +// td returns the total difficulty of the given block. +func (tc *testChain) td(hash common.Hash) *big.Int { + return tc.tdm[hash] +} + +// headersByHash returns headers in order from the given hash. +func (tc *testChain) headersByHash(origin common.Hash, amount int, skip int, reverse bool) []*types.Header { + num, _ := tc.hashToNumber(origin) + return tc.headersByNumber(num, amount, skip, reverse) +} + +// headersByNumber returns headers from the given number. +func (tc *testChain) headersByNumber(origin uint64, amount int, skip int, reverse bool) []*types.Header { + result := make([]*types.Header, 0, amount) + + if !reverse { + for num := origin; num < uint64(len(tc.chain)) && len(result) < amount; num += uint64(skip) + 1 { + if header, ok := tc.headerm[tc.chain[int(num)]]; ok { + result = append(result, header) + } + } + } else { + for num := int64(origin); num >= 0 && len(result) < amount; num -= int64(skip) + 1 { + if header, ok := tc.headerm[tc.chain[int(num)]]; ok { + result = append(result, header) + } + } + } + return result +} + +// receipts returns the receipts of the given block hashes. +func (tc *testChain) receipts(hashes []common.Hash) [][]*types.Receipt { + results := make([][]*types.Receipt, 0, len(hashes)) + for _, hash := range hashes { + if receipt, ok := tc.receiptm[hash]; ok { + results = append(results, receipt) + } + } + return results +} + +// bodies returns the block bodies of the given block hashes. +func (tc *testChain) bodies(hashes []common.Hash) ([][]*types.Transaction, [][]*types.Header) { + transactions := make([][]*types.Transaction, 0, len(hashes)) + uncles := make([][]*types.Header, 0, len(hashes)) + for _, hash := range hashes { + if block, ok := tc.blockm[hash]; ok { + transactions = append(transactions, block.Transactions()) + uncles = append(uncles, block.Uncles()) + } + } + return transactions, uncles +} + +func (tc *testChain) hashToNumber(target common.Hash) (uint64, bool) { + for num, hash := range tc.chain { + if hash == target { + return uint64(num), true + } + } + return 0, false +} diff --git a/les/downloader/types.go b/les/downloader/types.go new file mode 100644 index 000000000..ff70bfa0e --- /dev/null +++ b/les/downloader/types.go @@ -0,0 +1,79 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package downloader + +import ( + "fmt" + + "github.com/ethereum/go-ethereum/core/types" +) + +// peerDropFn is a callback type for dropping a peer detected as malicious. +type peerDropFn func(id string) + +// dataPack is a data message returned by a peer for some query. +type dataPack interface { + PeerId() string + Items() int + Stats() string +} + +// headerPack is a batch of block headers returned by a peer. +type headerPack struct { + peerID string + headers []*types.Header +} + +func (p *headerPack) PeerId() string { return p.peerID } +func (p *headerPack) Items() int { return len(p.headers) } +func (p *headerPack) Stats() string { return fmt.Sprintf("%d", len(p.headers)) } + +// bodyPack is a batch of block bodies returned by a peer. +type bodyPack struct { + peerID string + transactions [][]*types.Transaction + uncles [][]*types.Header +} + +func (p *bodyPack) PeerId() string { return p.peerID } +func (p *bodyPack) Items() int { + if len(p.transactions) <= len(p.uncles) { + return len(p.transactions) + } + return len(p.uncles) +} +func (p *bodyPack) Stats() string { return fmt.Sprintf("%d:%d", len(p.transactions), len(p.uncles)) } + +// receiptPack is a batch of receipts returned by a peer. +type receiptPack struct { + peerID string + receipts [][]*types.Receipt +} + +func (p *receiptPack) PeerId() string { return p.peerID } +func (p *receiptPack) Items() int { return len(p.receipts) } +func (p *receiptPack) Stats() string { return fmt.Sprintf("%d", len(p.receipts)) } + +// statePack is a batch of states returned by a peer. +type statePack struct { + peerID string + states [][]byte +} + +func (p *statePack) PeerId() string { return p.peerID } +func (p *statePack) Items() int { return len(p.states) } +func (p *statePack) Stats() string { return fmt.Sprintf("%d", len(p.states)) } diff --git a/les/fetcher.go b/les/fetcher.go index 5eea99674..d944d3285 100644 --- a/les/fetcher.go +++ b/les/fetcher.go @@ -27,8 +27,8 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/eth/fetcher" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/les/fetcher" "github.com/ethereum/go-ethereum/light" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p/enode" diff --git a/les/fetcher/block_fetcher.go b/les/fetcher/block_fetcher.go new file mode 100644 index 000000000..283008db0 --- /dev/null +++ b/les/fetcher/block_fetcher.go @@ -0,0 +1,889 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// This is a temporary package whilst working on the eth/66 blocking refactors. +// After that work is done, les needs to be refactored to use the new package, +// or alternatively use a stripped down version of it. Either way, we need to +// keep the changes scoped so duplicating temporarily seems the sanest. +package fetcher + +import ( + "errors" + "math/rand" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/prque" + "github.com/ethereum/go-ethereum/consensus" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" + "github.com/ethereum/go-ethereum/trie" +) + +const ( + lightTimeout = time.Millisecond // Time allowance before an announced header is explicitly requested + arriveTimeout = 500 * time.Millisecond // Time allowance before an announced block/transaction is explicitly requested + gatherSlack = 100 * time.Millisecond // Interval used to collate almost-expired announces with fetches + fetchTimeout = 5 * time.Second // Maximum allotted time to return an explicitly requested block/transaction +) + +const ( + maxUncleDist = 7 // Maximum allowed backward distance from the chain head + maxQueueDist = 32 // Maximum allowed distance from the chain head to queue + hashLimit = 256 // Maximum number of unique blocks or headers a peer may have announced + blockLimit = 64 // Maximum number of unique blocks a peer may have delivered +) + +var ( + blockAnnounceInMeter = metrics.NewRegisteredMeter("eth/fetcher/block/announces/in", nil) + blockAnnounceOutTimer = metrics.NewRegisteredTimer("eth/fetcher/block/announces/out", nil) + blockAnnounceDropMeter = metrics.NewRegisteredMeter("eth/fetcher/block/announces/drop", nil) + blockAnnounceDOSMeter = metrics.NewRegisteredMeter("eth/fetcher/block/announces/dos", nil) + + blockBroadcastInMeter = metrics.NewRegisteredMeter("eth/fetcher/block/broadcasts/in", nil) + blockBroadcastOutTimer = metrics.NewRegisteredTimer("eth/fetcher/block/broadcasts/out", nil) + blockBroadcastDropMeter = metrics.NewRegisteredMeter("eth/fetcher/block/broadcasts/drop", nil) + blockBroadcastDOSMeter = metrics.NewRegisteredMeter("eth/fetcher/block/broadcasts/dos", nil) + + headerFetchMeter = metrics.NewRegisteredMeter("eth/fetcher/block/headers", nil) + bodyFetchMeter = metrics.NewRegisteredMeter("eth/fetcher/block/bodies", nil) + + headerFilterInMeter = metrics.NewRegisteredMeter("eth/fetcher/block/filter/headers/in", nil) + headerFilterOutMeter = metrics.NewRegisteredMeter("eth/fetcher/block/filter/headers/out", nil) + bodyFilterInMeter = metrics.NewRegisteredMeter("eth/fetcher/block/filter/bodies/in", nil) + bodyFilterOutMeter = metrics.NewRegisteredMeter("eth/fetcher/block/filter/bodies/out", nil) +) + +var errTerminated = errors.New("terminated") + +// HeaderRetrievalFn is a callback type for retrieving a header from the local chain. +type HeaderRetrievalFn func(common.Hash) *types.Header + +// blockRetrievalFn is a callback type for retrieving a block from the local chain. +type blockRetrievalFn func(common.Hash) *types.Block + +// headerRequesterFn is a callback type for sending a header retrieval request. +type headerRequesterFn func(common.Hash) error + +// bodyRequesterFn is a callback type for sending a body retrieval request. +type bodyRequesterFn func([]common.Hash) error + +// headerVerifierFn is a callback type to verify a block's header for fast propagation. +type headerVerifierFn func(header *types.Header) error + +// blockBroadcasterFn is a callback type for broadcasting a block to connected peers. +type blockBroadcasterFn func(block *types.Block, propagate bool) + +// chainHeightFn is a callback type to retrieve the current chain height. +type chainHeightFn func() uint64 + +// headersInsertFn is a callback type to insert a batch of headers into the local chain. +type headersInsertFn func(headers []*types.Header) (int, error) + +// chainInsertFn is a callback type to insert a batch of blocks into the local chain. +type chainInsertFn func(types.Blocks) (int, error) + +// peerDropFn is a callback type for dropping a peer detected as malicious. +type peerDropFn func(id string) + +// blockAnnounce is the hash notification of the availability of a new block in the +// network. +type blockAnnounce struct { + hash common.Hash // Hash of the block being announced + number uint64 // Number of the block being announced (0 = unknown | old protocol) + header *types.Header // Header of the block partially reassembled (new protocol) + time time.Time // Timestamp of the announcement + + origin string // Identifier of the peer originating the notification + + fetchHeader headerRequesterFn // Fetcher function to retrieve the header of an announced block + fetchBodies bodyRequesterFn // Fetcher function to retrieve the body of an announced block +} + +// headerFilterTask represents a batch of headers needing fetcher filtering. +type headerFilterTask struct { + peer string // The source peer of block headers + headers []*types.Header // Collection of headers to filter + time time.Time // Arrival time of the headers +} + +// bodyFilterTask represents a batch of block bodies (transactions and uncles) +// needing fetcher filtering. +type bodyFilterTask struct { + peer string // The source peer of block bodies + transactions [][]*types.Transaction // Collection of transactions per block bodies + uncles [][]*types.Header // Collection of uncles per block bodies + time time.Time // Arrival time of the blocks' contents +} + +// blockOrHeaderInject represents a schedules import operation. +type blockOrHeaderInject struct { + origin string + + header *types.Header // Used for light mode fetcher which only cares about header. + block *types.Block // Used for normal mode fetcher which imports full block. +} + +// number returns the block number of the injected object. +func (inject *blockOrHeaderInject) number() uint64 { + if inject.header != nil { + return inject.header.Number.Uint64() + } + return inject.block.NumberU64() +} + +// number returns the block hash of the injected object. +func (inject *blockOrHeaderInject) hash() common.Hash { + if inject.header != nil { + return inject.header.Hash() + } + return inject.block.Hash() +} + +// BlockFetcher is responsible for accumulating block announcements from various peers +// and scheduling them for retrieval. +type BlockFetcher struct { + light bool // The indicator whether it's a light fetcher or normal one. + + // Various event channels + notify chan *blockAnnounce + inject chan *blockOrHeaderInject + + headerFilter chan chan *headerFilterTask + bodyFilter chan chan *bodyFilterTask + + done chan common.Hash + quit chan struct{} + + // Announce states + announces map[string]int // Per peer blockAnnounce counts to prevent memory exhaustion + announced map[common.Hash][]*blockAnnounce // Announced blocks, scheduled for fetching + fetching map[common.Hash]*blockAnnounce // Announced blocks, currently fetching + fetched map[common.Hash][]*blockAnnounce // Blocks with headers fetched, scheduled for body retrieval + completing map[common.Hash]*blockAnnounce // Blocks with headers, currently body-completing + + // Block cache + queue *prque.Prque // Queue containing the import operations (block number sorted) + queues map[string]int // Per peer block counts to prevent memory exhaustion + queued map[common.Hash]*blockOrHeaderInject // Set of already queued blocks (to dedup imports) + + // Callbacks + getHeader HeaderRetrievalFn // Retrieves a header from the local chain + getBlock blockRetrievalFn // Retrieves a block from the local chain + verifyHeader headerVerifierFn // Checks if a block's headers have a valid proof of work + broadcastBlock blockBroadcasterFn // Broadcasts a block to connected peers + chainHeight chainHeightFn // Retrieves the current chain's height + insertHeaders headersInsertFn // Injects a batch of headers into the chain + insertChain chainInsertFn // Injects a batch of blocks into the chain + dropPeer peerDropFn // Drops a peer for misbehaving + + // Testing hooks + announceChangeHook func(common.Hash, bool) // Method to call upon adding or deleting a hash from the blockAnnounce list + queueChangeHook func(common.Hash, bool) // Method to call upon adding or deleting a block from the import queue + fetchingHook func([]common.Hash) // Method to call upon starting a block (eth/61) or header (eth/62) fetch + completingHook func([]common.Hash) // Method to call upon starting a block body fetch (eth/62) + importedHook func(*types.Header, *types.Block) // Method to call upon successful header or block import (both eth/61 and eth/62) +} + +// NewBlockFetcher creates a block fetcher to retrieve blocks based on hash announcements. +func NewBlockFetcher(light bool, getHeader HeaderRetrievalFn, getBlock blockRetrievalFn, verifyHeader headerVerifierFn, broadcastBlock blockBroadcasterFn, chainHeight chainHeightFn, insertHeaders headersInsertFn, insertChain chainInsertFn, dropPeer peerDropFn) *BlockFetcher { + return &BlockFetcher{ + light: light, + notify: make(chan *blockAnnounce), + inject: make(chan *blockOrHeaderInject), + headerFilter: make(chan chan *headerFilterTask), + bodyFilter: make(chan chan *bodyFilterTask), + done: make(chan common.Hash), + quit: make(chan struct{}), + announces: make(map[string]int), + announced: make(map[common.Hash][]*blockAnnounce), + fetching: make(map[common.Hash]*blockAnnounce), + fetched: make(map[common.Hash][]*blockAnnounce), + completing: make(map[common.Hash]*blockAnnounce), + queue: prque.New(nil), + queues: make(map[string]int), + queued: make(map[common.Hash]*blockOrHeaderInject), + getHeader: getHeader, + getBlock: getBlock, + verifyHeader: verifyHeader, + broadcastBlock: broadcastBlock, + chainHeight: chainHeight, + insertHeaders: insertHeaders, + insertChain: insertChain, + dropPeer: dropPeer, + } +} + +// Start boots up the announcement based synchroniser, accepting and processing +// hash notifications and block fetches until termination requested. +func (f *BlockFetcher) Start() { + go f.loop() +} + +// Stop terminates the announcement based synchroniser, canceling all pending +// operations. +func (f *BlockFetcher) Stop() { + close(f.quit) +} + +// Notify announces the fetcher of the potential availability of a new block in +// the network. +func (f *BlockFetcher) Notify(peer string, hash common.Hash, number uint64, time time.Time, + headerFetcher headerRequesterFn, bodyFetcher bodyRequesterFn) error { + block := &blockAnnounce{ + hash: hash, + number: number, + time: time, + origin: peer, + fetchHeader: headerFetcher, + fetchBodies: bodyFetcher, + } + select { + case f.notify <- block: + return nil + case <-f.quit: + return errTerminated + } +} + +// Enqueue tries to fill gaps the fetcher's future import queue. +func (f *BlockFetcher) Enqueue(peer string, block *types.Block) error { + op := &blockOrHeaderInject{ + origin: peer, + block: block, + } + select { + case f.inject <- op: + return nil + case <-f.quit: + return errTerminated + } +} + +// FilterHeaders extracts all the headers that were explicitly requested by the fetcher, +// returning those that should be handled differently. +func (f *BlockFetcher) FilterHeaders(peer string, headers []*types.Header, time time.Time) []*types.Header { + log.Trace("Filtering headers", "peer", peer, "headers", len(headers)) + + // Send the filter channel to the fetcher + filter := make(chan *headerFilterTask) + + select { + case f.headerFilter <- filter: + case <-f.quit: + return nil + } + // Request the filtering of the header list + select { + case filter <- &headerFilterTask{peer: peer, headers: headers, time: time}: + case <-f.quit: + return nil + } + // Retrieve the headers remaining after filtering + select { + case task := <-filter: + return task.headers + case <-f.quit: + return nil + } +} + +// FilterBodies extracts all the block bodies that were explicitly requested by +// the fetcher, returning those that should be handled differently. +func (f *BlockFetcher) FilterBodies(peer string, transactions [][]*types.Transaction, uncles [][]*types.Header, time time.Time) ([][]*types.Transaction, [][]*types.Header) { + log.Trace("Filtering bodies", "peer", peer, "txs", len(transactions), "uncles", len(uncles)) + + // Send the filter channel to the fetcher + filter := make(chan *bodyFilterTask) + + select { + case f.bodyFilter <- filter: + case <-f.quit: + return nil, nil + } + // Request the filtering of the body list + select { + case filter <- &bodyFilterTask{peer: peer, transactions: transactions, uncles: uncles, time: time}: + case <-f.quit: + return nil, nil + } + // Retrieve the bodies remaining after filtering + select { + case task := <-filter: + return task.transactions, task.uncles + case <-f.quit: + return nil, nil + } +} + +// Loop is the main fetcher loop, checking and processing various notification +// events. +func (f *BlockFetcher) loop() { + // Iterate the block fetching until a quit is requested + var ( + fetchTimer = time.NewTimer(0) + completeTimer = time.NewTimer(0) + ) + <-fetchTimer.C // clear out the channel + <-completeTimer.C + defer fetchTimer.Stop() + defer completeTimer.Stop() + + for { + // Clean up any expired block fetches + for hash, announce := range f.fetching { + if time.Since(announce.time) > fetchTimeout { + f.forgetHash(hash) + } + } + // Import any queued blocks that could potentially fit + height := f.chainHeight() + for !f.queue.Empty() { + op := f.queue.PopItem().(*blockOrHeaderInject) + hash := op.hash() + if f.queueChangeHook != nil { + f.queueChangeHook(hash, false) + } + // If too high up the chain or phase, continue later + number := op.number() + if number > height+1 { + f.queue.Push(op, -int64(number)) + if f.queueChangeHook != nil { + f.queueChangeHook(hash, true) + } + break + } + // Otherwise if fresh and still unknown, try and import + if (number+maxUncleDist < height) || (f.light && f.getHeader(hash) != nil) || (!f.light && f.getBlock(hash) != nil) { + f.forgetBlock(hash) + continue + } + if f.light { + f.importHeaders(op.origin, op.header) + } else { + f.importBlocks(op.origin, op.block) + } + } + // Wait for an outside event to occur + select { + case <-f.quit: + // BlockFetcher terminating, abort all operations + return + + case notification := <-f.notify: + // A block was announced, make sure the peer isn't DOSing us + blockAnnounceInMeter.Mark(1) + + count := f.announces[notification.origin] + 1 + if count > hashLimit { + log.Debug("Peer exceeded outstanding announces", "peer", notification.origin, "limit", hashLimit) + blockAnnounceDOSMeter.Mark(1) + break + } + // If we have a valid block number, check that it's potentially useful + if notification.number > 0 { + if dist := int64(notification.number) - int64(f.chainHeight()); dist < -maxUncleDist || dist > maxQueueDist { + log.Debug("Peer discarded announcement", "peer", notification.origin, "number", notification.number, "hash", notification.hash, "distance", dist) + blockAnnounceDropMeter.Mark(1) + break + } + } + // All is well, schedule the announce if block's not yet downloading + if _, ok := f.fetching[notification.hash]; ok { + break + } + if _, ok := f.completing[notification.hash]; ok { + break + } + f.announces[notification.origin] = count + f.announced[notification.hash] = append(f.announced[notification.hash], notification) + if f.announceChangeHook != nil && len(f.announced[notification.hash]) == 1 { + f.announceChangeHook(notification.hash, true) + } + if len(f.announced) == 1 { + f.rescheduleFetch(fetchTimer) + } + + case op := <-f.inject: + // A direct block insertion was requested, try and fill any pending gaps + blockBroadcastInMeter.Mark(1) + + // Now only direct block injection is allowed, drop the header injection + // here silently if we receive. + if f.light { + continue + } + f.enqueue(op.origin, nil, op.block) + + case hash := <-f.done: + // A pending import finished, remove all traces of the notification + f.forgetHash(hash) + f.forgetBlock(hash) + + case <-fetchTimer.C: + // At least one block's timer ran out, check for needing retrieval + request := make(map[string][]common.Hash) + + for hash, announces := range f.announced { + // In current LES protocol(les2/les3), only header announce is + // available, no need to wait too much time for header broadcast. + timeout := arriveTimeout - gatherSlack + if f.light { + timeout = 0 + } + if time.Since(announces[0].time) > timeout { + // Pick a random peer to retrieve from, reset all others + announce := announces[rand.Intn(len(announces))] + f.forgetHash(hash) + + // If the block still didn't arrive, queue for fetching + if (f.light && f.getHeader(hash) == nil) || (!f.light && f.getBlock(hash) == nil) { + request[announce.origin] = append(request[announce.origin], hash) + f.fetching[hash] = announce + } + } + } + // Send out all block header requests + for peer, hashes := range request { + log.Trace("Fetching scheduled headers", "peer", peer, "list", hashes) + + // Create a closure of the fetch and schedule in on a new thread + fetchHeader, hashes := f.fetching[hashes[0]].fetchHeader, hashes + go func() { + if f.fetchingHook != nil { + f.fetchingHook(hashes) + } + for _, hash := range hashes { + headerFetchMeter.Mark(1) + fetchHeader(hash) // Suboptimal, but protocol doesn't allow batch header retrievals + } + }() + } + // Schedule the next fetch if blocks are still pending + f.rescheduleFetch(fetchTimer) + + case <-completeTimer.C: + // At least one header's timer ran out, retrieve everything + request := make(map[string][]common.Hash) + + for hash, announces := range f.fetched { + // Pick a random peer to retrieve from, reset all others + announce := announces[rand.Intn(len(announces))] + f.forgetHash(hash) + + // If the block still didn't arrive, queue for completion + if f.getBlock(hash) == nil { + request[announce.origin] = append(request[announce.origin], hash) + f.completing[hash] = announce + } + } + // Send out all block body requests + for peer, hashes := range request { + log.Trace("Fetching scheduled bodies", "peer", peer, "list", hashes) + + // Create a closure of the fetch and schedule in on a new thread + if f.completingHook != nil { + f.completingHook(hashes) + } + bodyFetchMeter.Mark(int64(len(hashes))) + go f.completing[hashes[0]].fetchBodies(hashes) + } + // Schedule the next fetch if blocks are still pending + f.rescheduleComplete(completeTimer) + + case filter := <-f.headerFilter: + // Headers arrived from a remote peer. Extract those that were explicitly + // requested by the fetcher, and return everything else so it's delivered + // to other parts of the system. + var task *headerFilterTask + select { + case task = <-filter: + case <-f.quit: + return + } + headerFilterInMeter.Mark(int64(len(task.headers))) + + // Split the batch of headers into unknown ones (to return to the caller), + // known incomplete ones (requiring body retrievals) and completed blocks. + unknown, incomplete, complete, lightHeaders := []*types.Header{}, []*blockAnnounce{}, []*types.Block{}, []*blockAnnounce{} + for _, header := range task.headers { + hash := header.Hash() + + // Filter fetcher-requested headers from other synchronisation algorithms + if announce := f.fetching[hash]; announce != nil && announce.origin == task.peer && f.fetched[hash] == nil && f.completing[hash] == nil && f.queued[hash] == nil { + // If the delivered header does not match the promised number, drop the announcer + if header.Number.Uint64() != announce.number { + log.Trace("Invalid block number fetched", "peer", announce.origin, "hash", header.Hash(), "announced", announce.number, "provided", header.Number) + f.dropPeer(announce.origin) + f.forgetHash(hash) + continue + } + // Collect all headers only if we are running in light + // mode and the headers are not imported by other means. + if f.light { + if f.getHeader(hash) == nil { + announce.header = header + lightHeaders = append(lightHeaders, announce) + } + f.forgetHash(hash) + continue + } + // Only keep if not imported by other means + if f.getBlock(hash) == nil { + announce.header = header + announce.time = task.time + + // If the block is empty (header only), short circuit into the final import queue + if header.TxHash == types.EmptyRootHash && header.UncleHash == types.EmptyUncleHash { + log.Trace("Block empty, skipping body retrieval", "peer", announce.origin, "number", header.Number, "hash", header.Hash()) + + block := types.NewBlockWithHeader(header) + block.ReceivedAt = task.time + + complete = append(complete, block) + f.completing[hash] = announce + continue + } + // Otherwise add to the list of blocks needing completion + incomplete = append(incomplete, announce) + } else { + log.Trace("Block already imported, discarding header", "peer", announce.origin, "number", header.Number, "hash", header.Hash()) + f.forgetHash(hash) + } + } else { + // BlockFetcher doesn't know about it, add to the return list + unknown = append(unknown, header) + } + } + headerFilterOutMeter.Mark(int64(len(unknown))) + select { + case filter <- &headerFilterTask{headers: unknown, time: task.time}: + case <-f.quit: + return + } + // Schedule the retrieved headers for body completion + for _, announce := range incomplete { + hash := announce.header.Hash() + if _, ok := f.completing[hash]; ok { + continue + } + f.fetched[hash] = append(f.fetched[hash], announce) + if len(f.fetched) == 1 { + f.rescheduleComplete(completeTimer) + } + } + // Schedule the header for light fetcher import + for _, announce := range lightHeaders { + f.enqueue(announce.origin, announce.header, nil) + } + // Schedule the header-only blocks for import + for _, block := range complete { + if announce := f.completing[block.Hash()]; announce != nil { + f.enqueue(announce.origin, nil, block) + } + } + + case filter := <-f.bodyFilter: + // Block bodies arrived, extract any explicitly requested blocks, return the rest + var task *bodyFilterTask + select { + case task = <-filter: + case <-f.quit: + return + } + bodyFilterInMeter.Mark(int64(len(task.transactions))) + blocks := []*types.Block{} + // abort early if there's nothing explicitly requested + if len(f.completing) > 0 { + for i := 0; i < len(task.transactions) && i < len(task.uncles); i++ { + // Match up a body to any possible completion request + var ( + matched = false + uncleHash common.Hash // calculated lazily and reused + txnHash common.Hash // calculated lazily and reused + ) + for hash, announce := range f.completing { + if f.queued[hash] != nil || announce.origin != task.peer { + continue + } + if uncleHash == (common.Hash{}) { + uncleHash = types.CalcUncleHash(task.uncles[i]) + } + if uncleHash != announce.header.UncleHash { + continue + } + if txnHash == (common.Hash{}) { + txnHash = types.DeriveSha(types.Transactions(task.transactions[i]), trie.NewStackTrie(nil)) + } + if txnHash != announce.header.TxHash { + continue + } + // Mark the body matched, reassemble if still unknown + matched = true + if f.getBlock(hash) == nil { + block := types.NewBlockWithHeader(announce.header).WithBody(task.transactions[i], task.uncles[i]) + block.ReceivedAt = task.time + blocks = append(blocks, block) + } else { + f.forgetHash(hash) + } + + } + if matched { + task.transactions = append(task.transactions[:i], task.transactions[i+1:]...) + task.uncles = append(task.uncles[:i], task.uncles[i+1:]...) + i-- + continue + } + } + } + bodyFilterOutMeter.Mark(int64(len(task.transactions))) + select { + case filter <- task: + case <-f.quit: + return + } + // Schedule the retrieved blocks for ordered import + for _, block := range blocks { + if announce := f.completing[block.Hash()]; announce != nil { + f.enqueue(announce.origin, nil, block) + } + } + } + } +} + +// rescheduleFetch resets the specified fetch timer to the next blockAnnounce timeout. +func (f *BlockFetcher) rescheduleFetch(fetch *time.Timer) { + // Short circuit if no blocks are announced + if len(f.announced) == 0 { + return + } + // Schedule announcement retrieval quickly for light mode + // since server won't send any headers to client. + if f.light { + fetch.Reset(lightTimeout) + return + } + // Otherwise find the earliest expiring announcement + earliest := time.Now() + for _, announces := range f.announced { + if earliest.After(announces[0].time) { + earliest = announces[0].time + } + } + fetch.Reset(arriveTimeout - time.Since(earliest)) +} + +// rescheduleComplete resets the specified completion timer to the next fetch timeout. +func (f *BlockFetcher) rescheduleComplete(complete *time.Timer) { + // Short circuit if no headers are fetched + if len(f.fetched) == 0 { + return + } + // Otherwise find the earliest expiring announcement + earliest := time.Now() + for _, announces := range f.fetched { + if earliest.After(announces[0].time) { + earliest = announces[0].time + } + } + complete.Reset(gatherSlack - time.Since(earliest)) +} + +// enqueue schedules a new header or block import operation, if the component +// to be imported has not yet been seen. +func (f *BlockFetcher) enqueue(peer string, header *types.Header, block *types.Block) { + var ( + hash common.Hash + number uint64 + ) + if header != nil { + hash, number = header.Hash(), header.Number.Uint64() + } else { + hash, number = block.Hash(), block.NumberU64() + } + // Ensure the peer isn't DOSing us + count := f.queues[peer] + 1 + if count > blockLimit { + log.Debug("Discarded delivered header or block, exceeded allowance", "peer", peer, "number", number, "hash", hash, "limit", blockLimit) + blockBroadcastDOSMeter.Mark(1) + f.forgetHash(hash) + return + } + // Discard any past or too distant blocks + if dist := int64(number) - int64(f.chainHeight()); dist < -maxUncleDist || dist > maxQueueDist { + log.Debug("Discarded delivered header or block, too far away", "peer", peer, "number", number, "hash", hash, "distance", dist) + blockBroadcastDropMeter.Mark(1) + f.forgetHash(hash) + return + } + // Schedule the block for future importing + if _, ok := f.queued[hash]; !ok { + op := &blockOrHeaderInject{origin: peer} + if header != nil { + op.header = header + } else { + op.block = block + } + f.queues[peer] = count + f.queued[hash] = op + f.queue.Push(op, -int64(number)) + if f.queueChangeHook != nil { + f.queueChangeHook(hash, true) + } + log.Debug("Queued delivered header or block", "peer", peer, "number", number, "hash", hash, "queued", f.queue.Size()) + } +} + +// importHeaders spawns a new goroutine to run a header insertion into the chain. +// If the header's number is at the same height as the current import phase, it +// updates the phase states accordingly. +func (f *BlockFetcher) importHeaders(peer string, header *types.Header) { + hash := header.Hash() + log.Debug("Importing propagated header", "peer", peer, "number", header.Number, "hash", hash) + + go func() { + defer func() { f.done <- hash }() + // If the parent's unknown, abort insertion + parent := f.getHeader(header.ParentHash) + if parent == nil { + log.Debug("Unknown parent of propagated header", "peer", peer, "number", header.Number, "hash", hash, "parent", header.ParentHash) + return + } + // Validate the header and if something went wrong, drop the peer + if err := f.verifyHeader(header); err != nil && err != consensus.ErrFutureBlock { + log.Debug("Propagated header verification failed", "peer", peer, "number", header.Number, "hash", hash, "err", err) + f.dropPeer(peer) + return + } + // Run the actual import and log any issues + if _, err := f.insertHeaders([]*types.Header{header}); err != nil { + log.Debug("Propagated header import failed", "peer", peer, "number", header.Number, "hash", hash, "err", err) + return + } + // Invoke the testing hook if needed + if f.importedHook != nil { + f.importedHook(header, nil) + } + }() +} + +// importBlocks spawns a new goroutine to run a block insertion into the chain. If the +// block's number is at the same height as the current import phase, it updates +// the phase states accordingly. +func (f *BlockFetcher) importBlocks(peer string, block *types.Block) { + hash := block.Hash() + + // Run the import on a new thread + log.Debug("Importing propagated block", "peer", peer, "number", block.Number(), "hash", hash) + go func() { + defer func() { f.done <- hash }() + + // If the parent's unknown, abort insertion + parent := f.getBlock(block.ParentHash()) + if parent == nil { + log.Debug("Unknown parent of propagated block", "peer", peer, "number", block.Number(), "hash", hash, "parent", block.ParentHash()) + return + } + // Quickly validate the header and propagate the block if it passes + switch err := f.verifyHeader(block.Header()); err { + case nil: + // All ok, quickly propagate to our peers + blockBroadcastOutTimer.UpdateSince(block.ReceivedAt) + go f.broadcastBlock(block, true) + + case consensus.ErrFutureBlock: + // Weird future block, don't fail, but neither propagate + + default: + // Something went very wrong, drop the peer + log.Debug("Propagated block verification failed", "peer", peer, "number", block.Number(), "hash", hash, "err", err) + f.dropPeer(peer) + return + } + // Run the actual import and log any issues + if _, err := f.insertChain(types.Blocks{block}); err != nil { + log.Debug("Propagated block import failed", "peer", peer, "number", block.Number(), "hash", hash, "err", err) + return + } + // If import succeeded, broadcast the block + blockAnnounceOutTimer.UpdateSince(block.ReceivedAt) + go f.broadcastBlock(block, false) + + // Invoke the testing hook if needed + if f.importedHook != nil { + f.importedHook(nil, block) + } + }() +} + +// forgetHash removes all traces of a block announcement from the fetcher's +// internal state. +func (f *BlockFetcher) forgetHash(hash common.Hash) { + // Remove all pending announces and decrement DOS counters + if announceMap, ok := f.announced[hash]; ok { + for _, announce := range announceMap { + f.announces[announce.origin]-- + if f.announces[announce.origin] <= 0 { + delete(f.announces, announce.origin) + } + } + delete(f.announced, hash) + if f.announceChangeHook != nil { + f.announceChangeHook(hash, false) + } + } + // Remove any pending fetches and decrement the DOS counters + if announce := f.fetching[hash]; announce != nil { + f.announces[announce.origin]-- + if f.announces[announce.origin] <= 0 { + delete(f.announces, announce.origin) + } + delete(f.fetching, hash) + } + + // Remove any pending completion requests and decrement the DOS counters + for _, announce := range f.fetched[hash] { + f.announces[announce.origin]-- + if f.announces[announce.origin] <= 0 { + delete(f.announces, announce.origin) + } + } + delete(f.fetched, hash) + + // Remove any pending completions and decrement the DOS counters + if announce := f.completing[hash]; announce != nil { + f.announces[announce.origin]-- + if f.announces[announce.origin] <= 0 { + delete(f.announces, announce.origin) + } + delete(f.completing, hash) + } +} + +// forgetBlock removes all traces of a queued block from the fetcher's internal +// state. +func (f *BlockFetcher) forgetBlock(hash common.Hash) { + if insert := f.queued[hash]; insert != nil { + f.queues[insert.origin]-- + if f.queues[insert.origin] == 0 { + delete(f.queues, insert.origin) + } + delete(f.queued, hash) + } +} diff --git a/les/fetcher/block_fetcher_test.go b/les/fetcher/block_fetcher_test.go new file mode 100644 index 000000000..b6d1125b5 --- /dev/null +++ b/les/fetcher/block_fetcher_test.go @@ -0,0 +1,896 @@ +// Copyright 2015 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package fetcher + +import ( + "errors" + "math/big" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus/ethash" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/trie" +) + +var ( + testdb = rawdb.NewMemoryDatabase() + testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + testAddress = crypto.PubkeyToAddress(testKey.PublicKey) + genesis = core.GenesisBlockForTesting(testdb, testAddress, big.NewInt(1000000000000000)) + unknownBlock = types.NewBlock(&types.Header{GasLimit: params.GenesisGasLimit, BaseFee: big.NewInt(params.InitialBaseFee)}, nil, nil, nil, trie.NewStackTrie(nil)) +) + +// makeChain creates a chain of n blocks starting at and including parent. +// the returned hash chain is ordered head->parent. In addition, every 3rd block +// contains a transaction and every 5th an uncle to allow testing correct block +// reassembly. +func makeChain(n int, seed byte, parent *types.Block) ([]common.Hash, map[common.Hash]*types.Block) { + blocks, _ := core.GenerateChain(params.TestChainConfig, parent, ethash.NewFaker(), testdb, n, func(i int, block *core.BlockGen) { + block.SetCoinbase(common.Address{seed}) + + // If the block number is multiple of 3, send a bonus transaction to the miner + if parent == genesis && i%3 == 0 { + signer := types.MakeSigner(params.TestChainConfig, block.Number()) + tx, err := types.SignTx(types.NewTransaction(block.TxNonce(testAddress), common.Address{seed}, big.NewInt(1000), params.TxGas, block.BaseFee(), nil), signer, testKey) + if err != nil { + panic(err) + } + block.AddTx(tx) + } + // If the block number is a multiple of 5, add a bonus uncle to the block + if i%5 == 0 { + block.AddUncle(&types.Header{ParentHash: block.PrevBlock(i - 1).Hash(), Number: big.NewInt(int64(i - 1))}) + } + }) + hashes := make([]common.Hash, n+1) + hashes[len(hashes)-1] = parent.Hash() + blockm := make(map[common.Hash]*types.Block, n+1) + blockm[parent.Hash()] = parent + for i, b := range blocks { + hashes[len(hashes)-i-2] = b.Hash() + blockm[b.Hash()] = b + } + return hashes, blockm +} + +// fetcherTester is a test simulator for mocking out local block chain. +type fetcherTester struct { + fetcher *BlockFetcher + + hashes []common.Hash // Hash chain belonging to the tester + headers map[common.Hash]*types.Header // Headers belonging to the tester + blocks map[common.Hash]*types.Block // Blocks belonging to the tester + drops map[string]bool // Map of peers dropped by the fetcher + + lock sync.RWMutex +} + +// newTester creates a new fetcher test mocker. +func newTester(light bool) *fetcherTester { + tester := &fetcherTester{ + hashes: []common.Hash{genesis.Hash()}, + headers: map[common.Hash]*types.Header{genesis.Hash(): genesis.Header()}, + blocks: map[common.Hash]*types.Block{genesis.Hash(): genesis}, + drops: make(map[string]bool), + } + tester.fetcher = NewBlockFetcher(light, tester.getHeader, tester.getBlock, tester.verifyHeader, tester.broadcastBlock, tester.chainHeight, tester.insertHeaders, tester.insertChain, tester.dropPeer) + tester.fetcher.Start() + + return tester +} + +// getHeader retrieves a header from the tester's block chain. +func (f *fetcherTester) getHeader(hash common.Hash) *types.Header { + f.lock.RLock() + defer f.lock.RUnlock() + + return f.headers[hash] +} + +// getBlock retrieves a block from the tester's block chain. +func (f *fetcherTester) getBlock(hash common.Hash) *types.Block { + f.lock.RLock() + defer f.lock.RUnlock() + + return f.blocks[hash] +} + +// verifyHeader is a nop placeholder for the block header verification. +func (f *fetcherTester) verifyHeader(header *types.Header) error { + return nil +} + +// broadcastBlock is a nop placeholder for the block broadcasting. +func (f *fetcherTester) broadcastBlock(block *types.Block, propagate bool) { +} + +// chainHeight retrieves the current height (block number) of the chain. +func (f *fetcherTester) chainHeight() uint64 { + f.lock.RLock() + defer f.lock.RUnlock() + + if f.fetcher.light { + return f.headers[f.hashes[len(f.hashes)-1]].Number.Uint64() + } + return f.blocks[f.hashes[len(f.hashes)-1]].NumberU64() +} + +// insertChain injects a new headers into the simulated chain. +func (f *fetcherTester) insertHeaders(headers []*types.Header) (int, error) { + f.lock.Lock() + defer f.lock.Unlock() + + for i, header := range headers { + // Make sure the parent in known + if _, ok := f.headers[header.ParentHash]; !ok { + return i, errors.New("unknown parent") + } + // Discard any new blocks if the same height already exists + if header.Number.Uint64() <= f.headers[f.hashes[len(f.hashes)-1]].Number.Uint64() { + return i, nil + } + // Otherwise build our current chain + f.hashes = append(f.hashes, header.Hash()) + f.headers[header.Hash()] = header + } + return 0, nil +} + +// insertChain injects a new blocks into the simulated chain. +func (f *fetcherTester) insertChain(blocks types.Blocks) (int, error) { + f.lock.Lock() + defer f.lock.Unlock() + + for i, block := range blocks { + // Make sure the parent in known + if _, ok := f.blocks[block.ParentHash()]; !ok { + return i, errors.New("unknown parent") + } + // Discard any new blocks if the same height already exists + if block.NumberU64() <= f.blocks[f.hashes[len(f.hashes)-1]].NumberU64() { + return i, nil + } + // Otherwise build our current chain + f.hashes = append(f.hashes, block.Hash()) + f.blocks[block.Hash()] = block + } + return 0, nil +} + +// dropPeer is an emulator for the peer removal, simply accumulating the various +// peers dropped by the fetcher. +func (f *fetcherTester) dropPeer(peer string) { + f.lock.Lock() + defer f.lock.Unlock() + + f.drops[peer] = true +} + +// makeHeaderFetcher retrieves a block header fetcher associated with a simulated peer. +func (f *fetcherTester) makeHeaderFetcher(peer string, blocks map[common.Hash]*types.Block, drift time.Duration) headerRequesterFn { + closure := make(map[common.Hash]*types.Block) + for hash, block := range blocks { + closure[hash] = block + } + // Create a function that return a header from the closure + return func(hash common.Hash) error { + // Gather the blocks to return + headers := make([]*types.Header, 0, 1) + if block, ok := closure[hash]; ok { + headers = append(headers, block.Header()) + } + // Return on a new thread + go f.fetcher.FilterHeaders(peer, headers, time.Now().Add(drift)) + + return nil + } +} + +// makeBodyFetcher retrieves a block body fetcher associated with a simulated peer. +func (f *fetcherTester) makeBodyFetcher(peer string, blocks map[common.Hash]*types.Block, drift time.Duration) bodyRequesterFn { + closure := make(map[common.Hash]*types.Block) + for hash, block := range blocks { + closure[hash] = block + } + // Create a function that returns blocks from the closure + return func(hashes []common.Hash) error { + // Gather the block bodies to return + transactions := make([][]*types.Transaction, 0, len(hashes)) + uncles := make([][]*types.Header, 0, len(hashes)) + + for _, hash := range hashes { + if block, ok := closure[hash]; ok { + transactions = append(transactions, block.Transactions()) + uncles = append(uncles, block.Uncles()) + } + } + // Return on a new thread + go f.fetcher.FilterBodies(peer, transactions, uncles, time.Now().Add(drift)) + + return nil + } +} + +// verifyFetchingEvent verifies that one single event arrive on a fetching channel. +func verifyFetchingEvent(t *testing.T, fetching chan []common.Hash, arrive bool) { + if arrive { + select { + case <-fetching: + case <-time.After(time.Second): + t.Fatalf("fetching timeout") + } + } else { + select { + case <-fetching: + t.Fatalf("fetching invoked") + case <-time.After(10 * time.Millisecond): + } + } +} + +// verifyCompletingEvent verifies that one single event arrive on an completing channel. +func verifyCompletingEvent(t *testing.T, completing chan []common.Hash, arrive bool) { + if arrive { + select { + case <-completing: + case <-time.After(time.Second): + t.Fatalf("completing timeout") + } + } else { + select { + case <-completing: + t.Fatalf("completing invoked") + case <-time.After(10 * time.Millisecond): + } + } +} + +// verifyImportEvent verifies that one single event arrive on an import channel. +func verifyImportEvent(t *testing.T, imported chan interface{}, arrive bool) { + if arrive { + select { + case <-imported: + case <-time.After(time.Second): + t.Fatalf("import timeout") + } + } else { + select { + case <-imported: + t.Fatalf("import invoked") + case <-time.After(20 * time.Millisecond): + } + } +} + +// verifyImportCount verifies that exactly count number of events arrive on an +// import hook channel. +func verifyImportCount(t *testing.T, imported chan interface{}, count int) { + for i := 0; i < count; i++ { + select { + case <-imported: + case <-time.After(time.Second): + t.Fatalf("block %d: import timeout", i+1) + } + } + verifyImportDone(t, imported) +} + +// verifyImportDone verifies that no more events are arriving on an import channel. +func verifyImportDone(t *testing.T, imported chan interface{}) { + select { + case <-imported: + t.Fatalf("extra block imported") + case <-time.After(50 * time.Millisecond): + } +} + +// verifyChainHeight verifies the chain height is as expected. +func verifyChainHeight(t *testing.T, fetcher *fetcherTester, height uint64) { + if fetcher.chainHeight() != height { + t.Fatalf("chain height mismatch, got %d, want %d", fetcher.chainHeight(), height) + } +} + +// Tests that a fetcher accepts block/header announcements and initiates retrievals +// for them, successfully importing into the local chain. +func TestFullSequentialAnnouncements(t *testing.T) { testSequentialAnnouncements(t, false) } +func TestLightSequentialAnnouncements(t *testing.T) { testSequentialAnnouncements(t, true) } + +func testSequentialAnnouncements(t *testing.T, light bool) { + // Create a chain of blocks to import + targetBlocks := 4 * hashLimit + hashes, blocks := makeChain(targetBlocks, 0, genesis) + + tester := newTester(light) + headerFetcher := tester.makeHeaderFetcher("valid", blocks, -gatherSlack) + bodyFetcher := tester.makeBodyFetcher("valid", blocks, 0) + + // Iteratively announce blocks until all are imported + imported := make(chan interface{}) + tester.fetcher.importedHook = func(header *types.Header, block *types.Block) { + if light { + if header == nil { + t.Fatalf("Fetcher try to import empty header") + } + imported <- header + } else { + if block == nil { + t.Fatalf("Fetcher try to import empty block") + } + imported <- block + } + } + for i := len(hashes) - 2; i >= 0; i-- { + tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) + verifyImportEvent(t, imported, true) + } + verifyImportDone(t, imported) + verifyChainHeight(t, tester, uint64(len(hashes)-1)) +} + +// Tests that if blocks are announced by multiple peers (or even the same buggy +// peer), they will only get downloaded at most once. +func TestFullConcurrentAnnouncements(t *testing.T) { testConcurrentAnnouncements(t, false) } +func TestLightConcurrentAnnouncements(t *testing.T) { testConcurrentAnnouncements(t, true) } + +func testConcurrentAnnouncements(t *testing.T, light bool) { + // Create a chain of blocks to import + targetBlocks := 4 * hashLimit + hashes, blocks := makeChain(targetBlocks, 0, genesis) + + // Assemble a tester with a built in counter for the requests + tester := newTester(light) + firstHeaderFetcher := tester.makeHeaderFetcher("first", blocks, -gatherSlack) + firstBodyFetcher := tester.makeBodyFetcher("first", blocks, 0) + secondHeaderFetcher := tester.makeHeaderFetcher("second", blocks, -gatherSlack) + secondBodyFetcher := tester.makeBodyFetcher("second", blocks, 0) + + counter := uint32(0) + firstHeaderWrapper := func(hash common.Hash) error { + atomic.AddUint32(&counter, 1) + return firstHeaderFetcher(hash) + } + secondHeaderWrapper := func(hash common.Hash) error { + atomic.AddUint32(&counter, 1) + return secondHeaderFetcher(hash) + } + // Iteratively announce blocks until all are imported + imported := make(chan interface{}) + tester.fetcher.importedHook = func(header *types.Header, block *types.Block) { + if light { + if header == nil { + t.Fatalf("Fetcher try to import empty header") + } + imported <- header + } else { + if block == nil { + t.Fatalf("Fetcher try to import empty block") + } + imported <- block + } + } + for i := len(hashes) - 2; i >= 0; i-- { + tester.fetcher.Notify("first", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), firstHeaderWrapper, firstBodyFetcher) + tester.fetcher.Notify("second", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout+time.Millisecond), secondHeaderWrapper, secondBodyFetcher) + tester.fetcher.Notify("second", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout-time.Millisecond), secondHeaderWrapper, secondBodyFetcher) + verifyImportEvent(t, imported, true) + } + verifyImportDone(t, imported) + + // Make sure no blocks were retrieved twice + if int(counter) != targetBlocks { + t.Fatalf("retrieval count mismatch: have %v, want %v", counter, targetBlocks) + } + verifyChainHeight(t, tester, uint64(len(hashes)-1)) +} + +// Tests that announcements arriving while a previous is being fetched still +// results in a valid import. +func TestFullOverlappingAnnouncements(t *testing.T) { testOverlappingAnnouncements(t, false) } +func TestLightOverlappingAnnouncements(t *testing.T) { testOverlappingAnnouncements(t, true) } + +func testOverlappingAnnouncements(t *testing.T, light bool) { + // Create a chain of blocks to import + targetBlocks := 4 * hashLimit + hashes, blocks := makeChain(targetBlocks, 0, genesis) + + tester := newTester(light) + headerFetcher := tester.makeHeaderFetcher("valid", blocks, -gatherSlack) + bodyFetcher := tester.makeBodyFetcher("valid", blocks, 0) + + // Iteratively announce blocks, but overlap them continuously + overlap := 16 + imported := make(chan interface{}, len(hashes)-1) + for i := 0; i < overlap; i++ { + imported <- nil + } + tester.fetcher.importedHook = func(header *types.Header, block *types.Block) { + if light { + if header == nil { + t.Fatalf("Fetcher try to import empty header") + } + imported <- header + } else { + if block == nil { + t.Fatalf("Fetcher try to import empty block") + } + imported <- block + } + } + + for i := len(hashes) - 2; i >= 0; i-- { + tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) + select { + case <-imported: + case <-time.After(time.Second): + t.Fatalf("block %d: import timeout", len(hashes)-i) + } + } + // Wait for all the imports to complete and check count + verifyImportCount(t, imported, overlap) + verifyChainHeight(t, tester, uint64(len(hashes)-1)) +} + +// Tests that announces already being retrieved will not be duplicated. +func TestFullPendingDeduplication(t *testing.T) { testPendingDeduplication(t, false) } +func TestLightPendingDeduplication(t *testing.T) { testPendingDeduplication(t, true) } + +func testPendingDeduplication(t *testing.T, light bool) { + // Create a hash and corresponding block + hashes, blocks := makeChain(1, 0, genesis) + + // Assemble a tester with a built in counter and delayed fetcher + tester := newTester(light) + headerFetcher := tester.makeHeaderFetcher("repeater", blocks, -gatherSlack) + bodyFetcher := tester.makeBodyFetcher("repeater", blocks, 0) + + delay := 50 * time.Millisecond + counter := uint32(0) + headerWrapper := func(hash common.Hash) error { + atomic.AddUint32(&counter, 1) + + // Simulate a long running fetch + go func() { + time.Sleep(delay) + headerFetcher(hash) + }() + return nil + } + checkNonExist := func() bool { + return tester.getBlock(hashes[0]) == nil + } + if light { + checkNonExist = func() bool { + return tester.getHeader(hashes[0]) == nil + } + } + // Announce the same block many times until it's fetched (wait for any pending ops) + for checkNonExist() { + tester.fetcher.Notify("repeater", hashes[0], 1, time.Now().Add(-arriveTimeout), headerWrapper, bodyFetcher) + time.Sleep(time.Millisecond) + } + time.Sleep(delay) + + // Check that all blocks were imported and none fetched twice + if int(counter) != 1 { + t.Fatalf("retrieval count mismatch: have %v, want %v", counter, 1) + } + verifyChainHeight(t, tester, 1) +} + +// Tests that announcements retrieved in a random order are cached and eventually +// imported when all the gaps are filled in. +func TestFullRandomArrivalImport(t *testing.T) { testRandomArrivalImport(t, false) } +func TestLightRandomArrivalImport(t *testing.T) { testRandomArrivalImport(t, true) } + +func testRandomArrivalImport(t *testing.T, light bool) { + // Create a chain of blocks to import, and choose one to delay + targetBlocks := maxQueueDist + hashes, blocks := makeChain(targetBlocks, 0, genesis) + skip := targetBlocks / 2 + + tester := newTester(light) + headerFetcher := tester.makeHeaderFetcher("valid", blocks, -gatherSlack) + bodyFetcher := tester.makeBodyFetcher("valid", blocks, 0) + + // Iteratively announce blocks, skipping one entry + imported := make(chan interface{}, len(hashes)-1) + tester.fetcher.importedHook = func(header *types.Header, block *types.Block) { + if light { + if header == nil { + t.Fatalf("Fetcher try to import empty header") + } + imported <- header + } else { + if block == nil { + t.Fatalf("Fetcher try to import empty block") + } + imported <- block + } + } + for i := len(hashes) - 1; i >= 0; i-- { + if i != skip { + tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) + time.Sleep(time.Millisecond) + } + } + // Finally announce the skipped entry and check full import + tester.fetcher.Notify("valid", hashes[skip], uint64(len(hashes)-skip-1), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) + verifyImportCount(t, imported, len(hashes)-1) + verifyChainHeight(t, tester, uint64(len(hashes)-1)) +} + +// Tests that direct block enqueues (due to block propagation vs. hash announce) +// are correctly schedule, filling and import queue gaps. +func TestQueueGapFill(t *testing.T) { + // Create a chain of blocks to import, and choose one to not announce at all + targetBlocks := maxQueueDist + hashes, blocks := makeChain(targetBlocks, 0, genesis) + skip := targetBlocks / 2 + + tester := newTester(false) + headerFetcher := tester.makeHeaderFetcher("valid", blocks, -gatherSlack) + bodyFetcher := tester.makeBodyFetcher("valid", blocks, 0) + + // Iteratively announce blocks, skipping one entry + imported := make(chan interface{}, len(hashes)-1) + tester.fetcher.importedHook = func(header *types.Header, block *types.Block) { imported <- block } + + for i := len(hashes) - 1; i >= 0; i-- { + if i != skip { + tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) + time.Sleep(time.Millisecond) + } + } + // Fill the missing block directly as if propagated + tester.fetcher.Enqueue("valid", blocks[hashes[skip]]) + verifyImportCount(t, imported, len(hashes)-1) + verifyChainHeight(t, tester, uint64(len(hashes)-1)) +} + +// Tests that blocks arriving from various sources (multiple propagations, hash +// announces, etc) do not get scheduled for import multiple times. +func TestImportDeduplication(t *testing.T) { + // Create two blocks to import (one for duplication, the other for stalling) + hashes, blocks := makeChain(2, 0, genesis) + + // Create the tester and wrap the importer with a counter + tester := newTester(false) + headerFetcher := tester.makeHeaderFetcher("valid", blocks, -gatherSlack) + bodyFetcher := tester.makeBodyFetcher("valid", blocks, 0) + + counter := uint32(0) + tester.fetcher.insertChain = func(blocks types.Blocks) (int, error) { + atomic.AddUint32(&counter, uint32(len(blocks))) + return tester.insertChain(blocks) + } + // Instrument the fetching and imported events + fetching := make(chan []common.Hash) + imported := make(chan interface{}, len(hashes)-1) + tester.fetcher.fetchingHook = func(hashes []common.Hash) { fetching <- hashes } + tester.fetcher.importedHook = func(header *types.Header, block *types.Block) { imported <- block } + + // Announce the duplicating block, wait for retrieval, and also propagate directly + tester.fetcher.Notify("valid", hashes[0], 1, time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) + <-fetching + + tester.fetcher.Enqueue("valid", blocks[hashes[0]]) + tester.fetcher.Enqueue("valid", blocks[hashes[0]]) + tester.fetcher.Enqueue("valid", blocks[hashes[0]]) + + // Fill the missing block directly as if propagated, and check import uniqueness + tester.fetcher.Enqueue("valid", blocks[hashes[1]]) + verifyImportCount(t, imported, 2) + + if counter != 2 { + t.Fatalf("import invocation count mismatch: have %v, want %v", counter, 2) + } +} + +// Tests that blocks with numbers much lower or higher than out current head get +// discarded to prevent wasting resources on useless blocks from faulty peers. +func TestDistantPropagationDiscarding(t *testing.T) { + // Create a long chain to import and define the discard boundaries + hashes, blocks := makeChain(3*maxQueueDist, 0, genesis) + head := hashes[len(hashes)/2] + + low, high := len(hashes)/2+maxUncleDist+1, len(hashes)/2-maxQueueDist-1 + + // Create a tester and simulate a head block being the middle of the above chain + tester := newTester(false) + + tester.lock.Lock() + tester.hashes = []common.Hash{head} + tester.blocks = map[common.Hash]*types.Block{head: blocks[head]} + tester.lock.Unlock() + + // Ensure that a block with a lower number than the threshold is discarded + tester.fetcher.Enqueue("lower", blocks[hashes[low]]) + time.Sleep(10 * time.Millisecond) + if !tester.fetcher.queue.Empty() { + t.Fatalf("fetcher queued stale block") + } + // Ensure that a block with a higher number than the threshold is discarded + tester.fetcher.Enqueue("higher", blocks[hashes[high]]) + time.Sleep(10 * time.Millisecond) + if !tester.fetcher.queue.Empty() { + t.Fatalf("fetcher queued future block") + } +} + +// Tests that announcements with numbers much lower or higher than out current +// head get discarded to prevent wasting resources on useless blocks from faulty +// peers. +func TestFullDistantAnnouncementDiscarding(t *testing.T) { testDistantAnnouncementDiscarding(t, false) } +func TestLightDistantAnnouncementDiscarding(t *testing.T) { testDistantAnnouncementDiscarding(t, true) } + +func testDistantAnnouncementDiscarding(t *testing.T, light bool) { + // Create a long chain to import and define the discard boundaries + hashes, blocks := makeChain(3*maxQueueDist, 0, genesis) + head := hashes[len(hashes)/2] + + low, high := len(hashes)/2+maxUncleDist+1, len(hashes)/2-maxQueueDist-1 + + // Create a tester and simulate a head block being the middle of the above chain + tester := newTester(light) + + tester.lock.Lock() + tester.hashes = []common.Hash{head} + tester.headers = map[common.Hash]*types.Header{head: blocks[head].Header()} + tester.blocks = map[common.Hash]*types.Block{head: blocks[head]} + tester.lock.Unlock() + + headerFetcher := tester.makeHeaderFetcher("lower", blocks, -gatherSlack) + bodyFetcher := tester.makeBodyFetcher("lower", blocks, 0) + + fetching := make(chan struct{}, 2) + tester.fetcher.fetchingHook = func(hashes []common.Hash) { fetching <- struct{}{} } + + // Ensure that a block with a lower number than the threshold is discarded + tester.fetcher.Notify("lower", hashes[low], blocks[hashes[low]].NumberU64(), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) + select { + case <-time.After(50 * time.Millisecond): + case <-fetching: + t.Fatalf("fetcher requested stale header") + } + // Ensure that a block with a higher number than the threshold is discarded + tester.fetcher.Notify("higher", hashes[high], blocks[hashes[high]].NumberU64(), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) + select { + case <-time.After(50 * time.Millisecond): + case <-fetching: + t.Fatalf("fetcher requested future header") + } +} + +// Tests that peers announcing blocks with invalid numbers (i.e. not matching +// the headers provided afterwards) get dropped as malicious. +func TestFullInvalidNumberAnnouncement(t *testing.T) { testInvalidNumberAnnouncement(t, false) } +func TestLightInvalidNumberAnnouncement(t *testing.T) { testInvalidNumberAnnouncement(t, true) } + +func testInvalidNumberAnnouncement(t *testing.T, light bool) { + // Create a single block to import and check numbers against + hashes, blocks := makeChain(1, 0, genesis) + + tester := newTester(light) + badHeaderFetcher := tester.makeHeaderFetcher("bad", blocks, -gatherSlack) + badBodyFetcher := tester.makeBodyFetcher("bad", blocks, 0) + + imported := make(chan interface{}) + announced := make(chan interface{}) + tester.fetcher.importedHook = func(header *types.Header, block *types.Block) { + if light { + if header == nil { + t.Fatalf("Fetcher try to import empty header") + } + imported <- header + } else { + if block == nil { + t.Fatalf("Fetcher try to import empty block") + } + imported <- block + } + } + // Announce a block with a bad number, check for immediate drop + tester.fetcher.announceChangeHook = func(hash common.Hash, b bool) { + announced <- nil + } + tester.fetcher.Notify("bad", hashes[0], 2, time.Now().Add(-arriveTimeout), badHeaderFetcher, badBodyFetcher) + verifyAnnounce := func() { + for i := 0; i < 2; i++ { + select { + case <-announced: + continue + case <-time.After(1 * time.Second): + t.Fatal("announce timeout") + return + } + } + } + verifyAnnounce() + verifyImportEvent(t, imported, false) + tester.lock.RLock() + dropped := tester.drops["bad"] + tester.lock.RUnlock() + + if !dropped { + t.Fatalf("peer with invalid numbered announcement not dropped") + } + goodHeaderFetcher := tester.makeHeaderFetcher("good", blocks, -gatherSlack) + goodBodyFetcher := tester.makeBodyFetcher("good", blocks, 0) + // Make sure a good announcement passes without a drop + tester.fetcher.Notify("good", hashes[0], 1, time.Now().Add(-arriveTimeout), goodHeaderFetcher, goodBodyFetcher) + verifyAnnounce() + verifyImportEvent(t, imported, true) + + tester.lock.RLock() + dropped = tester.drops["good"] + tester.lock.RUnlock() + + if dropped { + t.Fatalf("peer with valid numbered announcement dropped") + } + verifyImportDone(t, imported) +} + +// Tests that if a block is empty (i.e. header only), no body request should be +// made, and instead the header should be assembled into a whole block in itself. +func TestEmptyBlockShortCircuit(t *testing.T) { + // Create a chain of blocks to import + hashes, blocks := makeChain(32, 0, genesis) + + tester := newTester(false) + headerFetcher := tester.makeHeaderFetcher("valid", blocks, -gatherSlack) + bodyFetcher := tester.makeBodyFetcher("valid", blocks, 0) + + // Add a monitoring hook for all internal events + fetching := make(chan []common.Hash) + tester.fetcher.fetchingHook = func(hashes []common.Hash) { fetching <- hashes } + + completing := make(chan []common.Hash) + tester.fetcher.completingHook = func(hashes []common.Hash) { completing <- hashes } + + imported := make(chan interface{}) + tester.fetcher.importedHook = func(header *types.Header, block *types.Block) { + if block == nil { + t.Fatalf("Fetcher try to import empty block") + } + imported <- block + } + // Iteratively announce blocks until all are imported + for i := len(hashes) - 2; i >= 0; i-- { + tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), headerFetcher, bodyFetcher) + + // All announces should fetch the header + verifyFetchingEvent(t, fetching, true) + + // Only blocks with data contents should request bodies + verifyCompletingEvent(t, completing, len(blocks[hashes[i]].Transactions()) > 0 || len(blocks[hashes[i]].Uncles()) > 0) + + // Irrelevant of the construct, import should succeed + verifyImportEvent(t, imported, true) + } + verifyImportDone(t, imported) +} + +// Tests that a peer is unable to use unbounded memory with sending infinite +// block announcements to a node, but that even in the face of such an attack, +// the fetcher remains operational. +func TestHashMemoryExhaustionAttack(t *testing.T) { + // Create a tester with instrumented import hooks + tester := newTester(false) + + imported, announces := make(chan interface{}), int32(0) + tester.fetcher.importedHook = func(header *types.Header, block *types.Block) { imported <- block } + tester.fetcher.announceChangeHook = func(hash common.Hash, added bool) { + if added { + atomic.AddInt32(&announces, 1) + } else { + atomic.AddInt32(&announces, -1) + } + } + // Create a valid chain and an infinite junk chain + targetBlocks := hashLimit + 2*maxQueueDist + hashes, blocks := makeChain(targetBlocks, 0, genesis) + validHeaderFetcher := tester.makeHeaderFetcher("valid", blocks, -gatherSlack) + validBodyFetcher := tester.makeBodyFetcher("valid", blocks, 0) + + attack, _ := makeChain(targetBlocks, 0, unknownBlock) + attackerHeaderFetcher := tester.makeHeaderFetcher("attacker", nil, -gatherSlack) + attackerBodyFetcher := tester.makeBodyFetcher("attacker", nil, 0) + + // Feed the tester a huge hashset from the attacker, and a limited from the valid peer + for i := 0; i < len(attack); i++ { + if i < maxQueueDist { + tester.fetcher.Notify("valid", hashes[len(hashes)-2-i], uint64(i+1), time.Now(), validHeaderFetcher, validBodyFetcher) + } + tester.fetcher.Notify("attacker", attack[i], 1 /* don't distance drop */, time.Now(), attackerHeaderFetcher, attackerBodyFetcher) + } + if count := atomic.LoadInt32(&announces); count != hashLimit+maxQueueDist { + t.Fatalf("queued announce count mismatch: have %d, want %d", count, hashLimit+maxQueueDist) + } + // Wait for fetches to complete + verifyImportCount(t, imported, maxQueueDist) + + // Feed the remaining valid hashes to ensure DOS protection state remains clean + for i := len(hashes) - maxQueueDist - 2; i >= 0; i-- { + tester.fetcher.Notify("valid", hashes[i], uint64(len(hashes)-i-1), time.Now().Add(-arriveTimeout), validHeaderFetcher, validBodyFetcher) + verifyImportEvent(t, imported, true) + } + verifyImportDone(t, imported) +} + +// Tests that blocks sent to the fetcher (either through propagation or via hash +// announces and retrievals) don't pile up indefinitely, exhausting available +// system memory. +func TestBlockMemoryExhaustionAttack(t *testing.T) { + // Create a tester with instrumented import hooks + tester := newTester(false) + + imported, enqueued := make(chan interface{}), int32(0) + tester.fetcher.importedHook = func(header *types.Header, block *types.Block) { imported <- block } + tester.fetcher.queueChangeHook = func(hash common.Hash, added bool) { + if added { + atomic.AddInt32(&enqueued, 1) + } else { + atomic.AddInt32(&enqueued, -1) + } + } + // Create a valid chain and a batch of dangling (but in range) blocks + targetBlocks := hashLimit + 2*maxQueueDist + hashes, blocks := makeChain(targetBlocks, 0, genesis) + attack := make(map[common.Hash]*types.Block) + for i := byte(0); len(attack) < blockLimit+2*maxQueueDist; i++ { + hashes, blocks := makeChain(maxQueueDist-1, i, unknownBlock) + for _, hash := range hashes[:maxQueueDist-2] { + attack[hash] = blocks[hash] + } + } + // Try to feed all the attacker blocks make sure only a limited batch is accepted + for _, block := range attack { + tester.fetcher.Enqueue("attacker", block) + } + time.Sleep(200 * time.Millisecond) + if queued := atomic.LoadInt32(&enqueued); queued != blockLimit { + t.Fatalf("queued block count mismatch: have %d, want %d", queued, blockLimit) + } + // Queue up a batch of valid blocks, and check that a new peer is allowed to do so + for i := 0; i < maxQueueDist-1; i++ { + tester.fetcher.Enqueue("valid", blocks[hashes[len(hashes)-3-i]]) + } + time.Sleep(100 * time.Millisecond) + if queued := atomic.LoadInt32(&enqueued); queued != blockLimit+maxQueueDist-1 { + t.Fatalf("queued block count mismatch: have %d, want %d", queued, blockLimit+maxQueueDist-1) + } + // Insert the missing piece (and sanity check the import) + tester.fetcher.Enqueue("valid", blocks[hashes[len(hashes)-2]]) + verifyImportCount(t, imported, maxQueueDist) + + // Insert the remaining blocks in chunks to ensure clean DOS protection + for i := maxQueueDist; i < len(hashes)-1; i++ { + tester.fetcher.Enqueue("valid", blocks[hashes[len(hashes)-2-i]]) + verifyImportEvent(t, imported, true) + } + verifyImportDone(t, imported) +} diff --git a/les/handler_test.go b/les/handler_test.go index bb8ad3382..aba45764b 100644 --- a/les/handler_test.go +++ b/les/handler_test.go @@ -30,7 +30,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/eth/downloader" + "github.com/ethereum/go-ethereum/les/downloader" "github.com/ethereum/go-ethereum/light" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/params" diff --git a/les/sync.go b/les/sync.go index fa5ef4ff8..31cd06ca7 100644 --- a/les/sync.go +++ b/les/sync.go @@ -23,7 +23,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/eth/downloader" + "github.com/ethereum/go-ethereum/les/downloader" "github.com/ethereum/go-ethereum/light" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" From 578bc8164d64bef89eed0b063ff8175e567a5686 Mon Sep 17 00:00:00 2001 From: Jonathan Verhees Date: Mon, 13 Sep 2021 11:34:00 +0100 Subject: [PATCH 47/73] README: add londonBlock to example genesis config (#23565) Updated example genesis.json configuration section, added londonBlock within config. --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index c7adb135d..86f7713fd 100644 --- a/README.md +++ b/README.md @@ -231,7 +231,8 @@ aware of and agree upon. This consists of a small JSON file (e.g. call it `genes "constantinopleBlock": 0, "petersburgBlock": 0, "istanbulBlock": 0, - "berlinBlock": 0 + "berlinBlock": 0, + "londonBlock": 0 }, "alloc": {}, "coinbase": "0x0000000000000000000000000000000000000000", From babe9b993ef9f7ad7b40d47b39520d87fb010b62 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Mon, 13 Sep 2021 13:57:40 +0200 Subject: [PATCH 48/73] cmd/evm: transaction validation tool (#23494) * cmd/evm: transaction validation tool * cmd/evm: add hash to t9n tool * cmd/evm: lint nits * cmd/evm: nitpicks --- cmd/evm/internal/t8ntool/transaction.go | 136 ++++++++++++++++++++++++ cmd/evm/internal/t8ntool/transition.go | 2 +- cmd/evm/main.go | 15 ++- cmd/evm/t8n_test.go | 85 ++++++++++++++- cmd/evm/testdata/15/blockheader.rlp | 1 + cmd/evm/testdata/15/exp.json | 8 ++ cmd/evm/testdata/15/exp2.json | 10 ++ cmd/evm/testdata/15/exp3.json | 47 ++++++++ cmd/evm/testdata/15/signed_txs.rlp | 1 + 9 files changed, 301 insertions(+), 4 deletions(-) create mode 100644 cmd/evm/internal/t8ntool/transaction.go create mode 100644 cmd/evm/testdata/15/blockheader.rlp create mode 100644 cmd/evm/testdata/15/exp.json create mode 100644 cmd/evm/testdata/15/exp2.json create mode 100644 cmd/evm/testdata/15/exp3.json create mode 100644 cmd/evm/testdata/15/signed_txs.rlp diff --git a/cmd/evm/internal/t8ntool/transaction.go b/cmd/evm/internal/t8ntool/transaction.go new file mode 100644 index 000000000..aecbad79d --- /dev/null +++ b/cmd/evm/internal/t8ntool/transaction.go @@ -0,0 +1,136 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package t8ntool + +import ( + "encoding/json" + "errors" + "fmt" + "math/big" + "os" + "strings" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/params" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/tests" + "gopkg.in/urfave/cli.v1" +) + +type result struct { + Error error + Address common.Address + Hash common.Hash +} + +// MarshalJSON marshals as JSON with a hash. +func (r *result) MarshalJSON() ([]byte, error) { + type xx struct { + Error string `json:"error,omitempty"` + Address *common.Address `json:"address,omitempty"` + Hash *common.Hash `json:"hash,omitempty"` + } + var out xx + if r.Error != nil { + out.Error = r.Error.Error() + } + if r.Address != (common.Address{}) { + out.Address = &r.Address + } + if r.Hash != (common.Hash{}) { + out.Hash = &r.Hash + } + return json.Marshal(out) +} + +func Transaction(ctx *cli.Context) error { + // Configure the go-ethereum logger + glogger := log.NewGlogHandler(log.StreamHandler(os.Stderr, log.TerminalFormat(false))) + glogger.Verbosity(log.Lvl(ctx.Int(VerbosityFlag.Name))) + log.Root().SetHandler(glogger) + + var ( + err error + ) + // We need to load the transactions. May be either in stdin input or in files. + // Check if anything needs to be read from stdin + var ( + txStr = ctx.String(InputTxsFlag.Name) + inputData = &input{} + chainConfig *params.ChainConfig + ) + // Construct the chainconfig + if cConf, _, err := tests.GetChainConfig(ctx.String(ForknameFlag.Name)); err != nil { + return NewError(ErrorVMConfig, fmt.Errorf("failed constructing chain configuration: %v", err)) + } else { + chainConfig = cConf + } + // Set the chain id + chainConfig.ChainID = big.NewInt(ctx.Int64(ChainIDFlag.Name)) + var body hexutil.Bytes + if txStr == stdinSelector { + decoder := json.NewDecoder(os.Stdin) + if err := decoder.Decode(inputData); err != nil { + return NewError(ErrorJson, fmt.Errorf("failed unmarshaling stdin: %v", err)) + } + // Decode the body of already signed transactions + body = common.FromHex(inputData.TxRlp) + } else { + // Read input from file + inFile, err := os.Open(txStr) + if err != nil { + return NewError(ErrorIO, fmt.Errorf("failed reading txs file: %v", err)) + } + defer inFile.Close() + decoder := json.NewDecoder(inFile) + if strings.HasSuffix(txStr, ".rlp") { + if err := decoder.Decode(&body); err != nil { + return err + } + } else { + return NewError(ErrorIO, errors.New("only rlp supported")) + } + } + signer := types.MakeSigner(chainConfig, new(big.Int)) + // We now have the transactions in 'body', which is supposed to be an + // rlp list of transactions + it, err := rlp.NewListIterator([]byte(body)) + if err != nil { + return err + } + var results []result + for it.Next() { + var tx types.Transaction + err := rlp.DecodeBytes(it.Value(), &tx) + if err != nil { + results = append(results, result{Error: err}) + continue + } + sender, err := types.Sender(signer, &tx) + if err != nil { + results = append(results, result{Error: err}) + continue + } + results = append(results, result{Address: sender, Hash: tx.Hash()}) + } + out, err := json.MarshalIndent(results, "", " ") + fmt.Println(string(out)) + return err +} diff --git a/cmd/evm/internal/t8ntool/transition.go b/cmd/evm/internal/t8ntool/transition.go index 545962269..84da89bee 100644 --- a/cmd/evm/internal/t8ntool/transition.go +++ b/cmd/evm/internal/t8ntool/transition.go @@ -81,7 +81,7 @@ type input struct { TxRlp string `json:"txsRlp,omitempty"` } -func Main(ctx *cli.Context) error { +func Transition(ctx *cli.Context) error { // Configure the go-ethereum logger glogger := log.NewGlogHandler(log.StreamHandler(os.Stderr, log.TerminalFormat(false))) glogger.Verbosity(log.Lvl(ctx.Int(VerbosityFlag.Name))) diff --git a/cmd/evm/main.go b/cmd/evm/main.go index c32b6c382..f1bbdb5d7 100644 --- a/cmd/evm/main.go +++ b/cmd/evm/main.go @@ -135,7 +135,7 @@ var stateTransitionCommand = cli.Command{ Name: "transition", Aliases: []string{"t8n"}, Usage: "executes a full state transition", - Action: t8ntool.Main, + Action: t8ntool.Transition, Flags: []cli.Flag{ t8ntool.TraceFlag, t8ntool.TraceDisableMemoryFlag, @@ -154,6 +154,18 @@ var stateTransitionCommand = cli.Command{ t8ntool.VerbosityFlag, }, } +var transactionCommand = cli.Command{ + Name: "transaction", + Aliases: []string{"t9n"}, + Usage: "performs transaction validation", + Action: t8ntool.Transaction, + Flags: []cli.Flag{ + t8ntool.InputTxsFlag, + t8ntool.ChainIDFlag, + t8ntool.ForknameFlag, + t8ntool.VerbosityFlag, + }, +} func init() { app.Flags = []cli.Flag{ @@ -187,6 +199,7 @@ func init() { runCommand, stateTestCommand, stateTransitionCommand, + transactionCommand, } cli.CommandHelpTemplate = flags.OriginCommandHelpTemplate } diff --git a/cmd/evm/t8n_test.go b/cmd/evm/t8n_test.go index 5af7ee19c..ad518a91a 100644 --- a/cmd/evm/t8n_test.go +++ b/cmd/evm/t8n_test.go @@ -70,7 +70,6 @@ type t8nOutput struct { } func (args *t8nOutput) get() (out []string) { - out = append(out, "t8n") if args.body { out = append(out, "--output.body", "stdout") } else { @@ -173,7 +172,9 @@ func TestT8n(t *testing.T) { }, } { - args := append(tc.output.get(), tc.input.get(tc.base)...) + args := []string{"t8n"} + args = append(args, tc.output.get()...) + args = append(args, tc.input.get(tc.base)...) tt.Run("evm-test", args...) tt.Logf("args: %v\n", strings.Join(args, " ")) // Compare the expected output, if provided @@ -198,6 +199,86 @@ func TestT8n(t *testing.T) { } } +type t9nInput struct { + inTxs string + stFork string +} + +func (args *t9nInput) get(base string) []string { + var out []string + if opt := args.inTxs; opt != "" { + out = append(out, "--input.txs") + out = append(out, fmt.Sprintf("%v/%v", base, opt)) + } + if opt := args.stFork; opt != "" { + out = append(out, "--state.fork", opt) + } + return out +} + +func TestT9n(t *testing.T) { + tt := new(testT8n) + tt.TestCmd = cmdtest.NewTestCmd(t, tt) + for i, tc := range []struct { + base string + input t9nInput + expExitCode int + expOut string + }{ + { // London txs on homestead + base: "./testdata/15", + input: t9nInput{ + inTxs: "signed_txs.rlp", + stFork: "Homestead", + }, + expOut: "exp.json", + }, + { // London txs on homestead + base: "./testdata/15", + input: t9nInput{ + inTxs: "signed_txs.rlp", + stFork: "London", + }, + expOut: "exp2.json", + }, + { // An RLP list (a blockheader really) + base: "./testdata/15", + input: t9nInput{ + inTxs: "blockheader.rlp", + stFork: "London", + }, + expOut: "exp3.json", + }, + } { + + args := []string{"t9n"} + args = append(args, tc.input.get(tc.base)...) + + tt.Run("evm-test", args...) + tt.Logf("args:\n go run . %v\n", strings.Join(args, " ")) + // Compare the expected output, if provided + if tc.expOut != "" { + want, err := os.ReadFile(fmt.Sprintf("%v/%v", tc.base, tc.expOut)) + if err != nil { + t.Fatalf("test %d: could not read expected output: %v", i, err) + } + have := tt.Output() + ok, err := cmpJson(have, want) + switch { + case err != nil: + t.Logf(string(have)) + t.Fatalf("test %d, json parsing failed: %v", i, err) + case !ok: + t.Fatalf("test %d: output wrong, have \n%v\nwant\n%v\n", i, string(have), string(want)) + } + } + tt.WaitExit() + if have, want := tt.ExitStatus(), tc.expExitCode; have != want { + t.Fatalf("test %d: wrong exit code, have %d, want %d", i, have, want) + } + } +} + // cmpJson compares the JSON in two byte slices. func cmpJson(a, b []byte) (bool, error) { var j, j2 interface{} diff --git a/cmd/evm/testdata/15/blockheader.rlp b/cmd/evm/testdata/15/blockheader.rlp new file mode 100644 index 000000000..1124e8e2d --- /dev/null +++ b/cmd/evm/testdata/15/blockheader.rlp @@ -0,0 +1 @@ +"0xf901f0a00000000000000000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000000940000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000000a00000000000000000000000000000000000000000000000000000000000000000b90100000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000007b0101020383010203a00000000000000000000000000000000000000000000000000000000000000000880000000000000000" \ No newline at end of file diff --git a/cmd/evm/testdata/15/exp.json b/cmd/evm/testdata/15/exp.json new file mode 100644 index 000000000..03d970c56 --- /dev/null +++ b/cmd/evm/testdata/15/exp.json @@ -0,0 +1,8 @@ +[ + { + "error": "transaction type not supported" + }, + { + "error": "transaction type not supported" + } +] diff --git a/cmd/evm/testdata/15/exp2.json b/cmd/evm/testdata/15/exp2.json new file mode 100644 index 000000000..85d821f55 --- /dev/null +++ b/cmd/evm/testdata/15/exp2.json @@ -0,0 +1,10 @@ +[ + { + "address": "0xd02d72e067e77158444ef2020ff2d325f929b363", + "hash": "0xa98a24882ea90916c6a86da650fbc6b14238e46f0af04a131ce92be897507476" + }, + { + "address": "0xd02d72e067e77158444ef2020ff2d325f929b363", + "hash": "0x36bad80acce7040c45fd32764b5c2b2d2e6f778669fb41791f73f546d56e739a" + } +] diff --git a/cmd/evm/testdata/15/exp3.json b/cmd/evm/testdata/15/exp3.json new file mode 100644 index 000000000..6c46d267c --- /dev/null +++ b/cmd/evm/testdata/15/exp3.json @@ -0,0 +1,47 @@ +[ + { + "error": "transaction type not supported" + }, + { + "error": "transaction type not supported" + }, + { + "error": "transaction type not supported" + }, + { + "error": "transaction type not supported" + }, + { + "error": "transaction type not supported" + }, + { + "error": "transaction type not supported" + }, + { + "error": "transaction type not supported" + }, + { + "error": "rlp: expected List" + }, + { + "error": "rlp: expected List" + }, + { + "error": "rlp: expected List" + }, + { + "error": "rlp: expected List" + }, + { + "error": "rlp: expected List" + }, + { + "error": "rlp: expected input list for types.AccessListTx" + }, + { + "error": "transaction type not supported" + }, + { + "error": "transaction type not supported" + } +] diff --git a/cmd/evm/testdata/15/signed_txs.rlp b/cmd/evm/testdata/15/signed_txs.rlp new file mode 100644 index 000000000..9d1157ea4 --- /dev/null +++ b/cmd/evm/testdata/15/signed_txs.rlp @@ -0,0 +1 @@ +"0xf8d2b86702f864010180820fa08284d09411111111111111111111111111111111111111118080c001a0b7dfab36232379bb3d1497a4f91c1966b1f932eae3ade107bf5d723b9cb474e0a06261c359a10f2132f126d250485b90cf20f30340801244a08ef6142ab33d1904b86702f864010280820fa08284d09411111111111111111111111111111111111111118080c080a0d4ec563b6568cd42d998fc4134b36933c6568d01533b5adf08769270243c6c7fa072bf7c21eac6bbeae5143371eef26d5e279637f3bd73482b55979d76d935b1e9" \ No newline at end of file From b8d7c662cd8f87f7dae15c7a53be25ec527999cd Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Mon, 13 Sep 2021 18:59:52 +0200 Subject: [PATCH 49/73] core, rpc: disable memory output by default in traces (#23558) * core: cmd: invert disableMemory * core: fix missed inversion * cmd/evm: preserve Flags but change default value * Apply suggestions from code review Co-authored-by: Martin Holst Swende Co-authored-by: Martin Holst Swende --- cmd/evm/disasm.go | 2 +- cmd/evm/internal/t8ntool/flags.go | 4 ++-- cmd/evm/internal/t8ntool/transition.go | 8 ++++---- cmd/evm/main.go | 6 +++--- cmd/evm/runner.go | 10 +++++----- cmd/evm/staterunner.go | 8 ++++---- core/vm/logger.go | 16 ++++++++-------- core/vm/logger_json.go | 4 ++-- eth/tracers/tracers_test.go | 4 ++-- tests/state_test.go | 2 +- 10 files changed, 32 insertions(+), 32 deletions(-) diff --git a/cmd/evm/disasm.go b/cmd/evm/disasm.go index 68a09cbf5..f9719497f 100644 --- a/cmd/evm/disasm.go +++ b/cmd/evm/disasm.go @@ -46,7 +46,7 @@ func disasmCmd(ctx *cli.Context) error { case ctx.GlobalIsSet(InputFlag.Name): in = ctx.GlobalString(InputFlag.Name) default: - return errors.New("Missing filename or --input value") + return errors.New("missing filename or --input value") } code := strings.TrimSpace(in) diff --git a/cmd/evm/internal/t8ntool/flags.go b/cmd/evm/internal/t8ntool/flags.go index 626f974a0..05b6ed164 100644 --- a/cmd/evm/internal/t8ntool/flags.go +++ b/cmd/evm/internal/t8ntool/flags.go @@ -30,7 +30,7 @@ var ( Name: "trace", Usage: "Output full trace logs to files .jsonl", } - TraceDisableMemoryFlag = cli.BoolFlag{ + TraceDisableMemoryFlag = cli.BoolTFlag{ Name: "trace.nomemory", Usage: "Disable full memory dump in traces", } @@ -38,7 +38,7 @@ var ( Name: "trace.nostack", Usage: "Disable stack output in traces", } - TraceDisableReturnDataFlag = cli.BoolFlag{ + TraceDisableReturnDataFlag = cli.BoolTFlag{ Name: "trace.noreturndata", Usage: "Disable return data output in traces", } diff --git a/cmd/evm/internal/t8ntool/transition.go b/cmd/evm/internal/t8ntool/transition.go index 84da89bee..7407ed0a4 100644 --- a/cmd/evm/internal/t8ntool/transition.go +++ b/cmd/evm/internal/t8ntool/transition.go @@ -107,10 +107,10 @@ func Transition(ctx *cli.Context) error { if ctx.Bool(TraceFlag.Name) { // Configure the EVM logger logConfig := &vm.LogConfig{ - DisableStack: ctx.Bool(TraceDisableStackFlag.Name), - DisableMemory: ctx.Bool(TraceDisableMemoryFlag.Name), - DisableReturnData: ctx.Bool(TraceDisableReturnDataFlag.Name), - Debug: true, + DisableStack: ctx.Bool(TraceDisableStackFlag.Name), + EnableMemory: !ctx.Bool(TraceDisableMemoryFlag.Name), + EnableReturnData: !ctx.Bool(TraceDisableReturnDataFlag.Name), + Debug: true, } var prevFile *os.File // This one closes the last file diff --git a/cmd/evm/main.go b/cmd/evm/main.go index f1bbdb5d7..26064efc3 100644 --- a/cmd/evm/main.go +++ b/cmd/evm/main.go @@ -113,7 +113,7 @@ var ( Name: "receiver", Usage: "The transaction receiver (execution context)", } - DisableMemoryFlag = cli.BoolFlag{ + DisableMemoryFlag = cli.BoolTFlag{ Name: "nomemory", Usage: "disable memory output", } @@ -125,9 +125,9 @@ var ( Name: "nostorage", Usage: "disable storage output", } - DisableReturnDataFlag = cli.BoolFlag{ + DisableReturnDataFlag = cli.BoolTFlag{ Name: "noreturndata", - Usage: "disable return data output", + Usage: "enable return data output", } ) diff --git a/cmd/evm/runner.go b/cmd/evm/runner.go index e409d2692..cedbd2281 100644 --- a/cmd/evm/runner.go +++ b/cmd/evm/runner.go @@ -108,11 +108,11 @@ func runCmd(ctx *cli.Context) error { glogger.Verbosity(log.Lvl(ctx.GlobalInt(VerbosityFlag.Name))) log.Root().SetHandler(glogger) logconfig := &vm.LogConfig{ - DisableMemory: ctx.GlobalBool(DisableMemoryFlag.Name), - DisableStack: ctx.GlobalBool(DisableStackFlag.Name), - DisableStorage: ctx.GlobalBool(DisableStorageFlag.Name), - DisableReturnData: ctx.GlobalBool(DisableReturnDataFlag.Name), - Debug: ctx.GlobalBool(DebugFlag.Name), + EnableMemory: !ctx.GlobalBool(DisableMemoryFlag.Name), + DisableStack: ctx.GlobalBool(DisableStackFlag.Name), + DisableStorage: ctx.GlobalBool(DisableStorageFlag.Name), + EnableReturnData: !ctx.GlobalBool(DisableReturnDataFlag.Name), + Debug: ctx.GlobalBool(DebugFlag.Name), } var ( diff --git a/cmd/evm/staterunner.go b/cmd/evm/staterunner.go index d8bc4eae8..ab2704609 100644 --- a/cmd/evm/staterunner.go +++ b/cmd/evm/staterunner.go @@ -59,10 +59,10 @@ func stateTestCmd(ctx *cli.Context) error { // Configure the EVM logger config := &vm.LogConfig{ - DisableMemory: ctx.GlobalBool(DisableMemoryFlag.Name), - DisableStack: ctx.GlobalBool(DisableStackFlag.Name), - DisableStorage: ctx.GlobalBool(DisableStorageFlag.Name), - DisableReturnData: ctx.GlobalBool(DisableReturnDataFlag.Name), + EnableMemory: !ctx.GlobalBool(DisableMemoryFlag.Name), + DisableStack: ctx.GlobalBool(DisableStackFlag.Name), + DisableStorage: ctx.GlobalBool(DisableStorageFlag.Name), + EnableReturnData: !ctx.GlobalBool(DisableReturnDataFlag.Name), } var ( tracer vm.Tracer diff --git a/core/vm/logger.go b/core/vm/logger.go index 900a5e585..9be5e7e5e 100644 --- a/core/vm/logger.go +++ b/core/vm/logger.go @@ -46,12 +46,12 @@ func (s Storage) Copy() Storage { // LogConfig are the configuration options for structured logger the EVM type LogConfig struct { - DisableMemory bool // disable memory capture - DisableStack bool // disable stack capture - DisableStorage bool // disable storage capture - DisableReturnData bool // disable return data capture - Debug bool // print output during capture end - Limit int // maximum length of output, but zero means unlimited + EnableMemory bool // enable memory capture + DisableStack bool // disable stack capture + DisableStorage bool // disable storage capture + EnableReturnData bool // enable return data capture + Debug bool // print output during capture end + Limit int // maximum length of output, but zero means unlimited // Chain overrides, can be used to execute a trace using future fork rules Overrides *params.ChainConfig `json:"overrides,omitempty"` } @@ -160,7 +160,7 @@ func (l *StructLogger) CaptureState(env *EVM, pc uint64, op OpCode, gas, cost ui } // Copy a snapshot of the current memory state to a new buffer var mem []byte - if !l.cfg.DisableMemory { + if l.cfg.EnableMemory { mem = make([]byte, len(memory.Data())) copy(mem, memory.Data()) } @@ -199,7 +199,7 @@ func (l *StructLogger) CaptureState(env *EVM, pc uint64, op OpCode, gas, cost ui } } var rdata []byte - if !l.cfg.DisableReturnData { + if l.cfg.EnableReturnData { rdata = make([]byte, len(rData)) copy(rdata, rData) } diff --git a/core/vm/logger_json.go b/core/vm/logger_json.go index 5210f479f..8656ce9d7 100644 --- a/core/vm/logger_json.go +++ b/core/vm/logger_json.go @@ -61,13 +61,13 @@ func (l *JSONLogger) CaptureState(env *EVM, pc uint64, op OpCode, gas, cost uint RefundCounter: env.StateDB.GetRefund(), Err: err, } - if !l.cfg.DisableMemory { + if l.cfg.EnableMemory { log.Memory = memory.Data() } if !l.cfg.DisableStack { log.Stack = stack.data } - if !l.cfg.DisableReturnData { + if l.cfg.EnableReturnData { log.ReturnData = rData } l.encoder.Encode(log) diff --git a/eth/tracers/tracers_test.go b/eth/tracers/tracers_test.go index 8fbbf154b..e738aa3a3 100644 --- a/eth/tracers/tracers_test.go +++ b/eth/tracers/tracers_test.go @@ -353,8 +353,8 @@ func BenchmarkTransactionTrace(b *testing.B) { tracer := vm.NewStructLogger(&vm.LogConfig{ Debug: false, //DisableStorage: true, - //DisableMemory: true, - //DisableReturnData: true, + //EnableMemory: false, + //EnableReturnData: false, }) evm := vm.NewEVM(context, txContext, statedb, params.AllEthashProtocolChanges, vm.Config{Debug: true, Tracer: tracer}) msg, err := tx.AsMessage(signer, nil) diff --git a/tests/state_test.go b/tests/state_test.go index a7d9653a3..9554e7563 100644 --- a/tests/state_test.go +++ b/tests/state_test.go @@ -115,7 +115,7 @@ func withTrace(t *testing.T, gasLimit uint64, test func(vm.Config) error) { } buf := new(bytes.Buffer) w := bufio.NewWriter(buf) - tracer := vm.NewJSONLogger(&vm.LogConfig{DisableMemory: true}, w) + tracer := vm.NewJSONLogger(&vm.LogConfig{}, w) config.Debug, config.Tracer = true, tracer err2 := test(config) if !reflect.DeepEqual(err, err2) { From 84ff152de56b9d65f6888daef9dc00c97a6aafd3 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Wed, 15 Sep 2021 10:52:48 +0200 Subject: [PATCH 50/73] cmd/devp2p/internal/ethtest: only use eth66 if eth66 is negotiated (#23568) * cmd/devp2p/internal/ethtest: only use eth66 if eth66 is negotiated * cmd/devp2p/internal/ethtest: switch on concrete type not pointer * cmd/devp2p/internal/ethtest: switch on concrete type not pointer --- cmd/devp2p/internal/ethtest/helpers.go | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/cmd/devp2p/internal/ethtest/helpers.go b/cmd/devp2p/internal/ethtest/helpers.go index 6f7365483..88d8e143c 100644 --- a/cmd/devp2p/internal/ethtest/helpers.go +++ b/cmd/devp2p/internal/ethtest/helpers.go @@ -242,9 +242,17 @@ func (s *Suite) createSendAndRecvConns(isEth66 bool) (*Conn, *Conn, error) { return sendConn, recvConn, nil } +func (c *Conn) readAndServe(chain *Chain, timeout time.Duration) Message { + if c.negotiatedProtoVersion == 66 { + _, msg := c.readAndServe66(chain, timeout) + return msg + } + return c.readAndServe65(chain, timeout) +} + // readAndServe serves GetBlockHeaders requests while waiting // on another message from the node. -func (c *Conn) readAndServe(chain *Chain, timeout time.Duration) Message { +func (c *Conn) readAndServe65(chain *Chain, timeout time.Duration) Message { start := time.Now() for time.Since(start) < timeout { c.SetReadDeadline(time.Now().Add(5 * time.Second)) @@ -279,8 +287,8 @@ func (c *Conn) readAndServe66(chain *Chain, timeout time.Duration) (uint64, Mess switch msg := msg.(type) { case *Ping: c.Write(&Pong{}) - case *GetBlockHeaders: - headers, err := chain.GetHeaders(*msg) + case GetBlockHeaders: + headers, err := chain.GetHeaders(msg) if err != nil { return 0, errorf("could not get headers for inbound header request: %v", err) } From 7ada89d4e68801d958b0ffe311c57919969a62be Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Thu, 16 Sep 2021 13:45:16 +0200 Subject: [PATCH 51/73] eth/tracers: abort evm execution when trace is aborted (#23580) --- eth/tracers/tracer.go | 1 + 1 file changed, 1 insertion(+) diff --git a/eth/tracers/tracer.go b/eth/tracers/tracer.go index 2c52761f5..e1372b23b 100644 --- a/eth/tracers/tracer.go +++ b/eth/tracers/tracer.go @@ -600,6 +600,7 @@ func (jst *Tracer) CaptureState(env *vm.EVM, pc uint64, op vm.OpCode, gas, cost // If tracing was interrupted, set the error and stop if atomic.LoadUint32(&jst.interrupt) > 0 { jst.err = jst.reason + env.Cancel() return } jst.opWrapper.op = op From 401354976bb44f0ad4455ca1e0b5c0dc31d9a5f5 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi <1591639+s1na@users.noreply.github.com> Date: Fri, 17 Sep 2021 09:31:22 +0200 Subject: [PATCH 52/73] core,eth: call frame tracing (#23087) This change introduces 2 new optional methods; `enter()` and `exit()` for js tracers, and makes `step()` optiona. The two new methods are invoked when entering and exiting a call frame (but not invoked for the outermost scope, which has it's own methods). Currently these are the data fields passed to each of them: enter: type (opcode), from, to, input, gas, value exit: output, gasUsed, error The PR also comes with a re-write of the callTracer. As a backup we keep the previous tracing script under the name `callTracerLegacy`. Behaviour of both tracers are equivalent for the most part, although there are some small differences (improvements), where the new tracer is more correct / has more information. --- core/vm/access_list_tracer.go | 5 + core/vm/evm.go | 65 ++- core/vm/instructions.go | 4 + core/vm/logger.go | 12 + core/vm/logger_json.go | 5 + core/vm/runtime/runtime_test.go | 263 ++++++++++- core/vm/stack.go | 2 +- eth/tracers/internal/tracers/assets.go | 63 ++- eth/tracers/internal/tracers/call_tracer.js | 285 ++++-------- .../internal/tracers/call_tracer_legacy.js | 252 +++++++++++ .../create.json} | 0 .../deep_calls.json} | 0 .../delegatecall.json} | 0 .../inner_create_oog_outer_throw.json | 77 ++++ .../testdata/call_tracer/inner_instafail.json | 63 +++ .../inner_throw_outer_revert.json} | 0 .../oog.json} | 0 .../revert.json} | 0 .../revert_reason.json} | 0 .../testdata/call_tracer/selfdestruct.json | 75 ++++ eth/tracers/testdata/call_tracer/simple.json | 80 ++++ .../throw.json} | 0 .../testdata/call_tracer_legacy/create.json | 58 +++ .../call_tracer_legacy/deep_calls.json | 415 ++++++++++++++++++ .../call_tracer_legacy/delegatecall.json | 97 ++++ .../inner_create_oog_outer_throw.json} | 0 .../inner_instafail.json} | 0 .../inner_throw_outer_revert.json | 81 ++++ .../testdata/call_tracer_legacy/oog.json | 60 +++ .../testdata/call_tracer_legacy/revert.json | 58 +++ .../call_tracer_legacy/revert_reason.json | 64 +++ .../call_tracer_legacy/selfdestruct.json | 73 +++ .../simple.json} | 0 .../testdata/call_tracer_legacy/throw.json | 62 +++ eth/tracers/tracer.go | 235 ++++++++-- eth/tracers/tracer_test.go | 32 ++ eth/tracers/tracers_test.go | 90 +++- 37 files changed, 2290 insertions(+), 286 deletions(-) create mode 100644 eth/tracers/internal/tracers/call_tracer_legacy.js rename eth/tracers/testdata/{call_tracer_create.json => call_tracer/create.json} (100%) rename eth/tracers/testdata/{call_tracer_deep_calls.json => call_tracer/deep_calls.json} (100%) rename eth/tracers/testdata/{call_tracer_delegatecall.json => call_tracer/delegatecall.json} (100%) create mode 100644 eth/tracers/testdata/call_tracer/inner_create_oog_outer_throw.json create mode 100644 eth/tracers/testdata/call_tracer/inner_instafail.json rename eth/tracers/testdata/{call_tracer_inner_throw_outer_revert.json => call_tracer/inner_throw_outer_revert.json} (100%) rename eth/tracers/testdata/{call_tracer_oog.json => call_tracer/oog.json} (100%) rename eth/tracers/testdata/{call_tracer_revert.json => call_tracer/revert.json} (100%) rename eth/tracers/testdata/{call_tracer_revert_reason.json => call_tracer/revert_reason.json} (100%) create mode 100644 eth/tracers/testdata/call_tracer/selfdestruct.json create mode 100644 eth/tracers/testdata/call_tracer/simple.json rename eth/tracers/testdata/{call_tracer_throw.json => call_tracer/throw.json} (100%) create mode 100644 eth/tracers/testdata/call_tracer_legacy/create.json create mode 100644 eth/tracers/testdata/call_tracer_legacy/deep_calls.json create mode 100644 eth/tracers/testdata/call_tracer_legacy/delegatecall.json rename eth/tracers/testdata/{call_tracer_inner_create_oog_outer_throw.json => call_tracer_legacy/inner_create_oog_outer_throw.json} (100%) rename eth/tracers/testdata/{call_tracer_inner_instafail.json => call_tracer_legacy/inner_instafail.json} (100%) create mode 100644 eth/tracers/testdata/call_tracer_legacy/inner_throw_outer_revert.json create mode 100644 eth/tracers/testdata/call_tracer_legacy/oog.json create mode 100644 eth/tracers/testdata/call_tracer_legacy/revert.json create mode 100644 eth/tracers/testdata/call_tracer_legacy/revert_reason.json create mode 100644 eth/tracers/testdata/call_tracer_legacy/selfdestruct.json rename eth/tracers/testdata/{call_tracer_simple.json => call_tracer_legacy/simple.json} (100%) create mode 100644 eth/tracers/testdata/call_tracer_legacy/throw.json diff --git a/core/vm/access_list_tracer.go b/core/vm/access_list_tracer.go index cc5461d1c..11b4e2942 100644 --- a/core/vm/access_list_tracer.go +++ b/core/vm/access_list_tracer.go @@ -166,6 +166,11 @@ func (*AccessListTracer) CaptureFault(env *EVM, pc uint64, op OpCode, gas, cost func (*AccessListTracer) CaptureEnd(output []byte, gasUsed uint64, t time.Duration, err error) {} +func (*AccessListTracer) CaptureEnter(typ OpCode, from common.Address, to common.Address, input []byte, gas uint64, value *big.Int) { +} + +func (*AccessListTracer) CaptureExit(output []byte, gasUsed uint64, err error) {} + // AccessList returns the current accesslist maintained by the tracer. func (a *AccessListTracer) AccessList() types.AccessList { return a.list.accessList() diff --git a/core/vm/evm.go b/core/vm/evm.go index 896476673..3b4bd69d7 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -193,11 +193,19 @@ func (evm *EVM) Call(caller ContractRef, addr common.Address, input []byte, gas evm.Context.Transfer(evm.StateDB, caller.Address(), addr, value) // Capture the tracer start/end events in debug mode - if evm.Config.Debug && evm.depth == 0 { - evm.Config.Tracer.CaptureStart(evm, caller.Address(), addr, false, input, gas, value) - defer func(startGas uint64, startTime time.Time) { // Lazy evaluation of the parameters - evm.Config.Tracer.CaptureEnd(ret, startGas-gas, time.Since(startTime), err) - }(gas, time.Now()) + if evm.Config.Debug { + if evm.depth == 0 { + evm.Config.Tracer.CaptureStart(evm, caller.Address(), addr, false, input, gas, value) + defer func(startGas uint64, startTime time.Time) { // Lazy evaluation of the parameters + evm.Config.Tracer.CaptureEnd(ret, startGas-gas, time.Since(startTime), err) + }(gas, time.Now()) + } else { + // Handle tracer events for entering and exiting a call frame + evm.Config.Tracer.CaptureEnter(CALL, caller.Address(), addr, input, gas, value) + defer func(startGas uint64) { + evm.Config.Tracer.CaptureExit(ret, startGas-gas, err) + }(gas) + } } if isPrecompile { @@ -257,6 +265,14 @@ func (evm *EVM) CallCode(caller ContractRef, addr common.Address, input []byte, } var snapshot = evm.StateDB.Snapshot() + // Invoke tracer hooks that signal entering/exiting a call frame + if evm.Config.Debug { + evm.Config.Tracer.CaptureEnter(CALLCODE, caller.Address(), addr, input, gas, value) + defer func(startGas uint64) { + evm.Config.Tracer.CaptureExit(ret, startGas-gas, err) + }(gas) + } + // It is allowed to call precompiles, even via delegatecall if p, isPrecompile := evm.precompile(addr); isPrecompile { ret, gas, err = RunPrecompiledContract(p, input, gas) @@ -293,6 +309,14 @@ func (evm *EVM) DelegateCall(caller ContractRef, addr common.Address, input []by } var snapshot = evm.StateDB.Snapshot() + // Invoke tracer hooks that signal entering/exiting a call frame + if evm.Config.Debug { + evm.Config.Tracer.CaptureEnter(DELEGATECALL, caller.Address(), addr, input, gas, nil) + defer func(startGas uint64) { + evm.Config.Tracer.CaptureExit(ret, startGas-gas, err) + }(gas) + } + // It is allowed to call precompiles, even via delegatecall if p, isPrecompile := evm.precompile(addr); isPrecompile { ret, gas, err = RunPrecompiledContract(p, input, gas) @@ -338,6 +362,14 @@ func (evm *EVM) StaticCall(caller ContractRef, addr common.Address, input []byte // future scenarios evm.StateDB.AddBalance(addr, big0) + // Invoke tracer hooks that signal entering/exiting a call frame + if evm.Config.Debug { + evm.Config.Tracer.CaptureEnter(STATICCALL, caller.Address(), addr, input, gas, nil) + defer func(startGas uint64) { + evm.Config.Tracer.CaptureExit(ret, startGas-gas, err) + }(gas) + } + if p, isPrecompile := evm.precompile(addr); isPrecompile { ret, gas, err = RunPrecompiledContract(p, input, gas) } else { @@ -377,7 +409,7 @@ func (c *codeAndHash) Hash() common.Hash { } // create creates a new contract using code as deployment code. -func (evm *EVM) create(caller ContractRef, codeAndHash *codeAndHash, gas uint64, value *big.Int, address common.Address) ([]byte, common.Address, uint64, error) { +func (evm *EVM) create(caller ContractRef, codeAndHash *codeAndHash, gas uint64, value *big.Int, address common.Address, typ OpCode) ([]byte, common.Address, uint64, error) { // Depth check execution. Fail if we're trying to execute above the // limit. if evm.depth > int(params.CallCreateDepth) { @@ -415,9 +447,14 @@ func (evm *EVM) create(caller ContractRef, codeAndHash *codeAndHash, gas uint64, return nil, address, gas, nil } - if evm.Config.Debug && evm.depth == 0 { - evm.Config.Tracer.CaptureStart(evm, caller.Address(), address, true, codeAndHash.code, gas, value) + if evm.Config.Debug { + if evm.depth == 0 { + evm.Config.Tracer.CaptureStart(evm, caller.Address(), address, true, codeAndHash.code, gas, value) + } else { + evm.Config.Tracer.CaptureEnter(typ, caller.Address(), address, codeAndHash.code, gas, value) + } } + start := time.Now() ret, err := evm.interpreter.Run(contract, nil, false) @@ -455,8 +492,12 @@ func (evm *EVM) create(caller ContractRef, codeAndHash *codeAndHash, gas uint64, } } - if evm.Config.Debug && evm.depth == 0 { - evm.Config.Tracer.CaptureEnd(ret, gas-contract.Gas, time.Since(start), err) + if evm.Config.Debug { + if evm.depth == 0 { + evm.Config.Tracer.CaptureEnd(ret, gas-contract.Gas, time.Since(start), err) + } else { + evm.Config.Tracer.CaptureExit(ret, gas-contract.Gas, err) + } } return ret, address, contract.Gas, err } @@ -464,7 +505,7 @@ func (evm *EVM) create(caller ContractRef, codeAndHash *codeAndHash, gas uint64, // Create creates a new contract using code as deployment code. func (evm *EVM) Create(caller ContractRef, code []byte, gas uint64, value *big.Int) (ret []byte, contractAddr common.Address, leftOverGas uint64, err error) { contractAddr = crypto.CreateAddress(caller.Address(), evm.StateDB.GetNonce(caller.Address())) - return evm.create(caller, &codeAndHash{code: code}, gas, value, contractAddr) + return evm.create(caller, &codeAndHash{code: code}, gas, value, contractAddr, CREATE) } // Create2 creates a new contract using code as deployment code. @@ -474,7 +515,7 @@ func (evm *EVM) Create(caller ContractRef, code []byte, gas uint64, value *big.I func (evm *EVM) Create2(caller ContractRef, code []byte, gas uint64, endowment *big.Int, salt *uint256.Int) (ret []byte, contractAddr common.Address, leftOverGas uint64, err error) { codeAndHash := &codeAndHash{code: code} contractAddr = crypto.CreateAddress2(caller.Address(), salt.Bytes32(), codeAndHash.Hash().Bytes()) - return evm.create(caller, codeAndHash, gas, endowment, contractAddr) + return evm.create(caller, codeAndHash, gas, endowment, contractAddr, CREATE2) } // ChainConfig returns the environment's chain configuration diff --git a/core/vm/instructions.go b/core/vm/instructions.go index 6c8c6e6e6..bda480f08 100644 --- a/core/vm/instructions.go +++ b/core/vm/instructions.go @@ -791,6 +791,10 @@ func opSuicide(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([] balance := interpreter.evm.StateDB.GetBalance(scope.Contract.Address()) interpreter.evm.StateDB.AddBalance(beneficiary.Bytes20(), balance) interpreter.evm.StateDB.Suicide(scope.Contract.Address()) + if interpreter.cfg.Debug { + interpreter.cfg.Tracer.CaptureEnter(SELFDESTRUCT, scope.Contract.Address(), beneficiary.Bytes20(), []byte{}, 0, balance) + interpreter.cfg.Tracer.CaptureExit([]byte{}, 0, nil) + } return nil, nil } diff --git a/core/vm/logger.go b/core/vm/logger.go index 9be5e7e5e..52dc0b8a0 100644 --- a/core/vm/logger.go +++ b/core/vm/logger.go @@ -106,6 +106,8 @@ func (s *StructLog) ErrorString() string { type Tracer interface { CaptureStart(env *EVM, from common.Address, to common.Address, create bool, input []byte, gas uint64, value *big.Int) CaptureState(env *EVM, pc uint64, op OpCode, gas, cost uint64, scope *ScopeContext, rData []byte, depth int, err error) + CaptureEnter(typ OpCode, from common.Address, to common.Address, input []byte, gas uint64, value *big.Int) + CaptureExit(output []byte, gasUsed uint64, err error) CaptureFault(env *EVM, pc uint64, op OpCode, gas, cost uint64, scope *ScopeContext, depth int, err error) CaptureEnd(output []byte, gasUsed uint64, t time.Duration, err error) } @@ -225,6 +227,11 @@ func (l *StructLogger) CaptureEnd(output []byte, gasUsed uint64, t time.Duration } } +func (l *StructLogger) CaptureEnter(typ OpCode, from common.Address, to common.Address, input []byte, gas uint64, value *big.Int) { +} + +func (l *StructLogger) CaptureExit(output []byte, gasUsed uint64, err error) {} + // StructLogs returns the captured log entries. func (l *StructLogger) StructLogs() []StructLog { return l.logs } @@ -342,3 +349,8 @@ func (t *mdLogger) CaptureEnd(output []byte, gasUsed uint64, tm time.Duration, e fmt.Fprintf(t.out, "\nOutput: `0x%x`\nConsumed gas: `%d`\nError: `%v`\n", output, gasUsed, err) } + +func (t *mdLogger) CaptureEnter(typ OpCode, from common.Address, to common.Address, input []byte, gas uint64, value *big.Int) { +} + +func (t *mdLogger) CaptureExit(output []byte, gasUsed uint64, err error) {} diff --git a/core/vm/logger_json.go b/core/vm/logger_json.go index 8656ce9d7..479a00c0a 100644 --- a/core/vm/logger_json.go +++ b/core/vm/logger_json.go @@ -87,3 +87,8 @@ func (l *JSONLogger) CaptureEnd(output []byte, gasUsed uint64, t time.Duration, } l.encoder.Encode(endLog{common.Bytes2Hex(output), math.HexOrDecimal64(gasUsed), t, errMsg}) } + +func (l *JSONLogger) CaptureEnter(typ OpCode, from common.Address, to common.Address, input []byte, gas uint64, value *big.Int) { +} + +func (l *JSONLogger) CaptureExit(output []byte, gasUsed uint64, err error) {} diff --git a/core/vm/runtime/runtime_test.go b/core/vm/runtime/runtime_test.go index dcf2d0d44..9f4bafbc7 100644 --- a/core/vm/runtime/runtime_test.go +++ b/core/vm/runtime/runtime_test.go @@ -33,6 +33,7 @@ import ( "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" + "github.com/ethereum/go-ethereum/eth/tracers" "github.com/ethereum/go-ethereum/params" ) @@ -342,11 +343,21 @@ func (s *stepCounter) CaptureState(env *vm.EVM, pc uint64, op vm.OpCode, gas, co // benchmarkNonModifyingCode benchmarks code, but if the code modifies the // state, this should not be used, since it does not reset the state between runs. -func benchmarkNonModifyingCode(gas uint64, code []byte, name string, b *testing.B) { +func benchmarkNonModifyingCode(gas uint64, code []byte, name string, tracerCode string, b *testing.B) { cfg := new(Config) setDefaults(cfg) cfg.State, _ = state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) cfg.GasLimit = gas + if len(tracerCode) > 0 { + tracer, err := tracers.New(tracerCode, new(tracers.Context)) + if err != nil { + b.Fatal(err) + } + cfg.EVMConfig = vm.Config{ + Debug: true, + Tracer: tracer, + } + } var ( destination = common.BytesToAddress([]byte("contract")) vmenv = NewEnv(cfg) @@ -486,12 +497,12 @@ func BenchmarkSimpleLoop(b *testing.B) { // Tracer: tracer, // }}) // 100M gas - benchmarkNonModifyingCode(100000000, staticCallIdentity, "staticcall-identity-100M", b) - benchmarkNonModifyingCode(100000000, callIdentity, "call-identity-100M", b) - benchmarkNonModifyingCode(100000000, loopingCode, "loop-100M", b) - benchmarkNonModifyingCode(100000000, callInexistant, "call-nonexist-100M", b) - benchmarkNonModifyingCode(100000000, callEOA, "call-EOA-100M", b) - benchmarkNonModifyingCode(100000000, calllRevertingContractWithInput, "call-reverting-100M", b) + benchmarkNonModifyingCode(100000000, staticCallIdentity, "staticcall-identity-100M", "", b) + benchmarkNonModifyingCode(100000000, callIdentity, "call-identity-100M", "", b) + benchmarkNonModifyingCode(100000000, loopingCode, "loop-100M", "", b) + benchmarkNonModifyingCode(100000000, callInexistant, "call-nonexist-100M", "", b) + benchmarkNonModifyingCode(100000000, callEOA, "call-EOA-100M", "", b) + benchmarkNonModifyingCode(100000000, calllRevertingContractWithInput, "call-reverting-100M", "", b) //benchmarkNonModifyingCode(10000000, staticCallIdentity, "staticcall-identity-10M", b) //benchmarkNonModifyingCode(10000000, loopingCode, "loop-10M", b) @@ -688,3 +699,241 @@ func TestColdAccountAccessCost(t *testing.T) { } } } + +func TestRuntimeJSTracer(t *testing.T) { + jsTracers := []string{ + `{enters: 0, exits: 0, enterGas: 0, gasUsed: 0, steps:0, + step: function() { this.steps++}, + fault: function() {}, + result: function() { + return [this.enters, this.exits,this.enterGas,this.gasUsed, this.steps].join(",") + }, + enter: function(frame) { + this.enters++; + this.enterGas = frame.getGas(); + }, + exit: function(res) { + this.exits++; + this.gasUsed = res.getGasUsed(); + }}`, + `{enters: 0, exits: 0, enterGas: 0, gasUsed: 0, steps:0, + fault: function() {}, + result: function() { + return [this.enters, this.exits,this.enterGas,this.gasUsed, this.steps].join(",") + }, + enter: function(frame) { + this.enters++; + this.enterGas = frame.getGas(); + }, + exit: function(res) { + this.exits++; + this.gasUsed = res.getGasUsed(); + }}`} + tests := []struct { + code []byte + // One result per tracer + results []string + }{ + { + // CREATE + code: []byte{ + // Store initcode in memory at 0x00 (5 bytes left-padded to 32 bytes) + byte(vm.PUSH5), + // Init code: PUSH1 0, PUSH1 0, RETURN (3 steps) + byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, byte(vm.RETURN), + byte(vm.PUSH1), 0, + byte(vm.MSTORE), + // length, offset, value + byte(vm.PUSH1), 5, byte(vm.PUSH1), 27, byte(vm.PUSH1), 0, + byte(vm.CREATE), + byte(vm.POP), + }, + results: []string{`"1,1,4294935775,6,12"`, `"1,1,4294935775,6,0"`}, + }, + { + // CREATE2 + code: []byte{ + // Store initcode in memory at 0x00 (5 bytes left-padded to 32 bytes) + byte(vm.PUSH5), + // Init code: PUSH1 0, PUSH1 0, RETURN (3 steps) + byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, byte(vm.RETURN), + byte(vm.PUSH1), 0, + byte(vm.MSTORE), + // salt, length, offset, value + byte(vm.PUSH1), 1, byte(vm.PUSH1), 5, byte(vm.PUSH1), 27, byte(vm.PUSH1), 0, + byte(vm.CREATE2), + byte(vm.POP), + }, + results: []string{`"1,1,4294935766,6,13"`, `"1,1,4294935766,6,0"`}, + }, + { + // CALL + code: []byte{ + // outsize, outoffset, insize, inoffset + byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, + byte(vm.PUSH1), 0, // value + byte(vm.PUSH1), 0xbb, //address + byte(vm.GAS), // gas + byte(vm.CALL), + byte(vm.POP), + }, + results: []string{`"1,1,4294964716,6,13"`, `"1,1,4294964716,6,0"`}, + }, + { + // CALLCODE + code: []byte{ + // outsize, outoffset, insize, inoffset + byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, + byte(vm.PUSH1), 0, // value + byte(vm.PUSH1), 0xcc, //address + byte(vm.GAS), // gas + byte(vm.CALLCODE), + byte(vm.POP), + }, + results: []string{`"1,1,4294964716,6,13"`, `"1,1,4294964716,6,0"`}, + }, + { + // STATICCALL + code: []byte{ + // outsize, outoffset, insize, inoffset + byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, + byte(vm.PUSH1), 0xdd, //address + byte(vm.GAS), // gas + byte(vm.STATICCALL), + byte(vm.POP), + }, + results: []string{`"1,1,4294964719,6,12"`, `"1,1,4294964719,6,0"`}, + }, + { + // DELEGATECALL + code: []byte{ + // outsize, outoffset, insize, inoffset + byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, + byte(vm.PUSH1), 0xee, //address + byte(vm.GAS), // gas + byte(vm.DELEGATECALL), + byte(vm.POP), + }, + results: []string{`"1,1,4294964719,6,12"`, `"1,1,4294964719,6,0"`}, + }, + { + // CALL self-destructing contract + code: []byte{ + // outsize, outoffset, insize, inoffset + byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, + byte(vm.PUSH1), 0, // value + byte(vm.PUSH1), 0xff, //address + byte(vm.GAS), // gas + byte(vm.CALL), + byte(vm.POP), + }, + results: []string{`"2,2,0,5003,12"`, `"2,2,0,5003,0"`}, + }, + } + calleeCode := []byte{ + byte(vm.PUSH1), 0, + byte(vm.PUSH1), 0, + byte(vm.RETURN), + } + depressedCode := []byte{ + byte(vm.PUSH1), 0xaa, + byte(vm.SELFDESTRUCT), + } + main := common.HexToAddress("0xaa") + for i, jsTracer := range jsTracers { + for j, tc := range tests { + statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) + statedb.SetCode(main, tc.code) + statedb.SetCode(common.HexToAddress("0xbb"), calleeCode) + statedb.SetCode(common.HexToAddress("0xcc"), calleeCode) + statedb.SetCode(common.HexToAddress("0xdd"), calleeCode) + statedb.SetCode(common.HexToAddress("0xee"), calleeCode) + statedb.SetCode(common.HexToAddress("0xff"), depressedCode) + + tracer, err := tracers.New(jsTracer, new(tracers.Context)) + if err != nil { + t.Fatal(err) + } + _, _, err = Call(main, nil, &Config{ + State: statedb, + EVMConfig: vm.Config{ + Debug: true, + Tracer: tracer, + }}) + if err != nil { + t.Fatal("didn't expect error", err) + } + res, err := tracer.GetResult() + if err != nil { + t.Fatal(err) + } + if have, want := string(res), tc.results[i]; have != want { + t.Errorf("wrong result for tracer %d testcase %d, have \n%v\nwant\n%v\n", i, j, have, want) + } + } + } +} + +func TestJSTracerCreateTx(t *testing.T) { + jsTracer := ` + {enters: 0, exits: 0, + step: function() {}, + fault: function() {}, + result: function() { return [this.enters, this.exits].join(",") }, + enter: function(frame) { this.enters++ }, + exit: function(res) { this.exits++ }}` + code := []byte{byte(vm.PUSH1), 0, byte(vm.PUSH1), 0, byte(vm.RETURN)} + + statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase()), nil) + tracer, err := tracers.New(jsTracer, new(tracers.Context)) + if err != nil { + t.Fatal(err) + } + _, _, _, err = Create(code, &Config{ + State: statedb, + EVMConfig: vm.Config{ + Debug: true, + Tracer: tracer, + }}) + if err != nil { + t.Fatal(err) + } + + res, err := tracer.GetResult() + if err != nil { + t.Fatal(err) + } + if have, want := string(res), `"0,0"`; have != want { + t.Errorf("wrong result for tracer, have \n%v\nwant\n%v\n", have, want) + } +} + +func BenchmarkTracerStepVsCallFrame(b *testing.B) { + // Simply pushes and pops some values in a loop + code := []byte{ + byte(vm.JUMPDEST), + byte(vm.PUSH1), 0, + byte(vm.PUSH1), 0, + byte(vm.POP), + byte(vm.POP), + byte(vm.PUSH1), 0, // jumpdestination + byte(vm.JUMP), + } + + stepTracer := ` + { + step: function() {}, + fault: function() {}, + result: function() {}, + }` + callFrameTracer := ` + { + enter: function() {}, + exit: function() {}, + fault: function() {}, + result: function() {}, + }` + + benchmarkNonModifyingCode(10000000, code, "tracer-step-10M", stepTracer, b) + benchmarkNonModifyingCode(10000000, code, "tracer-call-frame-10M", callFrameTracer, b) +} diff --git a/core/vm/stack.go b/core/vm/stack.go index c71d2653a..220f97c89 100644 --- a/core/vm/stack.go +++ b/core/vm/stack.go @@ -91,7 +91,7 @@ func (st *Stack) Print() { fmt.Println("### stack ###") if len(st.data) > 0 { for i, val := range st.data { - fmt.Printf("%-3d %v\n", i, val) + fmt.Printf("%-3d %s\n", i, val.String()) } } else { fmt.Println("-- empty --") diff --git a/eth/tracers/internal/tracers/assets.go b/eth/tracers/internal/tracers/assets.go index 7f45ab286..52cc7f39f 100644 --- a/eth/tracers/internal/tracers/assets.go +++ b/eth/tracers/internal/tracers/assets.go @@ -2,7 +2,8 @@ // sources: // 4byte_tracer.js (2.933kB) // bigram_tracer.js (1.712kB) -// call_tracer.js (8.956kB) +// call_tracer.js (4.251kB) +// call_tracer_legacy.js (8.956kB) // evmdis_tracer.js (4.195kB) // noop_tracer.js (1.271kB) // opcount_tracer.js (1.372kB) @@ -117,7 +118,7 @@ func bigram_tracerJs() (*asset, error) { return a, nil } -var _call_tracerJs = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xd4\x5a\xdf\x6f\x1b\x37\xf2\x7f\x96\xfe\x8a\x89\x1f\x6a\x09\x51\x24\x39\xe9\xb7\x5f\xc0\xae\x7a\x50\x1d\x25\x35\xe0\xc6\x81\xad\x34\x08\x82\x3c\x50\xbb\xb3\x12\x6b\x8a\xdc\x92\x5c\xc9\xba\xd6\xff\xfb\x61\x86\xdc\xd5\xae\x24\x3b\xbe\x5e\x71\xe8\xbd\x69\x97\x33\xc3\xe1\xcc\x67\x7e\x71\x35\x18\xc0\xb9\xc9\x37\x56\xce\x17\x1e\x5e\x0e\x4f\xfe\x1f\xa6\x0b\x84\xb9\x79\x81\x7e\x81\x16\x8b\x25\x8c\x0b\xbf\x30\xd6\xb5\x07\x03\x98\x2e\xa4\x83\x4c\x2a\x04\xe9\x20\x17\xd6\x83\xc9\xc0\xef\xd0\x2b\x39\xb3\xc2\x6e\xfa\xed\xc1\x20\xf0\x1c\x5c\x26\x09\x99\x45\x04\x67\x32\xbf\x16\x16\x4f\x61\x63\x0a\x48\x84\x06\x8b\xa9\x74\xde\xca\x59\xe1\x11\xa4\x07\xa1\xd3\x81\xb1\xb0\x34\xa9\xcc\x36\x24\x52\x7a\x28\x74\x8a\x96\xb7\xf6\x68\x97\xae\xd4\xe3\xed\xbb\x0f\x70\x89\xce\xa1\x85\xb7\xa8\xd1\x0a\x05\xef\x8b\x99\x92\x09\x5c\xca\x04\xb5\x43\x10\x0e\x72\x7a\xe3\x16\x98\xc2\x8c\xc5\x11\xe3\x1b\x52\xe5\x26\xaa\x02\x6f\x4c\xa1\x53\xe1\xa5\xd1\x3d\x40\x49\x9a\xc3\x0a\xad\x93\x46\xc3\xab\x72\xab\x28\xb0\x07\xc6\x92\x90\x8e\xf0\x74\x00\x0b\x26\x27\xbe\x2e\x08\xbd\x01\x25\xfc\x96\xf5\x09\x06\xd9\x9e\x3b\x05\xa9\x79\x9b\x85\xc9\x11\xfc\x42\x78\x3a\xf5\x5a\x2a\x05\x33\x84\xc2\x61\x56\xa8\x1e\x49\x9b\x15\x1e\x3e\x5e\x4c\x7f\xba\xfa\x30\x85\xf1\xbb\x4f\xf0\x71\x7c\x7d\x3d\x7e\x37\xfd\x74\x06\x6b\xe9\x17\xa6\xf0\x80\x2b\x0c\xa2\xe4\x32\x57\x12\x53\x58\x0b\x6b\x85\xf6\x1b\x30\x19\x49\xf8\x79\x72\x7d\xfe\xd3\xf8\xdd\x74\xfc\xe3\xc5\xe5\xc5\xf4\x13\x18\x0b\x6f\x2e\xa6\xef\x26\x37\x37\xf0\xe6\xea\x1a\xc6\xf0\x7e\x7c\x3d\xbd\x38\xff\x70\x39\xbe\x86\xf7\x1f\xae\xdf\x5f\xdd\x4c\xfa\x70\x83\xa4\x15\x12\xff\xd7\x6d\x9e\xb1\xf7\x2c\x42\x8a\x5e\x48\xe5\x4a\x4b\x7c\x32\x05\xb8\x85\x29\x54\x0a\x0b\xb1\x42\xb0\x98\xa0\x5c\x61\x0a\x02\x12\x93\x6f\x9e\xec\x54\x92\x25\x94\xd1\x73\x3e\xf3\x83\x80\x84\x8b\x0c\xb4\xf1\x3d\x70\x88\xf0\xfd\xc2\xfb\xfc\x74\x30\x58\xaf\xd7\xfd\xb9\x2e\xfa\xc6\xce\x07\x2a\x88\x73\x83\x1f\xfa\x6d\x92\x99\x08\xa5\xa6\x56\x24\x68\xc9\x39\x02\xb2\x82\xcc\xaf\xcc\x5a\x83\xb7\x42\x3b\x91\x90\xab\xe9\x77\xc2\x60\x14\x1e\xf0\x8e\x9e\xbc\x23\xd0\x82\xc5\xdc\x58\xfa\xad\x54\x89\x33\xa9\x3d\x5a\x2d\x14\xcb\x76\xb0\x14\x29\xc2\x6c\x03\xa2\x2e\xb0\x57\x3f\x0c\xc1\x28\xb8\x1b\xa4\xce\x8c\x5d\x32\x2c\xfb\xed\xdf\xdb\xad\xa8\xa1\xf3\x22\xb9\x25\x05\x49\x7e\x52\x58\x8b\xda\x93\x29\x0b\xeb\xe4\x0a\x99\x04\x02\x4d\xb4\xe7\xe4\x97\x9f\x01\xef\x30\x29\x82\xa4\x56\x25\xe4\x14\x3e\xff\x7e\xff\xa5\xd7\x66\xd1\x29\xba\x04\x75\x8a\x29\x9f\xef\xd6\xc1\x7a\xc1\x16\x85\x35\x1e\xaf\x10\x7e\x2d\x9c\xaf\xd1\x64\xd6\x2c\x41\x68\x30\x05\x21\xbe\x6e\x1d\xa9\xbd\x61\x81\x82\x7e\x6b\xb4\xac\x51\xbf\xdd\xaa\x98\x4f\x21\x13\xca\x61\xdc\xd7\x79\xcc\xe9\x34\x52\xaf\xcc\x2d\x49\x36\x96\x20\x6c\x37\x60\xf2\xc4\xa4\x31\x18\xe8\x1c\xd5\x31\xd0\xf5\xdb\x2d\xe2\x3b\x85\xac\xd0\xbc\x6d\x47\x99\x79\x0f\xd2\x59\x17\x7e\x6f\xb7\x48\xec\xb9\xc8\x7d\x61\x91\xed\x89\xd6\x1a\xeb\x40\x2e\x97\x98\x4a\xe1\x51\x6d\xda\xad\xd6\x4a\xd8\xb0\x00\x23\x50\x66\xde\x9f\xa3\x9f\xd0\x63\xa7\x7b\xd6\x6e\xb5\x64\x06\x9d\xb0\xfa\x6c\x34\xe2\xec\x93\x49\x8d\x69\x10\xdf\xf2\x0b\xe9\xfa\x99\x28\x94\xaf\xf6\x25\xa6\x96\x45\x5f\x58\x4d\x3f\xef\x83\x16\x1f\x11\x8c\x56\x1b\x48\x28\xcb\x88\x19\x85\xa7\xdb\x38\x8f\xcb\x78\x38\xd7\x83\x4c\x38\x32\xa1\xcc\x60\x8d\x90\x5b\x7c\x91\x2c\x90\x7c\xa7\x13\x8c\x5a\xba\x8d\x63\xa7\x8e\x80\x76\xeb\x9b\xbc\xef\xcd\xbb\x62\x39\x43\xdb\xe9\xc2\x37\x30\xbc\xcb\x86\x5d\x18\x8d\xf8\x47\xa9\x7b\xe4\x89\xfa\x92\x14\x93\xc7\x83\x32\xff\x8d\xb7\x52\xcf\xc3\x59\xa3\xae\x17\x19\x08\xd0\xb8\x86\xc4\x68\x06\x35\x79\x65\x86\x52\xcf\x21\xb1\x28\x3c\xa6\x3d\x10\x69\x0a\xde\x04\xe4\x55\x38\x6b\x6e\x09\xdf\x7c\x03\x1d\xda\x6c\x04\xc7\xe7\xd7\x93\xf1\x74\x72\x0c\x7f\xfc\x01\xe1\xcd\x51\x78\xf3\xf2\xa8\x5b\xd3\x4c\xea\xab\x2c\x8b\xca\xb1\xc0\x7e\x8e\x78\xdb\x39\xe9\xf6\x57\x42\x15\x78\x95\x05\x35\x23\xed\x44\xa7\x30\x8a\x3c\xcf\x77\x79\x5e\x36\x78\x88\x69\x30\x80\xb1\x73\xb8\x9c\x29\xdc\x0f\xc8\x18\xb1\x1c\xbc\xce\x53\xc6\x22\xf4\x25\x66\x99\x2b\x24\x54\x95\xbb\x46\xf3\xb3\xc6\x2d\xbf\xc9\xf1\x14\x00\xc0\xe4\x3d\x7e\x41\xb1\xc0\x2f\xbc\xf9\x09\xef\xd8\x47\xa5\x09\x09\x55\xe3\x34\xb5\xe8\x5c\xa7\xdb\x0d\xe4\x52\xe7\x85\x3f\x6d\x90\x2f\x71\x69\xec\xa6\xef\x28\x21\x75\xf8\x68\xbd\x70\xd2\x92\x67\x2e\xdc\x85\x26\x9e\x88\xd4\xb7\xc2\x75\xb6\x4b\xe7\xc6\xf9\xd3\x72\x89\x1e\xca\x35\xb6\x05\xb1\x1d\x0f\xef\x8e\xf7\xad\x35\xec\x6e\x91\x70\xf2\x5d\x97\x58\xee\xcf\x2a\x7c\x57\x69\xa2\x9f\x17\x6e\xd1\x61\x38\x6d\x57\xb7\xa9\x60\x04\xde\x16\x78\x10\xfe\x0c\xa9\x7d\x38\x39\x54\x19\xe5\x12\x6f\x8b\x84\x61\x35\x17\x9c\x69\x38\xd2\x05\x65\x5e\x57\xcc\xd8\xe6\xde\x98\x7d\x74\x45\x70\xdd\x4c\x2e\xdf\xbc\x9e\xdc\x4c\xaf\x3f\x9c\x4f\x8f\x6b\x70\x52\x98\x79\x52\xaa\x79\x06\x85\x7a\xee\x17\xac\x3f\x89\x6b\xae\x7e\x26\x9e\x17\x27\x5f\xc2\x1b\x18\x1d\x08\xf9\xd6\xe3\x1c\xf0\xf9\x0b\xcb\xbe\xdf\x37\x5f\x93\x34\x18\xf3\xaf\x41\x92\x37\x4c\x5c\x92\x7b\x53\x12\x3c\xee\xe7\xbf\x18\x54\xe9\x8c\x28\x7e\x14\x4a\xe8\x04\x1f\xd1\x79\x1f\x6b\xf5\xa4\x79\x20\x0f\x2d\xd1\x2f\x4c\xca\x85\x21\x11\xa1\xb6\x94\x08\x4a\x8d\xc6\x7f\x3f\x1b\x8d\x2f\x2f\x6b\xb9\x88\x9f\xcf\xaf\x5e\xd7\xf3\xd3\xf1\xeb\xc9\xe5\xe4\xed\x78\x3a\xd9\xa5\xbd\x99\x8e\xa7\x17\xe7\xfc\xb6\x4c\x5d\x83\x01\xdc\xdc\xca\x9c\x2b\x0c\xe7\x6d\xb3\xcc\xb9\x55\xae\xf4\x75\x3d\xf0\x0b\x43\x4d\xa8\x8d\x05\x34\x13\x3a\x29\x0b\x9b\x2b\x01\xeb\x0d\xc1\xf5\x21\xe7\x9d\xec\x38\xaf\x82\xb0\x74\xef\x2d\xc6\x4d\xd3\x8e\x37\xa5\x5e\x5b\x83\x06\x34\x72\xf2\xe7\x04\xdb\x79\xfa\x21\xe1\x1f\x30\x84\x53\x38\x89\x59\xf4\x91\x34\xfd\x12\x9e\x93\xf8\x3f\x91\xac\x5f\x1d\xe0\xfc\x7b\xa6\xec\xbd\x40\xfb\xef\xa7\x72\x53\xf8\xab\x2c\x3b\x85\x5d\x23\x7e\xbb\x67\xc4\x8a\xfe\x12\xf5\x3e\xfd\xff\xed\xd1\x6f\xd3\x3e\xa1\xca\xe4\xf0\x6c\x0f\x22\x21\xe9\x3e\xdb\x89\x83\x68\x5c\x6e\xef\x58\x1a\x8c\x1e\x28\x34\x2f\x9b\x18\x7e\x28\x53\xfe\x47\x85\xe6\x60\x9b\x4a\xcd\x68\xb3\x11\xed\x81\x45\x6f\x25\xae\x68\xd4\x3c\x76\x2c\x92\x1a\x76\xb3\xa6\xf4\xd5\x87\x8f\x18\x24\x6a\x44\x4e\x2e\xb1\xc1\xa7\xfe\x8c\x7b\x5e\x6a\xd2\xe3\xa8\xc6\x10\x13\xdc\x87\x5b\x84\xa5\xd8\xd0\xa8\x96\x15\xfa\x76\x03\x73\xe1\x20\xdd\x68\xb1\x94\x89\x0b\xf2\xb8\xb9\xb7\x38\x17\x96\xc5\x5a\xfc\xad\x40\x47\x73\x1f\x01\x59\x24\xbe\x10\x4a\x6d\x60\x2e\x69\x78\x23\xee\xce\xcb\x57\xc3\x21\x38\x2f\x73\xd4\x69\x0f\xbe\x7b\x35\xf8\xee\x5b\xb0\x85\xc2\x6e\xbf\x5d\x2b\x61\xd5\x51\xa3\x37\x68\x21\xa2\xe7\x35\xe6\x7e\xd1\xe9\xc2\x0f\x0f\xd4\xc2\x07\x0a\xdb\x41\x5a\x78\x01\x27\x5f\xfa\xa4\xd7\xa8\x81\xdb\xe0\x49\x40\xe5\x30\x4a\xa3\x81\xf7\xea\xf5\x55\xe7\x56\x58\xa1\xc4\x0c\xbb\xa7\x3c\x00\xb3\xad\xd6\x22\x4e\x40\xe4\x14\xc8\x95\x90\x1a\x44\x92\x98\x42\x7b\x32\x7c\x39\xcc\xa8\x0d\xe5\xf7\x63\x5f\xca\xe3\x59\x51\x24\x09\x3a\x57\xa6\x7b\xf6\x1a\xa9\x23\x96\xc4\x0d\x52\x3b\x99\x62\xcd\x2b\x94\x1d\x0c\xa7\xe6\x48\x41\xa3\x74\x29\x70\x69\x1c\x6d\x32\x43\x58\x5b\x1a\xbc\x9c\xd4\x09\xdf\x3c\xa4\x48\xd6\x76\x60\x34\x08\x50\x86\xaf\x3b\x38\xc6\x41\xd8\xb9\xeb\x87\x7c\x4f\xdb\x52\xce\xd1\x66\xdd\x6f\x02\xb9\x0e\x55\x1e\x71\x76\x5a\x21\x0d\x78\x27\x9d\xe7\x8e\x9a\xb4\x94\x0e\x02\x92\xa5\x9e\xf7\x20\x37\x39\xe7\xe9\xaf\x95\xb3\x98\xac\xaf\x27\xbf\x4c\xae\xab\xc6\xe7\xe9\x4e\x2c\x67\x9e\xa3\x6a\x24\x04\x4b\xf3\x96\xc7\xf4\xe8\xc0\x10\x73\x00\x50\xa3\x07\x00\x45\xf2\xb7\xb5\xf1\x7d\xed\x38\x4a\x38\xbf\x75\xcc\x1c\xc3\x3c\x57\x57\xc0\x15\xca\xbb\x9d\xdc\xbd\x9b\x1c\x4c\x5e\x56\x08\x52\x8a\xd3\x0e\x25\xf6\xdd\x49\xa3\xb1\xb0\x1d\x38\xb6\xf8\xbc\xa8\xd9\x78\xcd\xed\x66\x20\xaa\xa5\x06\x5e\x2f\xfb\x56\x11\xaa\x01\xeb\x6e\x0a\x4f\x70\xa0\xfa\xbd\x4d\x7e\x73\xe1\x3e\x38\xf6\x7a\x4c\x7f\x33\x39\xbf\xd0\xbe\x53\x2e\x5e\x68\x78\x01\xe5\x03\x25\x75\x78\xd1\x88\xa2\x03\xd9\xb1\x95\xa2\x42\x8f\xb0\x15\x71\x06\x3b\xaf\x48\x50\x30\x07\x1b\xcd\xa2\xdf\x2f\xce\xc3\x28\x8d\x0c\xf6\xcc\xa2\xef\xe3\x6f\x85\x50\xae\x33\xac\x9a\x85\x70\x02\x6f\xb8\xbc\x8d\xf6\x3a\x49\xe2\x69\xf6\x8e\x67\x35\xb6\x68\x8d\x92\x2d\x74\x82\xe7\x26\xc5\x47\x25\x44\x11\x31\x6d\x54\xbe\x8c\xc0\x3c\xd4\x7b\xb7\xea\x04\x70\x54\x35\x04\x99\x90\xaa\xb0\x78\x74\x06\x07\xd2\x8e\x2b\x6c\x26\x12\xf6\xa5\x43\xe0\x69\xdd\x81\x33\x4b\x5c\x98\x75\x50\xe0\x50\xf2\xda\x07\x47\x85\x83\x9d\xf2\xc1\xd7\x4e\xc2\x41\xe1\xc4\x1c\x6b\xe0\xa8\x0c\x5e\x3a\xea\xe0\x15\xc2\x9f\x86\xce\xf3\xea\xf1\x09\x28\xba\xff\x6b\xe0\xb1\xe3\xe7\xbd\x3e\xa7\x24\xe2\x6e\xa7\xf6\x50\x2a\x1b\x9a\x91\xbf\x97\xe3\x9f\x1c\x61\xbb\xb4\xe1\x68\x4d\xe2\x70\xc0\x6d\x5f\xf3\x75\xf7\x57\xab\x0f\x79\xfe\xa1\x96\x89\x30\xaa\x7f\xc5\xc4\x6f\x71\xca\x5d\x0e\x3d\xe5\x16\x57\xd2\x14\x54\xc0\xf0\x7f\x69\x1c\xae\x5a\xbe\xfb\x76\xeb\x3e\xde\x0b\xb2\xdf\xea\x17\x83\xeb\x45\xbc\xd7\x0e\xdd\x52\xad\x7c\x18\xae\xad\xf1\xba\x30\x0b\x37\xce\x2d\xe6\x7f\xe4\x82\x30\x06\xba\x37\x39\xb5\x03\xb1\x3a\x29\x8b\x22\xdd\x54\x05\xb1\x17\x1a\x11\x58\x08\x9d\xc6\x61\x44\xa4\xa9\x24\x79\x0c\x42\xd2\x50\xcc\x85\xd4\xed\x83\x66\xfc\x6a\x15\x3e\x84\x8c\xbd\xde\xb6\x5e\x48\xe3\x10\x49\x13\x1f\x6b\xdc\x7e\x42\xc1\xdc\x09\xa2\xdd\xbb\xce\x78\x5d\x6a\xb4\x2b\x96\xdc\x09\x83\x58\x09\xa9\x04\x4d\x5f\xdc\x61\xe9\x14\x12\x85\x42\x87\x2f\x1c\x98\x79\xb3\x42\xeb\xda\x4f\x00\xf9\x9f\xc1\xf8\x4e\x56\x2c\x1f\xa3\x39\x9e\x1e\xb3\x4f\x8d\xd8\x70\xfc\x37\x4a\x78\x1f\xe1\x55\x33\x6f\x88\x2c\xe9\xf9\xe3\x17\x6a\xdf\x7e\x5a\x48\x71\xcf\x44\x34\x3f\xc0\xb0\xd6\x97\xff\x5d\x82\x6c\x1f\x62\x97\x55\x7f\x16\x0f\xef\x8d\xe9\x81\x42\xc1\x53\x52\xf9\x69\xaa\xec\x47\x1f\x1b\xda\xca\xe8\x0d\x1d\xdd\x5e\xf8\xf2\x9d\xde\x02\xcb\x1b\x90\xd0\xda\xcf\x10\x35\x48\x8f\x56\xd0\x3c\x44\xe8\x8a\x5f\x53\x48\x4b\xc7\xe2\xd8\x2f\x92\x82\x2e\x0a\x8e\x9f\x36\xa8\x30\x4b\x3d\xef\xb7\x5b\xe1\x7d\x2d\xde\x13\x7f\xb7\x8d\xf7\x50\x01\x99\x33\xde\x09\x54\x57\x02\x89\xbf\xe3\x6e\x91\xc7\xe6\x9d\x7b\x01\x5a\xa3\x57\x61\xa6\xde\xb9\x05\x60\xc6\x78\x13\xb0\x7b\x27\x46\x6b\xfc\xae\x01\x70\x26\x9d\x0b\x17\xc4\xec\x84\x84\xbf\xdb\x8f\x88\x92\x81\x82\xe1\xf4\x30\x03\x2d\x1d\x60\xda\xb9\x99\x20\x62\x7e\x15\x56\x43\x3d\x3f\xad\xaf\x86\x57\xf1\xa0\x72\x59\xb3\x8d\x5c\xb2\x6d\xee\xcf\x0e\x27\xb9\x61\x89\xc7\xc3\xc9\x8c\x6c\x5e\x01\xf6\x01\xd6\xfa\xac\xb1\x4f\xf2\x58\xaa\x64\xe9\x65\x66\x7b\x80\x95\xa5\xd7\x5a\x0e\x7f\xf7\x74\x91\x15\x71\x5d\xc5\x06\x4d\x43\x08\xdf\x36\xee\x2d\x1f\x9a\xb4\x68\x50\x89\x84\x65\x73\x35\x1a\x1d\x0d\xef\xaa\x0f\x23\x31\x57\x35\x68\x4a\x25\x42\x64\x84\xf3\x72\x54\xc8\x7f\x62\xdc\xb6\x1e\x83\xe5\x12\x58\x0c\x1f\x70\xb8\x9b\xa5\x10\x34\x33\x6e\x20\x0a\x47\xa3\xe8\x36\xb6\x52\x74\xd2\x62\x0a\x99\x44\x95\x82\x49\xd1\xf2\xa0\xfb\xab\x33\x3a\x7c\xaa\x43\x2b\x49\x62\xf8\x24\x19\xfe\x1d\xc0\x1f\x4a\xb5\x4c\xd0\x6f\x20\x43\xc1\xdf\xdc\xbc\x81\x5c\x38\x07\x4b\x14\x34\xda\x66\x85\x52\x1b\x30\x36\x45\x12\x5e\xcd\x7a\x14\xd6\x06\x0a\x87\xd6\xc1\x7a\x61\x62\xa9\xe5\x16\x2f\xa7\x6e\x55\xfa\x5e\xbc\xce\x91\x2e\x57\x62\x03\xd2\x53\x59\x8f\x87\xaa\x47\x7a\xf5\xa1\x8b\xbf\x96\x19\x32\xf0\x7e\x98\x97\x53\x61\x33\xce\xf9\x35\x3d\x35\x23\x3c\x0e\x45\xcd\xd8\xde\x5e\x74\x35\x03\xb9\x2c\x3d\xcd\x68\xad\x17\xb2\x66\x48\xf2\x0a\x3f\x35\x83\xb1\xd6\x6a\xf3\x02\x23\xa8\x62\xe0\xa7\x9d\xf0\x64\x2d\x63\x7c\x86\xcf\xba\x15\x39\x3f\xf5\x22\x60\xc8\x8b\x1d\x32\xce\x2d\x6e\x28\x9b\x07\x1b\xd5\x4a\x53\x78\xf1\xf9\x16\x37\x5f\x0e\x57\xa2\x08\xc7\x1a\x5d\x55\x7a\xca\xb0\x08\x6b\x8f\x24\x83\x4a\x0b\x39\x1a\x9e\x81\xfc\xbe\xce\x50\x56\x4f\x90\xcf\x9f\x97\x7b\xd6\xd7\x3f\xcb\x2f\x65\x84\x57\x88\xdf\x59\xef\x36\x34\x8a\x31\x12\x68\x28\x28\xda\xf7\xed\x7f\x05\x00\x00\xff\xff\xfb\x65\x93\x4f\xfc\x22\x00\x00") +var _call_tracerJs = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xd4\x57\x4f\x6f\xdb\xb8\x12\x3f\xcb\x9f\x62\x5e\x0e\xb5\x8d\xba\x56\xd2\x07\xf4\xe0\xd6\x05\xfc\x82\xa4\x35\x90\x97\x04\x8e\xb3\x45\x10\xe4\x40\x5b\x23\x89\x2d\x4d\x0a\x24\x15\xc7\xdb\xfa\xbb\x2f\x86\x94\x64\x49\xb1\xd3\xec\x69\xb1\x39\xc5\xc3\xdf\xfc\x66\x38\xff\x38\x0a\x43\x38\x55\xd9\x46\xf3\x24\xb5\xf0\xfe\xf8\xfd\x09\xcc\x53\x84\x44\xbd\x43\x9b\xa2\xc6\x7c\x05\x93\xdc\xa6\x4a\x9b\x4e\x18\xc2\x3c\xe5\x06\x62\x2e\x10\xb8\x81\x8c\x69\x0b\x2a\x06\xdb\xc2\x0b\xbe\xd0\x4c\x6f\x86\x9d\x30\xf4\x3a\x7b\x8f\x89\x21\xd6\x88\x60\x54\x6c\xd7\x4c\xe3\x08\x36\x2a\x87\x25\x93\xa0\x31\xe2\xc6\x6a\xbe\xc8\x2d\x02\xb7\xc0\x64\x14\x2a\x0d\x2b\x15\xf1\x78\x43\x94\xdc\x42\x2e\x23\xd4\xce\xb4\x45\xbd\x32\xa5\x1f\x5f\x2e\x6f\xe1\x02\x8d\x41\x0d\x5f\x50\xa2\x66\x02\xae\xf3\x85\xe0\x4b\xb8\xe0\x4b\x94\x06\x81\x19\xc8\x48\x62\x52\x8c\x60\xe1\xe8\x48\xf1\x9c\x5c\xb9\x29\x5c\x81\x73\x95\xcb\x88\x59\xae\xe4\x00\x90\x93\xe7\xf0\x88\xda\x70\x25\xe1\xbf\xa5\xa9\x82\x70\x00\x4a\x13\x49\x8f\x59\xba\x80\x06\x95\x91\x5e\x1f\x98\xdc\x80\x60\x76\xa7\xfa\x8a\x80\xec\xee\x1d\x01\x97\xce\x4c\xaa\x32\x04\x9b\x32\x4b\xb7\x5e\x73\x21\x60\x81\x90\x1b\x8c\x73\x31\x20\xb6\x45\x6e\xe1\xdb\x74\xfe\xf5\xea\x76\x0e\x93\xcb\x3b\xf8\x36\x99\xcd\x26\x97\xf3\xbb\x8f\xb0\xe6\x36\x55\xb9\x05\x7c\x44\x4f\xc5\x57\x99\xe0\x18\xc1\x9a\x69\xcd\xa4\xdd\x80\x8a\x89\xe1\xff\x67\xb3\xd3\xaf\x93\xcb\xf9\xe4\x7f\xd3\x8b\xe9\xfc\x0e\x94\x86\xf3\xe9\xfc\xf2\xec\xe6\x06\xce\xaf\x66\x30\x81\xeb\xc9\x6c\x3e\x3d\xbd\xbd\x98\xcc\xe0\xfa\x76\x76\x7d\x75\x73\x36\x84\x1b\x24\xaf\x90\xf4\x7f\x1f\xf3\xd8\x65\x4f\x23\x44\x68\x19\x17\xa6\x8c\xc4\x9d\xca\xc1\xa4\x2a\x17\x11\xa4\xec\x11\x41\xe3\x12\xf9\x23\x46\xc0\x60\xa9\xb2\xcd\xab\x93\x4a\x5c\x4c\x28\x99\xb8\x3b\x1f\x2c\x48\x98\xc6\x20\x95\x1d\x80\x41\x84\x4f\xa9\xb5\xd9\x28\x0c\xd7\xeb\xf5\x30\x91\xf9\x50\xe9\x24\x14\x9e\xce\x84\x9f\x87\x9d\x0e\x91\x2e\x99\x10\xe7\x9a\xad\x70\xae\xd9\x12\x35\xc5\xdd\x38\x7a\x89\x6b\x77\x08\x31\x9d\x82\xd5\x6c\xc9\x65\x02\x2b\xb4\xa9\x8a\x0c\x58\x05\x1a\x33\xa5\x6d\x91\x29\xe0\x32\x56\x7a\xe5\x2a\xca\x39\xbb\xa0\xc4\x70\x69\x51\x4b\x26\x60\x85\xc6\xb0\x04\x5d\x15\x33\x22\x93\x86\x2d\xad\x2b\x99\x9f\x1d\x00\x70\xa6\x8c\x65\xcb\x1f\x23\xb8\xff\xb9\x7d\x18\x38\x61\xcc\x72\x61\x47\x10\xe7\xd2\x61\x7b\x42\x25\x03\x88\x16\x7d\xf0\x3a\xf4\xf7\xc8\x34\x08\x94\x30\x06\x9b\x72\x33\xac\x68\x86\x02\x65\x62\xd3\x0a\xc7\x63\xe8\x11\xee\x33\x9c\xd4\xd5\x4b\x0a\x77\xd3\x67\x1c\x99\xca\x7a\xfd\x06\x96\x68\x9a\xa0\x7b\x81\xf2\xdd\xc9\x83\x17\xc0\x78\x3c\x76\x8d\x1b\x73\x89\x51\xdb\x10\xfd\xbd\xa8\x0c\xf7\x0f\x0d\x85\x6d\xe7\x95\xaa\xc3\x2c\x37\x69\x8f\xfe\xdd\xb9\xeb\x95\xb7\x3e\x92\x1a\x4d\x33\x94\x4b\xfb\xd4\x0e\x65\x18\xc2\xb5\xc6\x8c\xa6\x83\xca\xa9\xab\x8b\xa4\xb9\xd4\x36\x02\xee\xd9\x60\xdc\xba\x9f\xdd\x64\x38\x72\xc9\xb4\x4f\x43\xfa\x31\x68\x1c\xc7\x5a\xad\xdc\xb1\x55\x5f\xf1\x89\x3c\x18\x92\xa8\xdf\x44\x59\x35\x2a\xff\x29\x51\x56\xb5\x30\x8f\x4c\xe4\xce\x52\xf7\xf8\xa9\x0b\x6f\x9d\x3d\x27\x1b\x5a\x75\x63\x35\x97\x49\xef\xe4\x43\x4b\x27\x61\xc6\x13\x17\x3a\x0b\x9e\x4c\xa5\x75\xfc\x09\x33\xfd\x97\x35\x6f\x0d\x46\xa3\xfd\x9a\x74\xf4\x92\x36\x97\x59\x6e\x47\x8d\xfb\x38\x51\x0b\xa6\x72\xeb\x71\x3b\x98\x17\xd5\x70\xdb\x46\x35\xb7\xca\xe1\xb8\xac\xa2\xff\x1c\x2e\x41\x9f\xb7\xaa\xda\x0e\x30\xbc\xda\x1e\x6a\xad\xf4\x2b\xec\x79\xdc\x3e\x7b\xee\x64\x67\x0f\x50\x18\x74\xc6\xe8\xfe\x7f\x97\xbe\xd2\x39\x70\x81\x06\xbc\x41\x0b\x6f\xde\xec\x39\x3e\xc2\x27\x5c\xe6\xd4\x2d\xa0\xf1\x11\xb5\xc5\xe8\x08\x7e\xfd\x2a\xcd\xfa\xf4\x50\xc7\x1f\x1d\x3f\x1d\xf5\x9b\xae\x45\x28\xd0\x62\x13\x5a\x73\xab\xb3\xbb\x82\xcd\xb5\xf4\x91\x89\xb9\x64\x82\xff\x89\x85\x27\xfd\x7a\xff\x22\x0d\xd2\x5a\xfb\xba\xa1\xdc\x9e\x83\xc5\x10\xdb\xd7\x94\x0e\x3f\x4c\xd0\xce\x37\x19\xf6\xfa\xfb\x1a\xd3\x17\x5e\x05\x3c\xd7\x6a\xd5\xeb\xef\x69\xce\x16\x6e\xae\x9e\xa1\x8a\x92\x6f\x01\xa7\x24\x7d\x86\x75\x6d\xd9\x6c\xac\x4a\xe3\x0b\x33\xbd\x7e\xad\xb7\xba\x27\x1f\xba\x07\xdb\xa1\xd2\xfa\x83\x06\x41\xaf\xdf\x2a\x9c\x66\x50\x28\x52\x7e\x62\x8c\x0f\xd8\x2e\x58\x9a\x9d\xbd\xc7\x74\xfb\xc5\x68\xce\xe1\x32\x7b\x4f\xdc\xb6\x93\x37\xf3\x49\xfe\xe7\x9e\x32\x17\x83\x62\x80\xc1\x78\x5f\x0e\xbc\x8b\x45\x26\x08\xf6\x3c\x1b\xcf\xac\x97\xcd\xd8\x22\x38\x23\xf1\x9e\xb7\xb4\x80\xff\xee\xd5\x74\xbe\x96\x0d\x57\x2f\xac\x9d\x85\x2b\x77\xda\xeb\x37\x6d\x14\x23\xe5\x00\x63\xe9\x6c\x73\x6a\xd4\xfd\x73\x30\x6a\x21\xe7\x63\xf7\x74\x76\x36\x99\x9f\x75\x69\x0a\xec\x3d\x79\xdf\xdd\xe7\x3d\xec\x06\x82\xd7\x52\xcf\x20\xdb\x17\xde\x7d\xca\xf5\xbb\x31\x9c\xfc\xeb\x17\x91\x20\x0c\xa1\x1c\x72\xb4\x09\x6b\x64\x16\x0d\xad\xc2\x54\xb2\x6a\xf1\x1d\x97\xb4\x4e\xd2\x9a\x49\x1b\xa8\x83\x42\x84\x86\x6b\x8c\x20\xe6\x28\x22\x50\xf4\x4d\x44\xcb\xf6\x77\xa3\xa4\x23\x34\xa8\x39\x31\xba\xcd\x73\xe8\xbf\xdf\x38\x91\x4a\xbe\x44\xbb\x81\x18\x99\xcd\x35\xd2\xc2\x9a\x31\x63\x60\x85\x4c\x72\x99\xc4\xb9\x10\x1b\x50\x3a\x42\x22\xf7\x13\xd7\x38\x42\xab\x68\xa5\xd5\x06\xd6\xa9\x82\x48\xc9\x6e\xb1\xc6\x66\x1a\xe9\x0b\x65\x00\xdf\x73\x63\xe9\x3b\x26\x13\x6c\x03\xdc\x0e\x3b\x41\x79\xa9\xfa\x7e\x45\x21\x80\x9f\x9d\x20\xa0\xae\x30\x8a\x5e\x0f\x37\x9b\x83\x20\xd8\xed\x49\x65\x0d\x0d\x48\x5c\xed\x47\x4e\x4c\xbf\x9c\xb8\x5a\x88\x8a\xda\x71\xc2\x6a\x03\xda\x4d\x32\x27\xaf\xb6\x9c\xb2\xbb\x4b\xa9\xdf\x60\xea\x3d\xef\x4e\xaa\xed\xc4\x9d\xb8\x5f\x4e\x5e\xad\x23\xb5\xce\x73\x07\xae\x55\x46\x8d\x06\xf2\x5e\xf2\x55\xfd\x4e\x7c\xe5\xfd\x71\x45\x51\xc1\xdd\x2f\x92\x6f\x3b\x41\x40\x59\xec\x51\x70\x7e\xe0\x86\x3e\x05\x7d\x8c\x7c\xcc\x02\x2a\x6f\x2f\xb8\xff\x81\x9b\x87\xe7\xe5\x1c\x04\x41\x50\xf4\x54\x0d\x47\xe2\x6d\xc1\xbf\xa3\x38\xb4\x18\x05\x35\x27\xf8\xf8\xf8\x23\xf0\x4f\x75\x85\x62\xee\x7e\x04\xfe\xf6\x6d\x69\xb2\x7e\x7e\xcf\x1f\xca\x39\x5b\x3d\xdd\xad\xf3\x7e\xdd\xa1\xe2\xad\xf7\x90\x4e\xb0\xed\x6c\x3b\x7f\x05\x00\x00\xff\xff\x71\x10\x40\x55\x9b\x10\x00\x00") func call_tracerJsBytes() ([]byte, error) { return bindataRead( @@ -133,6 +134,26 @@ func call_tracerJs() (*asset, error) { } info := bindataFileInfo{name: "call_tracer.js", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info, digest: [32]uint8{0x96, 0x54, 0x29, 0x1, 0x3b, 0x86, 0xea, 0xb2, 0x35, 0xbd, 0x97, 0xb1, 0x17, 0x8c, 0x17, 0x79, 0x1c, 0x4c, 0x8e, 0x7b, 0xe2, 0x5f, 0x11, 0x59, 0xa0, 0x94, 0x35, 0x43, 0xec, 0x18, 0x2a, 0xd9}} + return a, nil +} + +var _call_tracer_legacyJs = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xd4\x5a\xdf\x6f\x1b\x37\xf2\x7f\x96\xfe\x8a\x89\x1f\x6a\x09\x51\x24\x39\xe9\xb7\x5f\xc0\xae\x7a\x50\x1d\x25\x35\xe0\xc6\x81\xad\x34\x08\x82\x3c\x50\xbb\xb3\x12\x6b\x8a\xdc\x92\x5c\xc9\xba\xd6\xff\xfb\x61\x86\xdc\xd5\xae\x24\x3b\xbe\x5e\x71\xe8\xbd\x69\x97\x33\xc3\xe1\xcc\x67\x7e\x71\x35\x18\xc0\xb9\xc9\x37\x56\xce\x17\x1e\x5e\x0e\x4f\xfe\x1f\xa6\x0b\x84\xb9\x79\x81\x7e\x81\x16\x8b\x25\x8c\x0b\xbf\x30\xd6\xb5\x07\x03\x98\x2e\xa4\x83\x4c\x2a\x04\xe9\x20\x17\xd6\x83\xc9\xc0\xef\xd0\x2b\x39\xb3\xc2\x6e\xfa\xed\xc1\x20\xf0\x1c\x5c\x26\x09\x99\x45\x04\x67\x32\xbf\x16\x16\x4f\x61\x63\x0a\x48\x84\x06\x8b\xa9\x74\xde\xca\x59\xe1\x11\xa4\x07\xa1\xd3\x81\xb1\xb0\x34\xa9\xcc\x36\x24\x52\x7a\x28\x74\x8a\x96\xb7\xf6\x68\x97\xae\xd4\xe3\xed\xbb\x0f\x70\x89\xce\xa1\x85\xb7\xa8\xd1\x0a\x05\xef\x8b\x99\x92\x09\x5c\xca\x04\xb5\x43\x10\x0e\x72\x7a\xe3\x16\x98\xc2\x8c\xc5\x11\xe3\x1b\x52\xe5\x26\xaa\x02\x6f\x4c\xa1\x53\xe1\xa5\xd1\x3d\x40\x49\x9a\xc3\x0a\xad\x93\x46\xc3\xab\x72\xab\x28\xb0\x07\xc6\x92\x90\x8e\xf0\x74\x00\x0b\x26\x27\xbe\x2e\x08\xbd\x01\x25\xfc\x96\xf5\x09\x06\xd9\x9e\x3b\x05\xa9\x79\x9b\x85\xc9\x11\xfc\x42\x78\x3a\xf5\x5a\x2a\x05\x33\x84\xc2\x61\x56\xa8\x1e\x49\x9b\x15\x1e\x3e\x5e\x4c\x7f\xba\xfa\x30\x85\xf1\xbb\x4f\xf0\x71\x7c\x7d\x3d\x7e\x37\xfd\x74\x06\x6b\xe9\x17\xa6\xf0\x80\x2b\x0c\xa2\xe4\x32\x57\x12\x53\x58\x0b\x6b\x85\xf6\x1b\x30\x19\x49\xf8\x79\x72\x7d\xfe\xd3\xf8\xdd\x74\xfc\xe3\xc5\xe5\xc5\xf4\x13\x18\x0b\x6f\x2e\xa6\xef\x26\x37\x37\xf0\xe6\xea\x1a\xc6\xf0\x7e\x7c\x3d\xbd\x38\xff\x70\x39\xbe\x86\xf7\x1f\xae\xdf\x5f\xdd\x4c\xfa\x70\x83\xa4\x15\x12\xff\xd7\x6d\x9e\xb1\xf7\x2c\x42\x8a\x5e\x48\xe5\x4a\x4b\x7c\x32\x05\xb8\x85\x29\x54\x0a\x0b\xb1\x42\xb0\x98\xa0\x5c\x61\x0a\x02\x12\x93\x6f\x9e\xec\x54\x92\x25\x94\xd1\x73\x3e\xf3\x83\x80\x84\x8b\x0c\xb4\xf1\x3d\x70\x88\xf0\xfd\xc2\xfb\xfc\x74\x30\x58\xaf\xd7\xfd\xb9\x2e\xfa\xc6\xce\x07\x2a\x88\x73\x83\x1f\xfa\x6d\x92\x99\x08\xa5\xa6\x56\x24\x68\xc9\x39\x02\xb2\x82\xcc\xaf\xcc\x5a\x83\xb7\x42\x3b\x91\x90\xab\xe9\x77\xc2\x60\x14\x1e\xf0\x8e\x9e\xbc\x23\xd0\x82\xc5\xdc\x58\xfa\xad\x54\x89\x33\xa9\x3d\x5a\x2d\x14\xcb\x76\xb0\x14\x29\xc2\x6c\x03\xa2\x2e\xb0\x57\x3f\x0c\xc1\x28\xb8\x1b\xa4\xce\x8c\x5d\x32\x2c\xfb\xed\xdf\xdb\xad\xa8\xa1\xf3\x22\xb9\x25\x05\x49\x7e\x52\x58\x8b\xda\x93\x29\x0b\xeb\xe4\x0a\x99\x04\x02\x4d\xb4\xe7\xe4\x97\x9f\x01\xef\x30\x29\x82\xa4\x56\x25\xe4\x14\x3e\xff\x7e\xff\xa5\xd7\x66\xd1\x29\xba\x04\x75\x8a\x29\x9f\xef\xd6\xc1\x7a\xc1\x16\x85\x35\x1e\xaf\x10\x7e\x2d\x9c\xaf\xd1\x64\xd6\x2c\x41\x68\x30\x05\x21\xbe\x6e\x1d\xa9\xbd\x61\x81\x82\x7e\x6b\xb4\xac\x51\xbf\xdd\xaa\x98\x4f\x21\x13\xca\x61\xdc\xd7\x79\xcc\xe9\x34\x52\xaf\xcc\x2d\x49\x36\x96\x20\x6c\x37\x60\xf2\xc4\xa4\x31\x18\xe8\x1c\xd5\x31\xd0\xf5\xdb\x2d\xe2\x3b\x85\xac\xd0\xbc\x6d\x47\x99\x79\x0f\xd2\x59\x17\x7e\x6f\xb7\x48\xec\xb9\xc8\x7d\x61\x91\xed\x89\xd6\x1a\xeb\x40\x2e\x97\x98\x4a\xe1\x51\x6d\xda\xad\xd6\x4a\xd8\xb0\x00\x23\x50\x66\xde\x9f\xa3\x9f\xd0\x63\xa7\x7b\xd6\x6e\xb5\x64\x06\x9d\xb0\xfa\x6c\x34\xe2\xec\x93\x49\x8d\x69\x10\xdf\xf2\x0b\xe9\xfa\x99\x28\x94\xaf\xf6\x25\xa6\x96\x45\x5f\x58\x4d\x3f\xef\x83\x16\x1f\x11\x8c\x56\x1b\x48\x28\xcb\x88\x19\x85\xa7\xdb\x38\x8f\xcb\x78\x38\xd7\x83\x4c\x38\x32\xa1\xcc\x60\x8d\x90\x5b\x7c\x91\x2c\x90\x7c\xa7\x13\x8c\x5a\xba\x8d\x63\xa7\x8e\x80\x76\xeb\x9b\xbc\xef\xcd\xbb\x62\x39\x43\xdb\xe9\xc2\x37\x30\xbc\xcb\x86\x5d\x18\x8d\xf8\x47\xa9\x7b\xe4\x89\xfa\x92\x14\x93\xc7\x83\x32\xff\x8d\xb7\x52\xcf\xc3\x59\xa3\xae\x17\x19\x08\xd0\xb8\x86\xc4\x68\x06\x35\x79\x65\x86\x52\xcf\x21\xb1\x28\x3c\xa6\x3d\x10\x69\x0a\xde\x04\xe4\x55\x38\x6b\x6e\x09\xdf\x7c\x03\x1d\xda\x6c\x04\xc7\xe7\xd7\x93\xf1\x74\x72\x0c\x7f\xfc\x01\xe1\xcd\x51\x78\xf3\xf2\xa8\x5b\xd3\x4c\xea\xab\x2c\x8b\xca\xb1\xc0\x7e\x8e\x78\xdb\x39\xe9\xf6\x57\x42\x15\x78\x95\x05\x35\x23\xed\x44\xa7\x30\x8a\x3c\xcf\x77\x79\x5e\x36\x78\x88\x69\x30\x80\xb1\x73\xb8\x9c\x29\xdc\x0f\xc8\x18\xb1\x1c\xbc\xce\x53\xc6\x22\xf4\x25\x66\x99\x2b\x24\x54\x95\xbb\x46\xf3\xb3\xc6\x2d\xbf\xc9\xf1\x14\x00\xc0\xe4\x3d\x7e\x41\xb1\xc0\x2f\xbc\xf9\x09\xef\xd8\x47\xa5\x09\x09\x55\xe3\x34\xb5\xe8\x5c\xa7\xdb\x0d\xe4\x52\xe7\x85\x3f\x6d\x90\x2f\x71\x69\xec\xa6\xef\x28\x21\x75\xf8\x68\xbd\x70\xd2\x92\x67\x2e\xdc\x85\x26\x9e\x88\xd4\xb7\xc2\x75\xb6\x4b\xe7\xc6\xf9\xd3\x72\x89\x1e\xca\x35\xb6\x05\xb1\x1d\x0f\xef\x8e\xf7\xad\x35\xec\x6e\x91\x70\xf2\x5d\x97\x58\xee\xcf\x2a\x7c\x57\x69\xa2\x9f\x17\x6e\xd1\x61\x38\x6d\x57\xb7\xa9\x60\x04\xde\x16\x78\x10\xfe\x0c\xa9\x7d\x38\x39\x54\x19\xe5\x12\x6f\x8b\x84\x61\x35\x17\x9c\x69\x38\xd2\x05\x65\x5e\x57\xcc\xd8\xe6\xde\x98\x7d\x74\x45\x70\xdd\x4c\x2e\xdf\xbc\x9e\xdc\x4c\xaf\x3f\x9c\x4f\x8f\x6b\x70\x52\x98\x79\x52\xaa\x79\x06\x85\x7a\xee\x17\xac\x3f\x89\x6b\xae\x7e\x26\x9e\x17\x27\x5f\xc2\x1b\x18\x1d\x08\xf9\xd6\xe3\x1c\xf0\xf9\x0b\xcb\xbe\xdf\x37\x5f\x93\x34\x18\xf3\xaf\x41\x92\x37\x4c\x5c\x92\x7b\x53\x12\x3c\xee\xe7\xbf\x18\x54\xe9\x8c\x28\x7e\x14\x4a\xe8\x04\x1f\xd1\x79\x1f\x6b\xf5\xa4\x79\x20\x0f\x2d\xd1\x2f\x4c\xca\x85\x21\x11\xa1\xb6\x94\x08\x4a\x8d\xc6\x7f\x3f\x1b\x8d\x2f\x2f\x6b\xb9\x88\x9f\xcf\xaf\x5e\xd7\xf3\xd3\xf1\xeb\xc9\xe5\xe4\xed\x78\x3a\xd9\xa5\xbd\x99\x8e\xa7\x17\xe7\xfc\xb6\x4c\x5d\x83\x01\xdc\xdc\xca\x9c\x2b\x0c\xe7\x6d\xb3\xcc\xb9\x55\xae\xf4\x75\x3d\xf0\x0b\x43\x4d\xa8\x8d\x05\x34\x13\x3a\x29\x0b\x9b\x2b\x01\xeb\x0d\xc1\xf5\x21\xe7\x9d\xec\x38\xaf\x82\xb0\x74\xef\x2d\xc6\x4d\xd3\x8e\x37\xa5\x5e\x5b\x83\x06\x34\x72\xf2\xe7\x04\xdb\x79\xfa\x21\xe1\x1f\x30\x84\x53\x38\x89\x59\xf4\x91\x34\xfd\x12\x9e\x93\xf8\x3f\x91\xac\x5f\x1d\xe0\xfc\x7b\xa6\xec\xbd\x40\xfb\xef\xa7\x72\x53\xf8\xab\x2c\x3b\x85\x5d\x23\x7e\xbb\x67\xc4\x8a\xfe\x12\xf5\x3e\xfd\xff\xed\xd1\x6f\xd3\x3e\xa1\xca\xe4\xf0\x6c\x0f\x22\x21\xe9\x3e\xdb\x89\x83\x68\x5c\x6e\xef\x58\x1a\x8c\x1e\x28\x34\x2f\x9b\x18\x7e\x28\x53\xfe\x47\x85\xe6\x60\x9b\x4a\xcd\x68\xb3\x11\xed\x81\x45\x6f\x25\xae\x68\xd4\x3c\x76\x2c\x92\x1a\x76\xb3\xa6\xf4\xd5\x87\x8f\x18\x24\x6a\x44\x4e\x2e\xb1\xc1\xa7\xfe\x8c\x7b\x5e\x6a\xd2\xe3\xa8\xc6\x10\x13\xdc\x87\x5b\x84\xa5\xd8\xd0\xa8\x96\x15\xfa\x76\x03\x73\xe1\x20\xdd\x68\xb1\x94\x89\x0b\xf2\xb8\xb9\xb7\x38\x17\x96\xc5\x5a\xfc\xad\x40\x47\x73\x1f\x01\x59\x24\xbe\x10\x4a\x6d\x60\x2e\x69\x78\x23\xee\xce\xcb\x57\xc3\x21\x38\x2f\x73\xd4\x69\x0f\xbe\x7b\x35\xf8\xee\x5b\xb0\x85\xc2\x6e\xbf\x5d\x2b\x61\xd5\x51\xa3\x37\x68\x21\xa2\xe7\x35\xe6\x7e\xd1\xe9\xc2\x0f\x0f\xd4\xc2\x07\x0a\xdb\x41\x5a\x78\x01\x27\x5f\xfa\xa4\xd7\xa8\x81\xdb\xe0\x49\x40\xe5\x30\x4a\xa3\x81\xf7\xea\xf5\x55\xe7\x56\x58\xa1\xc4\x0c\xbb\xa7\x3c\x00\xb3\xad\xd6\x22\x4e\x40\xe4\x14\xc8\x95\x90\x1a\x44\x92\x98\x42\x7b\x32\x7c\x39\xcc\xa8\x0d\xe5\xf7\x63\x5f\xca\xe3\x59\x51\x24\x09\x3a\x57\xa6\x7b\xf6\x1a\xa9\x23\x96\xc4\x0d\x52\x3b\x99\x62\xcd\x2b\x94\x1d\x0c\xa7\xe6\x48\x41\xa3\x74\x29\x70\x69\x1c\x6d\x32\x43\x58\x5b\x1a\xbc\x9c\xd4\x09\xdf\x3c\xa4\x48\xd6\x76\x60\x34\x08\x50\x86\xaf\x3b\x38\xc6\x41\xd8\xb9\xeb\x87\x7c\x4f\xdb\x52\xce\xd1\x66\xdd\x6f\x02\xb9\x0e\x55\x1e\x71\x76\x5a\x21\x0d\x78\x27\x9d\xe7\x8e\x9a\xb4\x94\x0e\x02\x92\xa5\x9e\xf7\x20\x37\x39\xe7\xe9\xaf\x95\xb3\x98\xac\xaf\x27\xbf\x4c\xae\xab\xc6\xe7\xe9\x4e\x2c\x67\x9e\xa3\x6a\x24\x04\x4b\xf3\x96\xc7\xf4\xe8\xc0\x10\x73\x00\x50\xa3\x07\x00\x45\xf2\xb7\xb5\xf1\x7d\xed\x38\x4a\x38\xbf\x75\xcc\x1c\xc3\x3c\x57\x57\xc0\x15\xca\xbb\x9d\xdc\xbd\x9b\x1c\x4c\x5e\x56\x08\x52\x8a\xd3\x0e\x25\xf6\xdd\x49\xa3\xb1\xb0\x1d\x38\xb6\xf8\xbc\xa8\xd9\x78\xcd\xed\x66\x20\xaa\xa5\x06\x5e\x2f\xfb\x56\x11\xaa\x01\xeb\x6e\x0a\x4f\x70\xa0\xfa\xbd\x4d\x7e\x73\xe1\x3e\x38\xf6\x7a\x4c\x7f\x33\x39\xbf\xd0\xbe\x53\x2e\x5e\x68\x78\x01\xe5\x03\x25\x75\x78\xd1\x88\xa2\x03\xd9\xb1\x95\xa2\x42\x8f\xb0\x15\x71\x06\x3b\xaf\x48\x50\x30\x07\x1b\xcd\xa2\xdf\x2f\xce\xc3\x28\x8d\x0c\xf6\xcc\xa2\xef\xe3\x6f\x85\x50\xae\x33\xac\x9a\x85\x70\x02\x6f\xb8\xbc\x8d\xf6\x3a\x49\xe2\x69\xf6\x8e\x67\x35\xb6\x68\x8d\x92\x2d\x74\x82\xe7\x26\xc5\x47\x25\x44\x11\x31\x6d\x54\xbe\x8c\xc0\x3c\xd4\x7b\xb7\xea\x04\x70\x54\x35\x04\x99\x90\xaa\xb0\x78\x74\x06\x07\xd2\x8e\x2b\x6c\x26\x12\xf6\xa5\x43\xe0\x69\xdd\x81\x33\x4b\x5c\x98\x75\x50\xe0\x50\xf2\xda\x07\x47\x85\x83\x9d\xf2\xc1\xd7\x4e\xc2\x41\xe1\xc4\x1c\x6b\xe0\xa8\x0c\x5e\x3a\xea\xe0\x15\xc2\x9f\x86\xce\xf3\xea\xf1\x09\x28\xba\xff\x6b\xe0\xb1\xe3\xe7\xbd\x3e\xa7\x24\xe2\x6e\xa7\xf6\x50\x2a\x1b\x9a\x91\xbf\x97\xe3\x9f\x1c\x61\xbb\xb4\xe1\x68\x4d\xe2\x70\xc0\x6d\x5f\xf3\x75\xf7\x57\xab\x0f\x79\xfe\xa1\x96\x89\x30\xaa\x7f\xc5\xc4\x6f\x71\xca\x5d\x0e\x3d\xe5\x16\x57\xd2\x14\x54\xc0\xf0\x7f\x69\x1c\xae\x5a\xbe\xfb\x76\xeb\x3e\xde\x0b\xb2\xdf\xea\x17\x83\xeb\x45\xbc\xd7\x0e\xdd\x52\xad\x7c\x18\xae\xad\xf1\xba\x30\x0b\x37\xce\x2d\xe6\x7f\xe4\x82\x30\x06\xba\x37\x39\xb5\x03\xb1\x3a\x29\x8b\x22\xdd\x54\x05\xb1\x17\x1a\x11\x58\x08\x9d\xc6\x61\x44\xa4\xa9\x24\x79\x0c\x42\xd2\x50\xcc\x85\xd4\xed\x83\x66\xfc\x6a\x15\x3e\x84\x8c\xbd\xde\xb6\x5e\x48\xe3\x10\x49\x13\x1f\x6b\xdc\x7e\x42\xc1\xdc\x09\xa2\xdd\xbb\xce\x78\x5d\x6a\xb4\x2b\x96\xdc\x09\x83\x58\x09\xa9\x04\x4d\x5f\xdc\x61\xe9\x14\x12\x85\x42\x87\x2f\x1c\x98\x79\xb3\x42\xeb\xda\x4f\x00\xf9\x9f\xc1\xf8\x4e\x56\x2c\x1f\xa3\x39\x9e\x1e\xb3\x4f\x8d\xd8\x70\xfc\x37\x4a\x78\x1f\xe1\x55\x33\x6f\x88\x2c\xe9\xf9\xe3\x17\x6a\xdf\x7e\x5a\x48\x71\xcf\x44\x34\x3f\xc0\xb0\xd6\x97\xff\x5d\x82\x6c\x1f\x62\x97\x55\x7f\x16\x0f\xef\x8d\xe9\x81\x42\xc1\x53\x52\xf9\x69\xaa\xec\x47\x1f\x1b\xda\xca\xe8\x0d\x1d\xdd\x5e\xf8\xf2\x9d\xde\x02\xcb\x1b\x90\xd0\xda\xcf\x10\x35\x48\x8f\x56\xd0\x3c\x44\xe8\x8a\x5f\x53\x48\x4b\xc7\xe2\xd8\x2f\x92\x82\x2e\x0a\x8e\x9f\x36\xa8\x30\x4b\x3d\xef\xb7\x5b\xe1\x7d\x2d\xde\x13\x7f\xb7\x8d\xf7\x50\x01\x99\x33\xde\x09\x54\x57\x02\x89\xbf\xe3\x6e\x91\xc7\xe6\x9d\x7b\x01\x5a\xa3\x57\x61\xa6\xde\xb9\x05\x60\xc6\x78\x13\xb0\x7b\x27\x46\x6b\xfc\xae\x01\x70\x26\x9d\x0b\x17\xc4\xec\x84\x84\xbf\xdb\x8f\x88\x92\x81\x82\xe1\xf4\x30\x03\x2d\x1d\x60\xda\xb9\x99\x20\x62\x7e\x15\x56\x43\x3d\x3f\xad\xaf\x86\x57\xf1\xa0\x72\x59\xb3\x8d\x5c\xb2\x6d\xee\xcf\x0e\x27\xb9\x61\x89\xc7\xc3\xc9\x8c\x6c\x5e\x01\xf6\x01\xd6\xfa\xac\xb1\x4f\xf2\x58\xaa\x64\xe9\x65\x66\x7b\x80\x95\xa5\xd7\x5a\x0e\x7f\xf7\x74\x91\x15\x71\x5d\xc5\x06\x4d\x43\x08\xdf\x36\xee\x2d\x1f\x9a\xb4\x68\x50\x89\x84\x65\x73\x35\x1a\x1d\x0d\xef\xaa\x0f\x23\x31\x57\x35\x68\x4a\x25\x42\x64\x84\xf3\x72\x54\xc8\x7f\x62\xdc\xb6\x1e\x83\xe5\x12\x58\x0c\x1f\x70\xb8\x9b\xa5\x10\x34\x33\x6e\x20\x0a\x47\xa3\xe8\x36\xb6\x52\x74\xd2\x62\x0a\x99\x44\x95\x82\x49\xd1\xf2\xa0\xfb\xab\x33\x3a\x7c\xaa\x43\x2b\x49\x62\xf8\x24\x19\xfe\x1d\xc0\x1f\x4a\xb5\x4c\xd0\x6f\x20\x43\xc1\xdf\xdc\xbc\x81\x5c\x38\x07\x4b\x14\x34\xda\x66\x85\x52\x1b\x30\x36\x45\x12\x5e\xcd\x7a\x14\xd6\x06\x0a\x87\xd6\xc1\x7a\x61\x62\xa9\xe5\x16\x2f\xa7\x6e\x55\xfa\x5e\xbc\xce\x91\x2e\x57\x62\x03\xd2\x53\x59\x8f\x87\xaa\x47\x7a\xf5\xa1\x8b\xbf\x96\x19\x32\xf0\x7e\x98\x97\x53\x61\x33\xce\xf9\x35\x3d\x35\x23\x3c\x0e\x45\xcd\xd8\xde\x5e\x74\x35\x03\xb9\x2c\x3d\xcd\x68\xad\x17\xb2\x66\x48\xf2\x0a\x3f\x35\x83\xb1\xd6\x6a\xf3\x02\x23\xa8\x62\xe0\xa7\x9d\xf0\x64\x2d\x63\x7c\x86\xcf\xba\x15\x39\x3f\xf5\x22\x60\xc8\x8b\x1d\x32\xce\x2d\x6e\x28\x9b\x07\x1b\xd5\x4a\x53\x78\xf1\xf9\x16\x37\x5f\x0e\x57\xa2\x08\xc7\x1a\x5d\x55\x7a\xca\xb0\x08\x6b\x8f\x24\x83\x4a\x0b\x39\x1a\x9e\x81\xfc\xbe\xce\x50\x56\x4f\x90\xcf\x9f\x97\x7b\xd6\xd7\x3f\xcb\x2f\x65\x84\x57\x88\xdf\x59\xef\x36\x34\x8a\x31\x12\x68\x28\x28\xda\xf7\xed\x7f\x05\x00\x00\xff\xff\xfb\x65\x93\x4f\xfc\x22\x00\x00") + +func call_tracer_legacyJsBytes() ([]byte, error) { + return bindataRead( + _call_tracer_legacyJs, + "call_tracer_legacy.js", + ) +} + +func call_tracer_legacyJs() (*asset, error) { + bytes, err := call_tracer_legacyJsBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "call_tracer_legacy.js", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} a := &asset{bytes: bytes, info: info, digest: [32]uint8{0x46, 0x79, 0xb6, 0xbc, 0xd2, 0xc, 0x25, 0xb1, 0x22, 0x56, 0xef, 0x77, 0xb9, 0x5e, 0x2e, 0xf4, 0xda, 0xb2, 0x2f, 0x53, 0xa4, 0xff, 0xc8, 0xac, 0xbb, 0x75, 0x22, 0x46, 0x59, 0xe3, 0x1d, 0x7d}} return a, nil } @@ -348,15 +369,16 @@ func AssetNames() []string { // _bindata is a table, holding each asset generator, mapped to its name. var _bindata = map[string]func() (*asset, error){ - "4byte_tracer.js": _4byte_tracerJs, - "bigram_tracer.js": bigram_tracerJs, - "call_tracer.js": call_tracerJs, - "evmdis_tracer.js": evmdis_tracerJs, - "noop_tracer.js": noop_tracerJs, - "opcount_tracer.js": opcount_tracerJs, - "prestate_tracer.js": prestate_tracerJs, - "trigram_tracer.js": trigram_tracerJs, - "unigram_tracer.js": unigram_tracerJs, + "4byte_tracer.js": _4byte_tracerJs, + "bigram_tracer.js": bigram_tracerJs, + "call_tracer.js": call_tracerJs, + "call_tracer_legacy.js": call_tracer_legacyJs, + "evmdis_tracer.js": evmdis_tracerJs, + "noop_tracer.js": noop_tracerJs, + "opcount_tracer.js": opcount_tracerJs, + "prestate_tracer.js": prestate_tracerJs, + "trigram_tracer.js": trigram_tracerJs, + "unigram_tracer.js": unigram_tracerJs, } // AssetDebug is true if the assets were built with the debug flag enabled. @@ -403,15 +425,16 @@ type bintree struct { } var _bintree = &bintree{nil, map[string]*bintree{ - "4byte_tracer.js": {_4byte_tracerJs, map[string]*bintree{}}, - "bigram_tracer.js": {bigram_tracerJs, map[string]*bintree{}}, - "call_tracer.js": {call_tracerJs, map[string]*bintree{}}, - "evmdis_tracer.js": {evmdis_tracerJs, map[string]*bintree{}}, - "noop_tracer.js": {noop_tracerJs, map[string]*bintree{}}, - "opcount_tracer.js": {opcount_tracerJs, map[string]*bintree{}}, - "prestate_tracer.js": {prestate_tracerJs, map[string]*bintree{}}, - "trigram_tracer.js": {trigram_tracerJs, map[string]*bintree{}}, - "unigram_tracer.js": {unigram_tracerJs, map[string]*bintree{}}, + "4byte_tracer.js": {_4byte_tracerJs, map[string]*bintree{}}, + "bigram_tracer.js": {bigram_tracerJs, map[string]*bintree{}}, + "call_tracer.js": {call_tracerJs, map[string]*bintree{}}, + "call_tracer_legacy.js": {call_tracer_legacyJs, map[string]*bintree{}}, + "evmdis_tracer.js": {evmdis_tracerJs, map[string]*bintree{}}, + "noop_tracer.js": {noop_tracerJs, map[string]*bintree{}}, + "opcount_tracer.js": {opcount_tracerJs, map[string]*bintree{}}, + "prestate_tracer.js": {prestate_tracerJs, map[string]*bintree{}}, + "trigram_tracer.js": {trigram_tracerJs, map[string]*bintree{}}, + "unigram_tracer.js": {unigram_tracerJs, map[string]*bintree{}}, }} // RestoreAsset restores an asset under the given directory. diff --git a/eth/tracers/internal/tracers/call_tracer.js b/eth/tracers/internal/tracers/call_tracer.js index 3ca737773..98cfa0e6d 100644 --- a/eth/tracers/internal/tracers/call_tracer.js +++ b/eth/tracers/internal/tracers/call_tracer.js @@ -1,4 +1,4 @@ -// Copyright 2017 The go-ethereum Authors +// Copyright 2021 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify @@ -14,212 +14,81 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -// callTracer is a full blown transaction tracer that extracts and reports all -// the internal calls made by a transaction, along with any useful information. + +// callFrameTracer uses the new call frame tracing methods to report useful information +// about internal messages of a transaction. { - // callstack is the current recursive call stack of the EVM execution. - callstack: [{}], - - // descended tracks whether we've just descended from an outer transaction into - // an inner call. - descended: false, - - // step is invoked for every opcode that the VM executes. - step: function(log, db) { - // Capture any errors immediately - var error = log.getError(); - if (error !== undefined) { - this.fault(log, db); - return; - } - // We only care about system opcodes, faster if we pre-check once - var syscall = (log.op.toNumber() & 0xf0) == 0xf0; - if (syscall) { - var op = log.op.toString(); - } - // If a new contract is being created, add to the call stack - if (syscall && (op == 'CREATE' || op == "CREATE2")) { - var inOff = log.stack.peek(1).valueOf(); - var inEnd = inOff + log.stack.peek(2).valueOf(); - - // Assemble the internal call report and store for completion - var call = { - type: op, - from: toHex(log.contract.getAddress()), - input: toHex(log.memory.slice(inOff, inEnd)), - gasIn: log.getGas(), - gasCost: log.getCost(), - value: '0x' + log.stack.peek(0).toString(16) - }; - this.callstack.push(call); - this.descended = true - return; - } - // If a contract is being self destructed, gather that as a subcall too - if (syscall && op == 'SELFDESTRUCT') { - var left = this.callstack.length; - if (this.callstack[left-1].calls === undefined) { - this.callstack[left-1].calls = []; - } - this.callstack[left-1].calls.push({ - type: op, - from: toHex(log.contract.getAddress()), - to: toHex(toAddress(log.stack.peek(0).toString(16))), - gasIn: log.getGas(), - gasCost: log.getCost(), - value: '0x' + db.getBalance(log.contract.getAddress()).toString(16) - }); - return - } - // If a new method invocation is being done, add to the call stack - if (syscall && (op == 'CALL' || op == 'CALLCODE' || op == 'DELEGATECALL' || op == 'STATICCALL')) { - // Skip any pre-compile invocations, those are just fancy opcodes - var to = toAddress(log.stack.peek(1).toString(16)); - if (isPrecompiled(to)) { - return - } - var off = (op == 'DELEGATECALL' || op == 'STATICCALL' ? 0 : 1); - - var inOff = log.stack.peek(2 + off).valueOf(); - var inEnd = inOff + log.stack.peek(3 + off).valueOf(); - - // Assemble the internal call report and store for completion - var call = { - type: op, - from: toHex(log.contract.getAddress()), - to: toHex(to), - input: toHex(log.memory.slice(inOff, inEnd)), - gasIn: log.getGas(), - gasCost: log.getCost(), - outOff: log.stack.peek(4 + off).valueOf(), - outLen: log.stack.peek(5 + off).valueOf() - }; - if (op != 'DELEGATECALL' && op != 'STATICCALL') { - call.value = '0x' + log.stack.peek(2).toString(16); - } - this.callstack.push(call); - this.descended = true - return; - } - // If we've just descended into an inner call, retrieve it's true allowance. We - // need to extract if from within the call as there may be funky gas dynamics - // with regard to requested and actually given gas (2300 stipend, 63/64 rule). - if (this.descended) { - if (log.getDepth() >= this.callstack.length) { - this.callstack[this.callstack.length - 1].gas = log.getGas(); - } else { - // TODO(karalabe): The call was made to a plain account. We currently don't - // have access to the true gas amount inside the call and so any amount will - // mostly be wrong since it depends on a lot of input args. Skip gas for now. - } - this.descended = false; - } - // If an existing call is returning, pop off the call stack - if (syscall && op == 'REVERT') { - this.callstack[this.callstack.length - 1].error = "execution reverted"; - return; - } - if (log.getDepth() == this.callstack.length - 1) { - // Pop off the last call and get the execution results - var call = this.callstack.pop(); - - if (call.type == 'CREATE' || call.type == "CREATE2") { - // If the call was a CREATE, retrieve the contract address and output code - call.gasUsed = '0x' + bigInt(call.gasIn - call.gasCost - log.getGas()).toString(16); - delete call.gasIn; delete call.gasCost; - - var ret = log.stack.peek(0); - if (!ret.equals(0)) { - call.to = toHex(toAddress(ret.toString(16))); - call.output = toHex(db.getCode(toAddress(ret.toString(16)))); - } else if (call.error === undefined) { - call.error = "internal failure"; // TODO(karalabe): surface these faults somehow - } - } else { - // If the call was a contract call, retrieve the gas usage and output - if (call.gas !== undefined) { - call.gasUsed = '0x' + bigInt(call.gasIn - call.gasCost + call.gas - log.getGas()).toString(16); - } - var ret = log.stack.peek(0); - if (!ret.equals(0)) { - call.output = toHex(log.memory.slice(call.outOff, call.outOff + call.outLen)); - } else if (call.error === undefined) { - call.error = "internal failure"; // TODO(karalabe): surface these faults somehow - } - delete call.gasIn; delete call.gasCost; - delete call.outOff; delete call.outLen; - } - if (call.gas !== undefined) { - call.gas = '0x' + bigInt(call.gas).toString(16); - } - // Inject the call into the previous one - var left = this.callstack.length; - if (this.callstack[left-1].calls === undefined) { - this.callstack[left-1].calls = []; - } - this.callstack[left-1].calls.push(call); - } - }, - - // fault is invoked when the actual execution of an opcode fails. - fault: function(log, db) { - // If the topmost call already reverted, don't handle the additional fault again - if (this.callstack[this.callstack.length - 1].error !== undefined) { - return; - } - // Pop off the just failed call - var call = this.callstack.pop(); - call.error = log.getError(); - - // Consume all available gas and clean any leftovers - if (call.gas !== undefined) { - call.gas = '0x' + bigInt(call.gas).toString(16); - call.gasUsed = call.gas - } - delete call.gasIn; delete call.gasCost; - delete call.outOff; delete call.outLen; - - // Flatten the failed call into its parent - var left = this.callstack.length; - if (left > 0) { - if (this.callstack[left-1].calls === undefined) { - this.callstack[left-1].calls = []; - } - this.callstack[left-1].calls.push(call); - return; - } - // Last call failed too, leave it in the stack - this.callstack.push(call); - }, - - // result is invoked when all the opcodes have been iterated over and returns - // the final result of the tracing. - result: function(ctx, db) { - var result = { - type: ctx.type, - from: toHex(ctx.from), - to: toHex(ctx.to), - value: '0x' + ctx.value.toString(16), - gas: '0x' + bigInt(ctx.gas).toString(16), - gasUsed: '0x' + bigInt(ctx.gasUsed).toString(16), - input: toHex(ctx.input), - output: toHex(ctx.output), - time: ctx.time, - }; - if (this.callstack[0].calls !== undefined) { - result.calls = this.callstack[0].calls; - } - if (this.callstack[0].error !== undefined) { - result.error = this.callstack[0].error; - } else if (ctx.error !== undefined) { - result.error = ctx.error; - } - if (result.error !== undefined && (result.error !== "execution reverted" || result.output ==="0x")) { - delete result.output; - } - return this.finalize(result); - }, + callstack: [{}], + fault: function(log, db) { + var len = this.callstack.length + if (len > 1) { + var call = this.callstack.pop() + if (this.callstack[len-1].calls === undefined) { + this.callstack[len-1].calls = [] + } + this.callstack[len-1].calls.push(call) + } + }, + result: function(ctx, db) { + // Prepare outer message info + var result = { + type: ctx.type, + from: toHex(ctx.from), + to: toHex(ctx.to), + value: '0x' + ctx.value.toString(16), + gas: '0x' + bigInt(ctx.gas).toString(16), + gasUsed: '0x' + bigInt(ctx.gasUsed).toString(16), + input: toHex(ctx.input), + output: toHex(ctx.output), + } + if (this.callstack[0].calls !== undefined) { + result.calls = this.callstack[0].calls + } + if (this.callstack[0].error !== undefined) { + result.error = this.callstack[0].error + } else if (ctx.error !== undefined) { + result.error = ctx.error + } + if (result.error !== undefined && (result.error !== "execution reverted" || result.output ==="0x")) { + delete result.output + } + return this.finalize(result) + }, + enter: function(frame) { + var call = { + type: frame.getType(), + from: toHex(frame.getFrom()), + to: toHex(frame.getTo()), + input: toHex(frame.getInput()), + gas: '0x' + bigInt(frame.getGas()).toString('16'), + } + if (frame.getValue() !== undefined){ + call.value='0x' + bigInt(frame.getValue()).toString(16) + } + this.callstack.push(call) + }, + exit: function(frameResult) { + var len = this.callstack.length + if (len > 1) { + var call = this.callstack.pop() + call.gasUsed = '0x' + bigInt(frameResult.getGasUsed()).toString('16') + var error = frameResult.getError() + if (error === undefined) { + call.output = toHex(frameResult.getOutput()) + } else { + call.error = error + if (call.type === 'CREATE' || call.type === 'CREATE2') { + delete call.to + } + } + len -= 1 + if (this.callstack[len-1].calls === undefined) { + this.callstack[len-1].calls = [] + } + this.callstack[len-1].calls.push(call) + } + }, // finalize recreates a call object using the final desired field oder for json // serialization. This is a nicety feature to pass meaningfully ordered results // to users who don't interpret it, just display it. @@ -239,14 +108,14 @@ } for (var key in sorted) { if (sorted[key] === undefined) { - delete sorted[key]; + delete sorted[key] } } if (sorted.calls !== undefined) { for (var i=0; i. + +// callTracer is a full blown transaction tracer that extracts and reports all +// the internal calls made by a transaction, along with any useful information. +{ + // callstack is the current recursive call stack of the EVM execution. + callstack: [{}], + + // descended tracks whether we've just descended from an outer transaction into + // an inner call. + descended: false, + + // step is invoked for every opcode that the VM executes. + step: function(log, db) { + // Capture any errors immediately + var error = log.getError(); + if (error !== undefined) { + this.fault(log, db); + return; + } + // We only care about system opcodes, faster if we pre-check once + var syscall = (log.op.toNumber() & 0xf0) == 0xf0; + if (syscall) { + var op = log.op.toString(); + } + // If a new contract is being created, add to the call stack + if (syscall && (op == 'CREATE' || op == "CREATE2")) { + var inOff = log.stack.peek(1).valueOf(); + var inEnd = inOff + log.stack.peek(2).valueOf(); + + // Assemble the internal call report and store for completion + var call = { + type: op, + from: toHex(log.contract.getAddress()), + input: toHex(log.memory.slice(inOff, inEnd)), + gasIn: log.getGas(), + gasCost: log.getCost(), + value: '0x' + log.stack.peek(0).toString(16) + }; + this.callstack.push(call); + this.descended = true + return; + } + // If a contract is being self destructed, gather that as a subcall too + if (syscall && op == 'SELFDESTRUCT') { + var left = this.callstack.length; + if (this.callstack[left-1].calls === undefined) { + this.callstack[left-1].calls = []; + } + this.callstack[left-1].calls.push({ + type: op, + from: toHex(log.contract.getAddress()), + to: toHex(toAddress(log.stack.peek(0).toString(16))), + gasIn: log.getGas(), + gasCost: log.getCost(), + value: '0x' + db.getBalance(log.contract.getAddress()).toString(16) + }); + return + } + // If a new method invocation is being done, add to the call stack + if (syscall && (op == 'CALL' || op == 'CALLCODE' || op == 'DELEGATECALL' || op == 'STATICCALL')) { + // Skip any pre-compile invocations, those are just fancy opcodes + var to = toAddress(log.stack.peek(1).toString(16)); + if (isPrecompiled(to)) { + return + } + var off = (op == 'DELEGATECALL' || op == 'STATICCALL' ? 0 : 1); + + var inOff = log.stack.peek(2 + off).valueOf(); + var inEnd = inOff + log.stack.peek(3 + off).valueOf(); + + // Assemble the internal call report and store for completion + var call = { + type: op, + from: toHex(log.contract.getAddress()), + to: toHex(to), + input: toHex(log.memory.slice(inOff, inEnd)), + gasIn: log.getGas(), + gasCost: log.getCost(), + outOff: log.stack.peek(4 + off).valueOf(), + outLen: log.stack.peek(5 + off).valueOf() + }; + if (op != 'DELEGATECALL' && op != 'STATICCALL') { + call.value = '0x' + log.stack.peek(2).toString(16); + } + this.callstack.push(call); + this.descended = true + return; + } + // If we've just descended into an inner call, retrieve it's true allowance. We + // need to extract if from within the call as there may be funky gas dynamics + // with regard to requested and actually given gas (2300 stipend, 63/64 rule). + if (this.descended) { + if (log.getDepth() >= this.callstack.length) { + this.callstack[this.callstack.length - 1].gas = log.getGas(); + } else { + // TODO(karalabe): The call was made to a plain account. We currently don't + // have access to the true gas amount inside the call and so any amount will + // mostly be wrong since it depends on a lot of input args. Skip gas for now. + } + this.descended = false; + } + // If an existing call is returning, pop off the call stack + if (syscall && op == 'REVERT') { + this.callstack[this.callstack.length - 1].error = "execution reverted"; + return; + } + if (log.getDepth() == this.callstack.length - 1) { + // Pop off the last call and get the execution results + var call = this.callstack.pop(); + + if (call.type == 'CREATE' || call.type == "CREATE2") { + // If the call was a CREATE, retrieve the contract address and output code + call.gasUsed = '0x' + bigInt(call.gasIn - call.gasCost - log.getGas()).toString(16); + delete call.gasIn; delete call.gasCost; + + var ret = log.stack.peek(0); + if (!ret.equals(0)) { + call.to = toHex(toAddress(ret.toString(16))); + call.output = toHex(db.getCode(toAddress(ret.toString(16)))); + } else if (call.error === undefined) { + call.error = "internal failure"; // TODO(karalabe): surface these faults somehow + } + } else { + // If the call was a contract call, retrieve the gas usage and output + if (call.gas !== undefined) { + call.gasUsed = '0x' + bigInt(call.gasIn - call.gasCost + call.gas - log.getGas()).toString(16); + } + var ret = log.stack.peek(0); + if (!ret.equals(0)) { + call.output = toHex(log.memory.slice(call.outOff, call.outOff + call.outLen)); + } else if (call.error === undefined) { + call.error = "internal failure"; // TODO(karalabe): surface these faults somehow + } + delete call.gasIn; delete call.gasCost; + delete call.outOff; delete call.outLen; + } + if (call.gas !== undefined) { + call.gas = '0x' + bigInt(call.gas).toString(16); + } + // Inject the call into the previous one + var left = this.callstack.length; + if (this.callstack[left-1].calls === undefined) { + this.callstack[left-1].calls = []; + } + this.callstack[left-1].calls.push(call); + } + }, + + // fault is invoked when the actual execution of an opcode fails. + fault: function(log, db) { + // If the topmost call already reverted, don't handle the additional fault again + if (this.callstack[this.callstack.length - 1].error !== undefined) { + return; + } + // Pop off the just failed call + var call = this.callstack.pop(); + call.error = log.getError(); + + // Consume all available gas and clean any leftovers + if (call.gas !== undefined) { + call.gas = '0x' + bigInt(call.gas).toString(16); + call.gasUsed = call.gas + } + delete call.gasIn; delete call.gasCost; + delete call.outOff; delete call.outLen; + + // Flatten the failed call into its parent + var left = this.callstack.length; + if (left > 0) { + if (this.callstack[left-1].calls === undefined) { + this.callstack[left-1].calls = []; + } + this.callstack[left-1].calls.push(call); + return; + } + // Last call failed too, leave it in the stack + this.callstack.push(call); + }, + + // result is invoked when all the opcodes have been iterated over and returns + // the final result of the tracing. + result: function(ctx, db) { + var result = { + type: ctx.type, + from: toHex(ctx.from), + to: toHex(ctx.to), + value: '0x' + ctx.value.toString(16), + gas: '0x' + bigInt(ctx.gas).toString(16), + gasUsed: '0x' + bigInt(ctx.gasUsed).toString(16), + input: toHex(ctx.input), + output: toHex(ctx.output), + time: ctx.time, + }; + if (this.callstack[0].calls !== undefined) { + result.calls = this.callstack[0].calls; + } + if (this.callstack[0].error !== undefined) { + result.error = this.callstack[0].error; + } else if (ctx.error !== undefined) { + result.error = ctx.error; + } + if (result.error !== undefined && (result.error !== "execution reverted" || result.output ==="0x")) { + delete result.output; + } + return this.finalize(result); + }, + + // finalize recreates a call object using the final desired field oder for json + // serialization. This is a nicety feature to pass meaningfully ordered results + // to users who don't interpret it, just display it. + finalize: function(call) { + var sorted = { + type: call.type, + from: call.from, + to: call.to, + value: call.value, + gas: call.gas, + gasUsed: call.gasUsed, + input: call.input, + output: call.output, + error: call.error, + time: call.time, + calls: call.calls, + } + for (var key in sorted) { + if (sorted[key] === undefined) { + delete sorted[key]; + } + } + if (sorted.calls !== undefined) { + for (var i=0; i 0 { + jst.err = jst.reason + return + } + + *jst.frame.typ = typ.String() + *jst.frame.from = from + *jst.frame.to = to + jst.frame.input = common.CopyBytes(input) + *jst.frame.gas = uint(gas) + jst.frame.value = nil + if value != nil { + jst.frame.value = new(big.Int).SetBytes(value.Bytes()) + } + + if _, err := jst.call(true, "enter", "frame"); err != nil { + jst.err = wrapError("enter", err) + } +} + +// CaptureExit is called when EVM exits a scope, even if the scope didn't +// execute any code. +func (jst *Tracer) CaptureExit(output []byte, gasUsed uint64, err error) { + if !jst.traceCallFrames { + return + } + if jst.err != nil { + return + } + // If tracing was interrupted, set the error and stop + if atomic.LoadUint32(&jst.interrupt) > 0 { + jst.err = jst.reason + return + } + + jst.frameResult.output = common.CopyBytes(output) + *jst.frameResult.gasUsed = uint(gasUsed) + jst.frameResult.errorValue = nil + if err != nil { + jst.frameResult.errorValue = new(string) + *jst.frameResult.errorValue = err.Error() + } + + if _, err := jst.call(true, "exit", "frameResult"); err != nil { + jst.err = wrapError("exit", err) + } +} + // GetResult calls the Javascript 'result' function and returns its value, or any accumulated error func (jst *Tracer) GetResult() (json.RawMessage, error) { // Transform the context into a JavaScript object and inject into the state obj := jst.vm.PushObject() for key, val := range jst.ctx { - switch val := val.(type) { - case uint64: - jst.vm.PushUint(uint(val)) - - case string: - jst.vm.PushString(val) - - case []byte: - ptr := jst.vm.PushFixedBuffer(len(val)) - copy(makeSlice(ptr, uint(len(val))), val) - - case common.Address: - ptr := jst.vm.PushFixedBuffer(20) - copy(makeSlice(ptr, 20), val[:]) - - case *big.Int: - pushBigInt(val, jst.vm) - - case int: - jst.vm.PushInt(val) - - case common.Hash: - ptr := jst.vm.PushFixedBuffer(32) - copy(makeSlice(ptr, 32), val[:]) - - default: - panic(fmt.Sprintf("unsupported type: %T", val)) - } - jst.vm.PutPropString(obj, key) + jst.addToObj(obj, key, val) } jst.vm.PutPropString(jst.stateObject, "ctx") @@ -699,3 +838,35 @@ func (jst *Tracer) GetResult() (json.RawMessage, error) { return result, jst.err } + +// addToObj pushes a field to a JS object. +func (jst *Tracer) addToObj(obj int, key string, val interface{}) { + pushValue(jst.vm, val) + jst.vm.PutPropString(obj, key) +} + +func pushValue(ctx *duktape.Context, val interface{}) { + switch val := val.(type) { + case uint64: + ctx.PushUint(uint(val)) + case string: + ctx.PushString(val) + case []byte: + ptr := ctx.PushFixedBuffer(len(val)) + copy(makeSlice(ptr, uint(len(val))), val) + case common.Address: + ptr := ctx.PushFixedBuffer(20) + copy(makeSlice(ptr, 20), val[:]) + case *big.Int: + pushBigInt(val, ctx) + case int: + ctx.PushInt(val) + case uint: + ctx.PushUint(val) + case common.Hash: + ptr := ctx.PushFixedBuffer(32) + copy(makeSlice(ptr, 32), val[:]) + default: + panic(fmt.Sprintf("unsupported type: %T", val)) + } +} diff --git a/eth/tracers/tracer_test.go b/eth/tracers/tracer_test.go index d2d3e57c4..3decca225 100644 --- a/eth/tracers/tracer_test.go +++ b/eth/tracers/tracer_test.go @@ -236,3 +236,35 @@ func TestIsPrecompile(t *testing.T) { t.Errorf("Tracer should consider blake2f as precompile in istanbul") } } + +func TestEnterExit(t *testing.T) { + // test that either both or none of enter() and exit() are defined + if _, err := New("{step: function() {}, fault: function() {}, result: function() { return null; }, enter: function() {}}", new(Context)); err == nil { + t.Fatal("tracer creation should've failed without exit() definition") + } + if _, err := New("{step: function() {}, fault: function() {}, result: function() { return null; }, enter: function() {}, exit: function() {}}", new(Context)); err != nil { + t.Fatal(err) + } + + // test that the enter and exit method are correctly invoked and the values passed + tracer, err := New("{enters: 0, exits: 0, enterGas: 0, gasUsed: 0, step: function() {}, fault: function() {}, result: function() { return {enters: this.enters, exits: this.exits, enterGas: this.enterGas, gasUsed: this.gasUsed} }, enter: function(frame) { this.enters++; this.enterGas = frame.getGas(); }, exit: function(res) { this.exits++; this.gasUsed = res.getGasUsed(); }}", new(Context)) + if err != nil { + t.Fatal(err) + } + + scope := &vm.ScopeContext{ + Contract: vm.NewContract(&account{}, &account{}, big.NewInt(0), 0), + } + + tracer.CaptureEnter(vm.CALL, scope.Contract.Caller(), scope.Contract.Address(), []byte{}, 1000, new(big.Int)) + tracer.CaptureExit([]byte{}, 400, nil) + + have, err := tracer.GetResult() + if err != nil { + t.Fatal(err) + } + want := `{"enters":1,"exits":1,"enterGas":1000,"gasUsed":400}` + if string(have) != want { + t.Errorf("Number of invocations of enter() and exit() is wrong. Have %s, want %s\n", have, want) + } +} diff --git a/eth/tracers/tracers_test.go b/eth/tracers/tracers_test.go index e738aa3a3..fb817fbc5 100644 --- a/eth/tracers/tracers_test.go +++ b/eth/tracers/tracers_test.go @@ -203,21 +203,25 @@ func TestPrestateTracerCreate2(t *testing.T) { // Iterates over all the input-output datasets in the tracer test harness and // runs the JavaScript tracers against them. -func TestCallTracer(t *testing.T) { - files, err := ioutil.ReadDir("testdata") +func TestCallTracerLegacy(t *testing.T) { + testCallTracer("callTracerLegacy", "call_tracer_legacy", t) +} + +func testCallTracer(tracer string, dirPath string, t *testing.T) { + files, err := ioutil.ReadDir(filepath.Join("testdata", dirPath)) if err != nil { t.Fatalf("failed to retrieve tracer test suite: %v", err) } for _, file := range files { - if !strings.HasPrefix(file.Name(), "call_tracer_") { + if !strings.HasSuffix(file.Name(), ".json") { continue } file := file // capture range variable - t.Run(camel(strings.TrimSuffix(strings.TrimPrefix(file.Name(), "call_tracer_"), ".json")), func(t *testing.T) { + t.Run(camel(strings.TrimSuffix(file.Name(), ".json")), func(t *testing.T) { t.Parallel() // Call tracer test found, read if from disk - blob, err := ioutil.ReadFile(filepath.Join("testdata", file.Name())) + blob, err := ioutil.ReadFile(filepath.Join("testdata", dirPath, file.Name())) if err != nil { t.Fatalf("failed to read testcase: %v", err) } @@ -248,7 +252,7 @@ func TestCallTracer(t *testing.T) { _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false) // Create the tracer, the EVM environment and run it - tracer, err := New("callTracer", new(Context)) + tracer, err := New(tracer, new(Context)) if err != nil { t.Fatalf("failed to create call tracer: %v", err) } @@ -283,6 +287,10 @@ func TestCallTracer(t *testing.T) { } } +func TestCallTracer(t *testing.T) { + testCallTracer("callTracer", "call_tracer", t) +} + // jsonEqual is similar to reflect.DeepEqual, but does a 'bounce' via json prior to // comparison func jsonEqual(x, y interface{}) bool { @@ -378,3 +386,73 @@ func BenchmarkTransactionTrace(b *testing.B) { tracer.Reset() } } + +func BenchmarkTracers(b *testing.B) { + files, err := ioutil.ReadDir(filepath.Join("testdata", "call_tracer")) + if err != nil { + b.Fatalf("failed to retrieve tracer test suite: %v", err) + } + for _, file := range files { + if !strings.HasSuffix(file.Name(), ".json") { + continue + } + file := file // capture range variable + b.Run(camel(strings.TrimSuffix(file.Name(), ".json")), func(b *testing.B) { + blob, err := ioutil.ReadFile(filepath.Join("testdata", "call_tracer", file.Name())) + if err != nil { + b.Fatalf("failed to read testcase: %v", err) + } + test := new(callTracerTest) + if err := json.Unmarshal(blob, test); err != nil { + b.Fatalf("failed to parse testcase: %v", err) + } + benchTracer("callTracer", test, b) + }) + } +} + +func benchTracer(tracerName string, test *callTracerTest, b *testing.B) { + // Configure a blockchain with the given prestate + tx := new(types.Transaction) + if err := rlp.DecodeBytes(common.FromHex(test.Input), tx); err != nil { + b.Fatalf("failed to parse testcase input: %v", err) + } + signer := types.MakeSigner(test.Genesis.Config, new(big.Int).SetUint64(uint64(test.Context.Number))) + msg, err := tx.AsMessage(signer, nil) + if err != nil { + b.Fatalf("failed to prepare transaction for tracing: %v", err) + } + origin, _ := signer.Sender(tx) + txContext := vm.TxContext{ + Origin: origin, + GasPrice: tx.GasPrice(), + } + context := vm.BlockContext{ + CanTransfer: core.CanTransfer, + Transfer: core.Transfer, + Coinbase: test.Context.Miner, + BlockNumber: new(big.Int).SetUint64(uint64(test.Context.Number)), + Time: new(big.Int).SetUint64(uint64(test.Context.Time)), + Difficulty: (*big.Int)(test.Context.Difficulty), + GasLimit: uint64(test.Context.GasLimit), + } + _, statedb := tests.MakePreState(rawdb.NewMemoryDatabase(), test.Genesis.Alloc, false) + + // Create the tracer, the EVM environment and run it + tracer, err := New(tracerName, new(Context)) + if err != nil { + b.Fatalf("failed to create call tracer: %v", err) + } + evm := vm.NewEVM(context, txContext, statedb, test.Genesis.Config, vm.Config{Debug: true, Tracer: tracer}) + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + snap := statedb.Snapshot() + st := core.NewStateTransition(evm, msg, new(core.GasPool).AddGas(tx.Gas())) + if _, err = st.TransitionDb(); err != nil { + b.Fatalf("failed to execute transaction: %v", err) + } + statedb.RevertToSnapshot(snap) + } +} From 1b34283810d22421724eba7bbda4074f2f4720c5 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Mon, 20 Sep 2021 10:23:44 +0200 Subject: [PATCH 53/73] ethclient, accounts/keystore: fix flaky tests (#23599) * ethclient/gethclient: fix flaky test (due to map key ordering) * accounts/keystore: fix test failing due to rand collision due to low time resolution on windows --- accounts/keystore/account_cache_test.go | 4 +-- ethclient/gethclient/gethclient_test.go | 37 +++++++++++++------------ 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/accounts/keystore/account_cache_test.go b/accounts/keystore/account_cache_test.go index fe9233c04..a847545bc 100644 --- a/accounts/keystore/account_cache_test.go +++ b/accounts/keystore/account_cache_test.go @@ -96,7 +96,7 @@ func TestWatchNoDir(t *testing.T) { // Create ks but not the directory that it watches. rand.Seed(time.Now().UnixNano()) - dir := filepath.Join(os.TempDir(), fmt.Sprintf("eth-keystore-watch-test-%d-%d", os.Getpid(), rand.Int())) + dir := filepath.Join(os.TempDir(), fmt.Sprintf("eth-keystore-watchnodir-test-%d-%d", os.Getpid(), rand.Int())) ks := NewKeyStore(dir, LightScryptN, LightScryptP) list := ks.Accounts() @@ -322,7 +322,7 @@ func TestUpdatedKeyfileContents(t *testing.T) { // Create a temporary kesytore to test with rand.Seed(time.Now().UnixNano()) - dir := filepath.Join(os.TempDir(), fmt.Sprintf("eth-keystore-watch-test-%d-%d", os.Getpid(), rand.Int())) + dir := filepath.Join(os.TempDir(), fmt.Sprintf("eth-keystore-updatedkeyfilecontents-test-%d-%d", os.Getpid(), rand.Int())) ks := NewKeyStore(dir, LightScryptN, LightScryptP) list := ks.Accounts() diff --git a/ethclient/gethclient/gethclient_test.go b/ethclient/gethclient/gethclient_test.go index c0b9d0c2d..ca2cb1abd 100644 --- a/ethclient/gethclient/gethclient_test.go +++ b/ethclient/gethclient/gethclient_test.go @@ -97,37 +97,40 @@ func TestGethClient(t *testing.T) { defer backend.Close() defer client.Close() - tests := map[string]struct { + tests := []struct { + name string test func(t *testing.T) }{ - "TestAccessList": { + { + "TestAccessList", func(t *testing.T) { testAccessList(t, client) }, }, - "TestGetProof": { + { + "TestGetProof", func(t *testing.T) { testGetProof(t, client) }, - }, - "TestGCStats": { + }, { + "TestGCStats", func(t *testing.T) { testGCStats(t, client) }, - }, - "TestMemStats": { + }, { + "TestMemStats", func(t *testing.T) { testMemStats(t, client) }, - }, - "TestGetNodeInfo": { + }, { + "TestGetNodeInfo", func(t *testing.T) { testGetNodeInfo(t, client) }, - }, - "TestSetHead": { + }, { + "TestSetHead", func(t *testing.T) { testSetHead(t, client) }, - }, - "TestSubscribePendingTxs": { + }, { + "TestSubscribePendingTxs", func(t *testing.T) { testSubscribePendingTransactions(t, client) }, - }, - "TestCallContract": { + }, { + "TestCallContract", func(t *testing.T) { testCallContract(t, client) }, }, } t.Parallel() - for name, tt := range tests { - t.Run(name, tt.test) + for _, tt := range tests { + t.Run(tt.name, tt.test) } } From 62e3b83af69afdee8ce2f242b64efbd98bf44db2 Mon Sep 17 00:00:00 2001 From: Pete <90942436+peteyburgers@users.noreply.github.com> Date: Mon, 20 Sep 2021 21:33:15 +1000 Subject: [PATCH 54/73] docs: fix spelling on 2021-08-22 postmortem (#23595) Spelling update --- docs/postmortems/2021-08-22-split-postmortem.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/postmortems/2021-08-22-split-postmortem.md b/docs/postmortems/2021-08-22-split-postmortem.md index 888f7d77c..429f22d70 100644 --- a/docs/postmortems/2021-08-22-split-postmortem.md +++ b/docs/postmortems/2021-08-22-split-postmortem.md @@ -16,7 +16,7 @@ This is a post-mortem concerning the minority split that occurred on Ethereum ma ### 2021-08-17 RETURNDATA corruption via datacopy -On 2021-08-17, Guido Vranken submitted a report to bounty@ethereum.org. This co-incided with a geth-meetup in Berlin, so the geth team could fairly quickly analyse the issue. +On 2021-08-17, Guido Vranken submitted a report to bounty@ethereum.org. This coincided with a geth-meetup in Berlin, so the geth team could fairly quickly analyse the issue. He submitted a proof of concept which called the `dataCopy` precompile, where the input slice and output slice were overlapping but shifted. Doing a `copy` where the `src` and `dest` overlaps is not a problem in itself, however, the `returnData`slice was _also_ using the same memory as a backing-array. From e28f713ada67af2380bd7052cb85b2e255a62a6a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 20 Sep 2021 16:29:07 +0300 Subject: [PATCH 55/73] internal: support optional filter expression for debug.stacks (#23605) * internal: support optional filter expression for debug.stacks * internal/debug: fix string regexp * internal/debug: support searching for line numbers too --- go.mod | 1 + go.sum | 6 ++++++ internal/debug/api.go | 40 +++++++++++++++++++++++++++++++++++-- internal/web3ext/web3ext.go | 3 ++- 4 files changed, 47 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index ce19afc85..b63368085 100644 --- a/go.mod +++ b/go.mod @@ -35,6 +35,7 @@ require ( github.com/google/uuid v1.1.5 github.com/gorilla/websocket v1.4.2 github.com/graph-gophers/graphql-go v0.0.0-20201113091052-beb923fada29 + github.com/hashicorp/go-bexpr v0.1.10 // indirect github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d github.com/holiman/bloomfilter/v2 v2.0.3 github.com/holiman/uint256 v1.2.0 diff --git a/go.sum b/go.sum index 3d8a04236..0890b8c5e 100644 --- a/go.sum +++ b/go.sum @@ -216,6 +216,8 @@ github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0U github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/graph-gophers/graphql-go v0.0.0-20201113091052-beb923fada29 h1:sezaKhEfPFg8W0Enm61B9Gs911H8iesGY5R8NDPtd1M= github.com/graph-gophers/graphql-go v0.0.0-20201113091052-beb923fada29/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= +github.com/hashicorp/go-bexpr v0.1.10 h1:9kuI5PFotCboP3dkDYFr/wi0gg0QVbSNz5oFRpxn4uE= +github.com/hashicorp/go-bexpr v0.1.10/go.mod h1:oxlubA2vC/gFVfX1A6JGp7ls7uCDlfJn732ehYYg+g0= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d h1:dg1dEPuWpEqDnvIw251EVy4zlP8gWbsGj4BsUKCRpYs= @@ -307,6 +309,10 @@ github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mitchellh/mapstructure v1.4.1 h1:CpVNEelQCZBooIPDn+AR3NpivK/TIKU8bDxdASFVQag= +github.com/mitchellh/mapstructure v1.4.1/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/mitchellh/pointerstructure v1.2.0 h1:O+i9nHnXS3l/9Wu7r4NrEdwA2VFTicjUEN1uBnDo34A= +github.com/mitchellh/pointerstructure v1.2.0/go.mod h1:BRAsLI5zgXmw97Lf6s25bs8ohIXc3tViBH44KcwB2g4= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= diff --git a/internal/debug/api.go b/internal/debug/api.go index efd862677..b1f815085 100644 --- a/internal/debug/api.go +++ b/internal/debug/api.go @@ -27,6 +27,7 @@ import ( "os" "os/user" "path/filepath" + "regexp" "runtime" "runtime/debug" "runtime/pprof" @@ -35,6 +36,7 @@ import ( "time" "github.com/ethereum/go-ethereum/log" + "github.com/hashicorp/go-bexpr" ) // Handler is the global debugging handler. @@ -189,10 +191,44 @@ func (*HandlerT) WriteMemProfile(file string) error { return writeProfile("heap", file) } -// Stacks returns a printed representation of the stacks of all goroutines. -func (*HandlerT) Stacks() string { +// Stacks returns a printed representation of the stacks of all goroutines. It +// also permits the following optional filters to be used: +// - filter: boolean expression of packages to filter for +func (*HandlerT) Stacks(filter *string) string { buf := new(bytes.Buffer) pprof.Lookup("goroutine").WriteTo(buf, 2) + + // If any filtering was requested, execute them now + if filter != nil && len(*filter) > 0 { + expanded := *filter + + // The input filter is a logical expression of package names. Transform + // it into a proper boolean expression that can be fed into a parser and + // interpreter: + // + // E.g. (eth || snap) && !p2p -> (eth in Value || snap in Value) && p2p not in Value + expanded = regexp.MustCompile("[:/\\.A-Za-z0-9_-]+").ReplaceAllString(expanded, "`$0` in Value") + expanded = regexp.MustCompile("!(`[:/\\.A-Za-z0-9_-]+`)").ReplaceAllString(expanded, "$1 not") + expanded = strings.Replace(expanded, "||", "or", -1) + expanded = strings.Replace(expanded, "&&", "and", -1) + log.Info("Expanded filter expression", "filter", *filter, "expanded", expanded) + + expr, err := bexpr.CreateEvaluator(expanded) + if err != nil { + log.Error("Failed to parse filter expression", "expanded", expanded, "err", err) + return "" + } + // Split the goroutine dump into segments and filter each + dump := buf.String() + buf.Reset() + + for _, trace := range strings.Split(dump, "\n\n") { + if ok, _ := expr.Evaluate(map[string]string{"Value": trace}); ok { + buf.WriteString(trace) + buf.WriteString("\n\n") + } + } + } return buf.String() } diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go index 927dba189..fe15cb050 100644 --- a/internal/web3ext/web3ext.go +++ b/internal/web3ext/web3ext.go @@ -278,7 +278,8 @@ web3._extend({ new web3._extend.Method({ name: 'stacks', call: 'debug_stacks', - params: 0, + params: 1, + inputFormatter: [null], outputFormatter: console.log }), new web3._extend.Method({ From 9bf495bfc996b6465f910f92f6279b0776a1589d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 20 Sep 2021 18:55:35 +0300 Subject: [PATCH 56/73] internal/debug: make gosimple linter happy --- internal/debug/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/debug/api.go b/internal/debug/api.go index b1f815085..1ea0c6377 100644 --- a/internal/debug/api.go +++ b/internal/debug/api.go @@ -207,7 +207,7 @@ func (*HandlerT) Stacks(filter *string) string { // interpreter: // // E.g. (eth || snap) && !p2p -> (eth in Value || snap in Value) && p2p not in Value - expanded = regexp.MustCompile("[:/\\.A-Za-z0-9_-]+").ReplaceAllString(expanded, "`$0` in Value") + expanded = regexp.MustCompile(`[:/\.A-Za-z0-9_-]+`).ReplaceAllString(expanded, "`$0` in Value") expanded = regexp.MustCompile("!(`[:/\\.A-Za-z0-9_-]+`)").ReplaceAllString(expanded, "$1 not") expanded = strings.Replace(expanded, "||", "or", -1) expanded = strings.Replace(expanded, "&&", "and", -1) From 5b246af54e3d549a22a34ddd09201924f2560c9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 20 Sep 2021 19:12:20 +0300 Subject: [PATCH 57/73] go.mod: fix module file to prevent go from updating it during build --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index b63368085..5f4330af3 100644 --- a/go.mod +++ b/go.mod @@ -35,7 +35,7 @@ require ( github.com/google/uuid v1.1.5 github.com/gorilla/websocket v1.4.2 github.com/graph-gophers/graphql-go v0.0.0-20201113091052-beb923fada29 - github.com/hashicorp/go-bexpr v0.1.10 // indirect + github.com/hashicorp/go-bexpr v0.1.10 github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d github.com/holiman/bloomfilter/v2 v2.0.3 github.com/holiman/uint256 v1.2.0 From d8211c7ec726fc38bc3db49b36d70633ffae4ce2 Mon Sep 17 00:00:00 2001 From: Thad Guidry Date: Tue, 21 Sep 2021 03:19:46 -0500 Subject: [PATCH 58/73] console: add note about typing exit to exit (#23602) * add explicit note about typing exit in console * Add note about typing exit as alternative --- cmd/geth/consolecmd_test.go | 4 ++-- console/console.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/geth/consolecmd_test.go b/cmd/geth/consolecmd_test.go index c3f41b187..845ede2f9 100644 --- a/cmd/geth/consolecmd_test.go +++ b/cmd/geth/consolecmd_test.go @@ -75,7 +75,7 @@ at block: 0 ({{niltime}}) datadir: {{.Datadir}} modules: {{apis}} -To exit, press ctrl-d +To exit, press ctrl-d or type exit > {{.InputLine "exit"}} `) geth.ExpectExit() @@ -149,7 +149,7 @@ at block: 0 ({{niltime}}){{if ipc}} datadir: {{datadir}}{{end}} modules: {{apis}} -To exit, press ctrl-d +To exit, press ctrl-d or type exit > {{.InputLine "exit" }} `) attach.ExpectExit() diff --git a/console/console.go b/console/console.go index ae9f28da0..dd39300d0 100644 --- a/console/console.go +++ b/console/console.go @@ -324,7 +324,7 @@ func (c *Console) Welcome() { sort.Strings(modules) message += " modules: " + strings.Join(modules, " ") + "\n" } - message += "\nTo exit, press ctrl-d" + message += "\nTo exit, press ctrl-d or type exit" fmt.Fprintln(c.printer, message) } From b61ef24ccef9b4d14f4f3c97298fa6d4c1b6b47d Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Tue, 21 Sep 2021 10:46:08 +0200 Subject: [PATCH 59/73] consensus/clique: prevent 0 len extradata from panicing (#23538) Closes #23522 Co-authored-by: Martin Holst Swende --- core/genesis.go | 5 ++++- core/genesis_test.go | 16 ++++++++++++++++ miner/worker_test.go | 6 ++++-- 3 files changed, 24 insertions(+), 3 deletions(-) diff --git a/core/genesis.go b/core/genesis.go index c1f226c34..38ace4920 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -310,7 +310,7 @@ func (g *Genesis) ToBlock(db ethdb.Database) *types.Block { func (g *Genesis) Commit(db ethdb.Database) (*types.Block, error) { block := g.ToBlock(db) if block.Number().Sign() != 0 { - return nil, fmt.Errorf("can't commit genesis block with number > 0") + return nil, errors.New("can't commit genesis block with number > 0") } config := g.Config if config == nil { @@ -319,6 +319,9 @@ func (g *Genesis) Commit(db ethdb.Database) (*types.Block, error) { if err := config.CheckConfigForkOrder(); err != nil { return nil, err } + if config.Clique != nil && len(block.Extra()) == 0 { + return nil, errors.New("can't start clique chain without signers") + } rawdb.WriteTd(db, block.Hash(), block.NumberU64(), g.Difficulty) rawdb.WriteBlock(db, block) rawdb.WriteReceipts(db, block.Hash(), block.NumberU64(), nil) diff --git a/core/genesis_test.go b/core/genesis_test.go index 52c4384ab..055be2796 100644 --- a/core/genesis_test.go +++ b/core/genesis_test.go @@ -39,6 +39,22 @@ func TestDefaultGenesisBlock(t *testing.T) { if block.Hash() != params.RopstenGenesisHash { t.Errorf("wrong ropsten genesis hash, got %v, want %v", block.Hash(), params.RopstenGenesisHash) } + block = DefaultRinkebyGenesisBlock().ToBlock(nil) + if block.Hash() != params.RinkebyGenesisHash { + t.Errorf("wrong rinkeby genesis hash, got %v, want %v", block.Hash(), params.RinkebyGenesisHash) + } + block = DefaultGoerliGenesisBlock().ToBlock(nil) + if block.Hash() != params.GoerliGenesisHash { + t.Errorf("wrong goerli genesis hash, got %v, want %v", block.Hash(), params.GoerliGenesisHash) + } +} + +func TestInvalidCliqueConfig(t *testing.T) { + block := DefaultGoerliGenesisBlock() + block.ExtraData = []byte{} + if _, err := block.Commit(nil); err == nil { + t.Fatal("Expected error on invalid clique config") + } } func TestSetupGenesis(t *testing.T) { diff --git a/miner/worker_test.go b/miner/worker_test.go index 9c4dc0f37..5b35c66dc 100644 --- a/miner/worker_test.go +++ b/miner/worker_test.go @@ -74,8 +74,10 @@ var ( func init() { testTxPoolConfig = core.DefaultTxPoolConfig testTxPoolConfig.Journal = "" - ethashChainConfig = params.TestChainConfig - cliqueChainConfig = params.TestChainConfig + ethashChainConfig = new(params.ChainConfig) + *ethashChainConfig = *params.TestChainConfig + cliqueChainConfig = new(params.ChainConfig) + *cliqueChainConfig = *params.TestChainConfig cliqueChainConfig.Clique = ¶ms.CliqueConfig{ Period: 10, Epoch: 30000, From ca9bce9a45f02dfb4f306c897311408e265550ae Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi <1591639+s1na@users.noreply.github.com> Date: Tue, 21 Sep 2021 22:26:35 +0200 Subject: [PATCH 60/73] rpc: set scheme for ws and ipc conns to the server (#23615) * rpc: set scheme for ws and ipc conns to the server * rpc: review fixes * rpc: bugfix * rpc: empty default scheme --- rpc/client.go | 44 +++++++++++++++++++++++++++++++++----------- 1 file changed, 33 insertions(+), 11 deletions(-) diff --git a/rpc/client.go b/rpc/client.go index 198ce6357..e9deb3f6d 100644 --- a/rpc/client.go +++ b/rpc/client.go @@ -59,6 +59,12 @@ const ( maxClientSubscriptionBuffer = 20000 ) +const ( + httpScheme = "http" + wsScheme = "ws" + ipcScheme = "ipc" +) + // BatchElem is an element in a batch request. type BatchElem struct { Method string @@ -75,7 +81,7 @@ type BatchElem struct { // Client represents a connection to an RPC server. type Client struct { idgen func() ID // for subscriptions - isHTTP bool + scheme string // connection type: http, ws or ipc services *serviceRegistry idCounter uint32 @@ -111,6 +117,10 @@ type clientConn struct { func (c *Client) newClientConn(conn ServerCodec) *clientConn { ctx := context.WithValue(context.Background(), clientContextKey{}, c) + // Http connections have already set the scheme + if !c.isHTTP() && c.scheme != "" { + ctx = context.WithValue(ctx, "scheme", c.scheme) + } handler := newHandler(ctx, conn, c.idgen, c.services) return &clientConn{conn, handler} } @@ -136,7 +146,7 @@ func (op *requestOp) wait(ctx context.Context, c *Client) (*jsonrpcMessage, erro select { case <-ctx.Done(): // Send the timeout to dispatch so it can remove the request IDs. - if !c.isHTTP { + if !c.isHTTP() { select { case c.reqTimeout <- op: case <-c.closing: @@ -203,10 +213,18 @@ func newClient(initctx context.Context, connect reconnectFunc) (*Client, error) } func initClient(conn ServerCodec, idgen func() ID, services *serviceRegistry) *Client { - _, isHTTP := conn.(*httpConn) + scheme := "" + switch conn.(type) { + case *httpConn: + scheme = httpScheme + case *websocketCodec: + scheme = wsScheme + case *jsonCodec: + scheme = ipcScheme + } c := &Client{ idgen: idgen, - isHTTP: isHTTP, + scheme: scheme, services: services, writeConn: conn, close: make(chan struct{}), @@ -219,7 +237,7 @@ func initClient(conn ServerCodec, idgen func() ID, services *serviceRegistry) *C reqSent: make(chan error, 1), reqTimeout: make(chan *requestOp), } - if !isHTTP { + if !c.isHTTP() { go c.dispatch(conn) } return c @@ -250,7 +268,7 @@ func (c *Client) SupportedModules() (map[string]string, error) { // Close closes the client, aborting any in-flight requests. func (c *Client) Close() { - if c.isHTTP { + if c.isHTTP() { return } select { @@ -264,7 +282,7 @@ func (c *Client) Close() { // This method only works for clients using HTTP, it doesn't have // any effect for clients using another transport. func (c *Client) SetHeader(key, value string) { - if !c.isHTTP { + if !c.isHTTP() { return } conn := c.writeConn.(*httpConn) @@ -298,7 +316,7 @@ func (c *Client) CallContext(ctx context.Context, result interface{}, method str } op := &requestOp{ids: []json.RawMessage{msg.ID}, resp: make(chan *jsonrpcMessage, 1)} - if c.isHTTP { + if c.isHTTP() { err = c.sendHTTP(ctx, op, msg) } else { err = c.send(ctx, op, msg) @@ -357,7 +375,7 @@ func (c *Client) BatchCallContext(ctx context.Context, b []BatchElem) error { } var err error - if c.isHTTP { + if c.isHTTP() { err = c.sendBatchHTTP(ctx, op, msgs) } else { err = c.send(ctx, op, msgs) @@ -402,7 +420,7 @@ func (c *Client) Notify(ctx context.Context, method string, args ...interface{}) } msg.ID = nil - if c.isHTTP { + if c.isHTTP() { return c.sendHTTP(ctx, op, msg) } return c.send(ctx, op, msg) @@ -440,7 +458,7 @@ func (c *Client) Subscribe(ctx context.Context, namespace string, channel interf if chanVal.IsNil() { panic("channel given to Subscribe must not be nil") } - if c.isHTTP { + if c.isHTTP() { return nil, ErrNotificationsUnsupported } @@ -642,3 +660,7 @@ func (c *Client) read(codec ServerCodec) { c.readOp <- readOp{msgs, batch} } } + +func (c *Client) isHTTP() bool { + return c.scheme == httpScheme +} From 57a3fab8a75eeb9c2f4fab770b73b51b9fe672c5 Mon Sep 17 00:00:00 2001 From: gary rong Date: Wed, 22 Sep 2021 16:18:18 +0800 Subject: [PATCH 61/73] accounts/abi: fix resolving single struct argument (#23573) --- accounts/abi/argument.go | 2 +- accounts/abi/bind/bind_test.go | 71 ++++++++++++++++++++++++++++++++++ accounts/abi/unpack_test.go | 18 +++++---- 3 files changed, 83 insertions(+), 8 deletions(-) diff --git a/accounts/abi/argument.go b/accounts/abi/argument.go index e6d524559..261b4d1b8 100644 --- a/accounts/abi/argument.go +++ b/accounts/abi/argument.go @@ -137,7 +137,7 @@ func (arguments Arguments) copyAtomic(v interface{}, marshalledValues interface{ dst := reflect.ValueOf(v).Elem() src := reflect.ValueOf(marshalledValues) - if dst.Kind() == reflect.Struct && src.Kind() != reflect.Struct { + if dst.Kind() == reflect.Struct { return set(dst.Field(0), src) } return set(dst, src) diff --git a/accounts/abi/bind/bind_test.go b/accounts/abi/bind/bind_test.go index d49b436db..6d9052f88 100644 --- a/accounts/abi/bind/bind_test.go +++ b/accounts/abi/bind/bind_test.go @@ -1785,6 +1785,77 @@ var bindTests = []struct { nil, nil, }, + // Test resolving single struct argument + { + `NewSingleStructArgument`, + ` + pragma solidity ^0.8.0; + + contract NewSingleStructArgument { + struct MyStruct{ + uint256 a; + uint256 b; + } + event StructEvent(MyStruct s); + function TestEvent() public { + emit StructEvent(MyStruct({a: 1, b: 2})); + } + } + `, + []string{"608060405234801561001057600080fd5b50610113806100206000396000f3fe6080604052348015600f57600080fd5b506004361060285760003560e01c806324ec1d3f14602d575b600080fd5b60336035565b005b7fb4b2ff75e30cb4317eaae16dd8a187dd89978df17565104caa6c2797caae27d460405180604001604052806001815260200160028152506040516078919060ba565b60405180910390a1565b6040820160008201516096600085018260ad565b50602082015160a7602085018260ad565b50505050565b60b48160d3565b82525050565b600060408201905060cd60008301846082565b92915050565b600081905091905056fea26469706673582212208823628796125bf9941ce4eda18da1be3cf2931b231708ab848e1bd7151c0c9a64736f6c63430008070033"}, + []string{`[{"anonymous":false,"inputs":[{"components":[{"internalType":"uint256","name":"a","type":"uint256"},{"internalType":"uint256","name":"b","type":"uint256"}],"indexed":false,"internalType":"struct Test.MyStruct","name":"s","type":"tuple"}],"name":"StructEvent","type":"event"},{"inputs":[],"name":"TestEvent","outputs":[],"stateMutability":"nonpayable","type":"function"}]`}, + ` + "math/big" + + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/eth/ethconfig" + `, + ` + var ( + key, _ = crypto.GenerateKey() + user, _ = bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) + sim = backends.NewSimulatedBackend(core.GenesisAlloc{user.From: {Balance: big.NewInt(1000000000000000000)}}, ethconfig.Defaults.Miner.GasCeil) + ) + defer sim.Close() + + _, _, d, err := DeployNewSingleStructArgument(user, sim) + if err != nil { + t.Fatalf("Failed to deploy contract %v", err) + } + sim.Commit() + + _, err = d.TestEvent(user) + if err != nil { + t.Fatalf("Failed to call contract %v", err) + } + sim.Commit() + + it, err := d.FilterStructEvent(nil) + if err != nil { + t.Fatalf("Failed to filter contract event %v", err) + } + var count int + for it.Next() { + if it.Event.S.A.Cmp(big.NewInt(1)) != 0 { + t.Fatal("Unexpected contract event") + } + if it.Event.S.B.Cmp(big.NewInt(2)) != 0 { + t.Fatal("Unexpected contract event") + } + count += 1 + } + if count != 1 { + t.Fatal("Unexpected contract event number") + } + `, + nil, + nil, + nil, + nil, + }, } // Tests that packages generated by the binder can be successfully compiled and diff --git a/accounts/abi/unpack_test.go b/accounts/abi/unpack_test.go index b88f77805..e617f8abc 100644 --- a/accounts/abi/unpack_test.go +++ b/accounts/abi/unpack_test.go @@ -762,20 +762,24 @@ func TestUnpackTuple(t *testing.T) { buff.Write(common.Hex2Bytes("ffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffff")) // ret[b] = -1 // If the result is single tuple, use struct as return value container directly. - v := struct { + type v struct { A *big.Int B *big.Int - }{new(big.Int), new(big.Int)} + } + type r struct { + Result v + } + var ret0 = new(r) + err = abi.UnpackIntoInterface(ret0, "tuple", buff.Bytes()) - err = abi.UnpackIntoInterface(&v, "tuple", buff.Bytes()) if err != nil { t.Error(err) } else { - if v.A.Cmp(big.NewInt(1)) != 0 { - t.Errorf("unexpected value unpacked: want %x, got %x", 1, v.A) + if ret0.Result.A.Cmp(big.NewInt(1)) != 0 { + t.Errorf("unexpected value unpacked: want %x, got %x", 1, ret0.Result.A) } - if v.B.Cmp(big.NewInt(-1)) != 0 { - t.Errorf("unexpected value unpacked: want %x, got %x", -1, v.B) + if ret0.Result.B.Cmp(big.NewInt(-1)) != 0 { + t.Errorf("unexpected value unpacked: want %x, got %x", -1, ret0.Result.B) } } From 51ececb64e9cacaaa9b736b5bc2455225d061edb Mon Sep 17 00:00:00 2001 From: Jordan Krage Date: Mon, 27 Sep 2021 09:31:45 -0500 Subject: [PATCH 62/73] rpc: set pong read deadline (#23556) This PR adds a 30s timeout for the remote part to answer a ping message, thus detecting (silent) disconnnects --- rpc/websocket.go | 6 +++ rpc/websocket_test.go | 89 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 95 insertions(+) diff --git a/rpc/websocket.go b/rpc/websocket.go index afeb4c208..5571324af 100644 --- a/rpc/websocket.go +++ b/rpc/websocket.go @@ -37,6 +37,7 @@ const ( wsWriteBuffer = 1024 wsPingInterval = 60 * time.Second wsPingWriteTimeout = 5 * time.Second + wsPongTimeout = 30 * time.Second wsMessageSizeLimit = 15 * 1024 * 1024 ) @@ -241,6 +242,10 @@ type websocketCodec struct { func newWebsocketCodec(conn *websocket.Conn) ServerCodec { conn.SetReadLimit(wsMessageSizeLimit) + conn.SetPongHandler(func(appData string) error { + conn.SetReadDeadline(time.Time{}) + return nil + }) wc := &websocketCodec{ jsonCodec: NewFuncCodec(conn, conn.WriteJSON, conn.ReadJSON).(*jsonCodec), conn: conn, @@ -287,6 +292,7 @@ func (wc *websocketCodec) pingLoop() { wc.jsonCodec.encMu.Lock() wc.conn.SetWriteDeadline(time.Now().Add(wsPingWriteTimeout)) wc.conn.WriteMessage(websocket.PingMessage, nil) + wc.conn.SetReadDeadline(time.Now().Add(wsPongTimeout)) wc.jsonCodec.encMu.Unlock() timer.Reset(wsPingInterval) } diff --git a/rpc/websocket_test.go b/rpc/websocket_test.go index 4976853ba..248609283 100644 --- a/rpc/websocket_test.go +++ b/rpc/websocket_test.go @@ -18,11 +18,15 @@ package rpc import ( "context" + "io" "net" "net/http" "net/http/httptest" + "net/http/httputil" + "net/url" "reflect" "strings" + "sync/atomic" "testing" "time" @@ -188,6 +192,63 @@ func TestClientWebsocketLargeMessage(t *testing.T) { } } +func TestClientWebsocketSevered(t *testing.T) { + t.Parallel() + + var ( + server = wsPingTestServer(t, nil) + ctx = context.Background() + ) + defer server.Shutdown(ctx) + + u, err := url.Parse("http://" + server.Addr) + if err != nil { + t.Fatal(err) + } + rproxy := httputil.NewSingleHostReverseProxy(u) + var severable *severableReadWriteCloser + rproxy.ModifyResponse = func(response *http.Response) error { + severable = &severableReadWriteCloser{ReadWriteCloser: response.Body.(io.ReadWriteCloser)} + response.Body = severable + return nil + } + frontendProxy := httptest.NewServer(rproxy) + defer frontendProxy.Close() + + wsURL := "ws:" + strings.TrimPrefix(frontendProxy.URL, "http:") + client, err := DialWebsocket(ctx, wsURL, "") + if err != nil { + t.Fatalf("client dial error: %v", err) + } + defer client.Close() + + resultChan := make(chan int) + sub, err := client.EthSubscribe(ctx, resultChan, "foo") + if err != nil { + t.Fatalf("client subscribe error: %v", err) + } + + // sever the connection + severable.Sever() + + // Wait for subscription error. + timeout := time.NewTimer(3 * wsPingInterval) + defer timeout.Stop() + for { + select { + case err := <-sub.Err(): + t.Log("client subscription error:", err) + return + case result := <-resultChan: + t.Error("unexpected result:", result) + return + case <-timeout.C: + t.Error("didn't get any error within the test timeout") + return + } + } +} + // wsPingTestServer runs a WebSocket server which accepts a single subscription request. // When a value arrives on sendPing, the server sends a ping frame, waits for a matching // pong and finally delivers a single subscription result. @@ -290,3 +351,31 @@ func wsPingTestHandler(t *testing.T, conn *websocket.Conn, shutdown, sendPing <- } } } + +// severableReadWriteCloser wraps an io.ReadWriteCloser and provides a Sever() method to drop writes and read empty. +type severableReadWriteCloser struct { + io.ReadWriteCloser + severed int32 // atomic +} + +func (s *severableReadWriteCloser) Sever() { + atomic.StoreInt32(&s.severed, 1) +} + +func (s *severableReadWriteCloser) Read(p []byte) (n int, err error) { + if atomic.LoadInt32(&s.severed) > 0 { + return 0, nil + } + return s.ReadWriteCloser.Read(p) +} + +func (s *severableReadWriteCloser) Write(p []byte) (n int, err error) { + if atomic.LoadInt32(&s.severed) > 0 { + return len(p), nil + } + return s.ReadWriteCloser.Write(p) +} + +func (s *severableReadWriteCloser) Close() error { + return s.ReadWriteCloser.Close() +} From 12674d493e2958208a19db0818f75cd94ba1b638 Mon Sep 17 00:00:00 2001 From: Zachinquarantine Date: Mon, 27 Sep 2021 12:42:48 -0400 Subject: [PATCH 63/73] cmd/puppeth: remove shh from Dockerfile (#23634) --- cmd/puppeth/module_explorer.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/puppeth/module_explorer.go b/cmd/puppeth/module_explorer.go index 3ce9d612b..05821dfc5 100644 --- a/cmd/puppeth/module_explorer.go +++ b/cmd/puppeth/module_explorer.go @@ -35,8 +35,8 @@ FROM puppeth/blockscout:latest ADD genesis.json /genesis.json RUN \ echo 'geth --cache 512 init /genesis.json' > explorer.sh && \ - echo $'geth --networkid {{.NetworkID}} --syncmode "full" --gcmode "archive" --port {{.EthPort}} --bootnodes {{.Bootnodes}} --ethstats \'{{.Ethstats}}\' --cache=512 --http --http.api "net,web3,eth,shh,debug" --http.corsdomain "*" --http.vhosts "*" --ws --ws.origins "*" --exitwhensynced' >> explorer.sh && \ - echo $'exec geth --networkid {{.NetworkID}} --syncmode "full" --gcmode "archive" --port {{.EthPort}} --bootnodes {{.Bootnodes}} --ethstats \'{{.Ethstats}}\' --cache=512 --http --http.api "net,web3,eth,shh,debug" --http.corsdomain "*" --http.vhosts "*" --ws --ws.origins "*" &' >> explorer.sh && \ + echo $'geth --networkid {{.NetworkID}} --syncmode "full" --gcmode "archive" --port {{.EthPort}} --bootnodes {{.Bootnodes}} --ethstats \'{{.Ethstats}}\' --cache=512 --http --http.api "net,web3,eth,debug" --http.corsdomain "*" --http.vhosts "*" --ws --ws.origins "*" --exitwhensynced' >> explorer.sh && \ + echo $'exec geth --networkid {{.NetworkID}} --syncmode "full" --gcmode "archive" --port {{.EthPort}} --bootnodes {{.Bootnodes}} --ethstats \'{{.Ethstats}}\' --cache=512 --http --http.api "net,web3,eth,debug" --http.corsdomain "*" --http.vhosts "*" --ws --ws.origins "*" &' >> explorer.sh && \ echo '/usr/local/bin/docker-entrypoint.sh postgres &' >> explorer.sh && \ echo 'sleep 5' >> explorer.sh && \ echo 'mix do ecto.drop --force, ecto.create, ecto.migrate' >> explorer.sh && \ From ac7baeab57405c64592b1646a91e0a2bb33d8d6c Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi <1591639+s1na@users.noreply.github.com> Date: Mon, 27 Sep 2021 18:44:16 +0200 Subject: [PATCH 64/73] eth/tracers: re-write of 4byte tracer using enter/exit (#23622) * eth/tracers: add re-write of 4byte tracer using enter/exit * eth/tracers: fix 4byte indent --- eth/tracers/internal/tracers/4byte_tracer.js | 37 ++------ .../internal/tracers/4byte_tracer_legacy.js | 86 +++++++++++++++++++ eth/tracers/internal/tracers/assets.go | 67 ++++++++++----- 3 files changed, 139 insertions(+), 51 deletions(-) create mode 100644 eth/tracers/internal/tracers/4byte_tracer_legacy.js diff --git a/eth/tracers/internal/tracers/4byte_tracer.js b/eth/tracers/internal/tracers/4byte_tracer.js index 462b4ad4c..9ec3209f8 100644 --- a/eth/tracers/internal/tracers/4byte_tracer.js +++ b/eth/tracers/internal/tracers/4byte_tracer.js @@ -31,48 +31,27 @@ // ids aggregates the 4byte ids found. ids : {}, - // callType returns 'false' for non-calls, or the peek-index for the first param - // after 'value', i.e. meminstart. - callType: function(opstr){ - switch(opstr){ - case "CALL": case "CALLCODE": - // gas, addr, val, memin, meminsz, memout, memoutsz - return 3; // stack ptr to memin - - case "DELEGATECALL": case "STATICCALL": - // gas, addr, memin, meminsz, memout, memoutsz - return 2; // stack ptr to memin - } - return false; - }, - // store save the given indentifier and datasize. store: function(id, size){ var key = "" + toHex(id) + "-" + size; this.ids[key] = this.ids[key] + 1 || 1; }, - // step is invoked for every opcode that the VM executes. - step: function(log, db) { - // Skip any opcodes that are not internal calls - var ct = this.callType(log.op.toString()); - if (!ct) { - return; - } + enter: function(frame) { // Skip any pre-compile invocations, those are just fancy opcodes - if (isPrecompiled(toAddress(log.stack.peek(1).toString(16)))) { + if (isPrecompiled(frame.getTo())) { return; } - // Gather internal call details - var inSz = log.stack.peek(ct + 1).valueOf(); - if (inSz >= 4) { - var inOff = log.stack.peek(ct).valueOf(); - this.store(log.memory.slice(inOff, inOff + 4), inSz-4); + var input = frame.getInput() + if (input.length >= 4) { + this.store(slice(input, 0, 4), input.length - 4); } }, + exit: function(frameResult) {}, + // fault is invoked when the actual execution of an opcode fails. - fault: function(log, db) { }, + fault: function(log, db) {}, // result is invoked when all the opcodes have been iterated over and returns // the final result of the tracing. diff --git a/eth/tracers/internal/tracers/4byte_tracer_legacy.js b/eth/tracers/internal/tracers/4byte_tracer_legacy.js new file mode 100644 index 000000000..462b4ad4c --- /dev/null +++ b/eth/tracers/internal/tracers/4byte_tracer_legacy.js @@ -0,0 +1,86 @@ +// Copyright 2017 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +// 4byteTracer searches for 4byte-identifiers, and collects them for post-processing. +// It collects the methods identifiers along with the size of the supplied data, so +// a reversed signature can be matched against the size of the data. +// +// Example: +// > debug.traceTransaction( "0x214e597e35da083692f5386141e69f47e973b2c56e7a8073b1ea08fd7571e9de", {tracer: "4byteTracer"}) +// { +// 0x27dc297e-128: 1, +// 0x38cc4831-0: 2, +// 0x524f3889-96: 1, +// 0xadf59f99-288: 1, +// 0xc281d19e-0: 1 +// } +{ + // ids aggregates the 4byte ids found. + ids : {}, + + // callType returns 'false' for non-calls, or the peek-index for the first param + // after 'value', i.e. meminstart. + callType: function(opstr){ + switch(opstr){ + case "CALL": case "CALLCODE": + // gas, addr, val, memin, meminsz, memout, memoutsz + return 3; // stack ptr to memin + + case "DELEGATECALL": case "STATICCALL": + // gas, addr, memin, meminsz, memout, memoutsz + return 2; // stack ptr to memin + } + return false; + }, + + // store save the given indentifier and datasize. + store: function(id, size){ + var key = "" + toHex(id) + "-" + size; + this.ids[key] = this.ids[key] + 1 || 1; + }, + + // step is invoked for every opcode that the VM executes. + step: function(log, db) { + // Skip any opcodes that are not internal calls + var ct = this.callType(log.op.toString()); + if (!ct) { + return; + } + // Skip any pre-compile invocations, those are just fancy opcodes + if (isPrecompiled(toAddress(log.stack.peek(1).toString(16)))) { + return; + } + // Gather internal call details + var inSz = log.stack.peek(ct + 1).valueOf(); + if (inSz >= 4) { + var inOff = log.stack.peek(ct).valueOf(); + this.store(log.memory.slice(inOff, inOff + 4), inSz-4); + } + }, + + // fault is invoked when the actual execution of an opcode fails. + fault: function(log, db) { }, + + // result is invoked when all the opcodes have been iterated over and returns + // the final result of the tracing. + result: function(ctx) { + // Save the outer calldata also + if (ctx.input.length >= 4) { + this.store(slice(ctx.input, 0, 4), ctx.input.length-4) + } + return this.ids; + }, +} diff --git a/eth/tracers/internal/tracers/assets.go b/eth/tracers/internal/tracers/assets.go index 52cc7f39f..c1469c758 100644 --- a/eth/tracers/internal/tracers/assets.go +++ b/eth/tracers/internal/tracers/assets.go @@ -1,6 +1,7 @@ // Code generated by go-bindata. DO NOT EDIT. // sources: -// 4byte_tracer.js (2.933kB) +// 4byte_tracer.js (2.224kB) +// 4byte_tracer_legacy.js (2.933kB) // bigram_tracer.js (1.712kB) // call_tracer.js (4.251kB) // call_tracer_legacy.js (8.956kB) @@ -78,7 +79,7 @@ func (fi bindataFileInfo) Sys() interface{} { return nil } -var __4byte_tracerJs = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x94\x56\x5b\x6f\xdb\x4a\x0e\x7e\xb6\x7f\x05\xd7\x2f\xb5\x51\x59\x8e\x2f\x89\x2f\xd9\x16\xf0\xe6\xa4\x6d\x80\x9c\x24\x88\xdd\x3d\x28\x16\xfb\x30\x9e\xa1\xac\xd9\xc8\x33\xc2\x0c\xe5\x4b\x73\xf2\xdf\x17\x1c\x49\x89\x93\xd3\x62\xbb\x4f\x96\x47\xc3\x8f\x1f\xc9\x8f\xa4\x7a\x3d\xb8\xb0\xf9\xc1\xe9\x75\x4a\x30\x38\xe9\x8f\x61\x99\x22\xac\x6d\x17\x29\x45\x87\xc5\x06\xe6\x05\xa5\xd6\xf9\x66\xaf\x07\xcb\x54\x7b\x48\x74\x86\xa0\x3d\xe4\xc2\x11\xd8\x04\xe8\xcd\xfd\x4c\xaf\x9c\x70\x87\xb8\xd9\xeb\x95\x36\x3f\x7c\xcd\x08\x89\x43\x04\x6f\x13\xda\x09\x87\x33\x38\xd8\x02\xa4\x30\xe0\x50\x69\x4f\x4e\xaf\x0a\x42\xd0\x04\xc2\xa8\x9e\x75\xb0\xb1\x4a\x27\x07\x86\xd4\x04\x85\x51\xe8\x82\x6b\x42\xb7\xf1\x35\x8f\xcf\x37\x5f\xe1\x1a\xbd\x47\x07\x9f\xd1\xa0\x13\x19\xdc\x15\xab\x4c\x4b\xb8\xd6\x12\x8d\x47\x10\x1e\x72\x3e\xf1\x29\x2a\x58\x05\x38\x36\xfc\xc4\x54\x16\x15\x15\xf8\x64\x0b\xa3\x04\x69\x6b\x22\x40\xcd\xcc\x61\x8b\xce\x6b\x6b\x60\x58\xbb\xaa\x00\x23\xb0\x8e\x41\xda\x82\x38\x00\x07\x36\x67\xbb\x0e\x08\x73\x80\x4c\xd0\x8b\xe9\x2f\x24\xe4\x25\x6e\x05\xda\x04\x37\xa9\xcd\x11\x28\x15\xc4\x51\xef\x74\x96\xc1\x0a\xa1\xf0\x98\x14\x59\xc4\x68\xab\x82\xe0\x8f\xab\xe5\x97\xdb\xaf\x4b\x98\xdf\x7c\x83\x3f\xe6\xf7\xf7\xf3\x9b\xe5\xb7\x73\xd8\x69\x4a\x6d\x41\x80\x5b\x2c\xa1\xf4\x26\xcf\x34\x2a\xd8\x09\xe7\x84\xa1\x03\xd8\x84\x11\x7e\xbf\xbc\xbf\xf8\x32\xbf\x59\xce\xff\x71\x75\x7d\xb5\xfc\x06\xd6\xc1\xa7\xab\xe5\xcd\xe5\x62\x01\x9f\x6e\xef\x61\x0e\x77\xf3\xfb\xe5\xd5\xc5\xd7\xeb\xf9\x3d\xdc\x7d\xbd\xbf\xbb\x5d\x5c\xc6\xb0\x40\x66\x85\x6c\xff\xbf\x73\x9e\x84\xea\x39\x04\x85\x24\x74\xe6\xeb\x4c\x7c\xb3\x05\xf8\xd4\x16\x99\x82\x54\x6c\x11\x1c\x4a\xd4\x5b\x54\x20\x40\xda\xfc\xf0\xcb\x45\x65\x2c\x91\x59\xb3\x0e\x31\xff\x54\x90\x70\x95\x80\xb1\x14\x81\x47\x84\xbf\xa7\x44\xf9\xac\xd7\xdb\xed\x76\xf1\xda\x14\xb1\x75\xeb\x5e\x56\xc2\xf9\xde\xc7\xb8\xc9\x98\xa3\xd5\x81\x70\xe9\x84\x44\x07\x1e\x85\x93\x29\xfa\x10\x4c\x78\xd1\xd5\x0a\x0d\xe9\x44\xa3\xf3\x11\x8b\x14\xa4\xcd\x32\x94\xe4\x99\xc1\x26\x5c\xcc\xad\xa7\x6e\xee\xac\x44\xef\xb5\x59\x73\xe0\x70\x45\xaf\x2e\xc2\x06\x29\xb5\xca\xc3\x11\xdc\xdb\x68\xbc\xfe\x8e\x75\x36\x7c\x91\x97\x65\x54\x82\x44\x04\xde\x86\xe8\xc1\x21\xcb\x0c\x15\x78\xbd\x36\x82\x0a\x87\xa1\x97\x56\x08\x1b\x41\x92\xc5\x2e\xd6\x42\x1b\x4f\x7f\x01\x64\x9c\xba\x22\x97\x7b\xb1\xc9\x33\x9c\xf1\x33\xc0\x47\x50\xb8\x2a\xd6\x31\x71\x0a\x96\x4e\x18\x2f\x24\x8b\xbb\x0d\xad\x93\xfd\xa0\x3f\xc2\xd3\xe9\x18\x87\xa7\x4a\x9c\x4c\x86\x67\xd3\x41\x72\x3a\x9c\x9c\xf5\x47\x7d\x3c\x9b\x26\xa3\x31\x4e\xc7\xc3\xd5\x40\x9e\x9e\xe1\x58\x4c\x4e\xc6\xc3\x55\x1f\xc5\xc9\x24\x51\xe3\xd3\x71\x1f\xa7\x0a\x5b\x11\x3c\x06\x60\x37\x83\xd6\x51\xa6\x5b\x4f\x9d\xd2\xfb\x63\xf9\x03\x70\xb2\x1f\x8c\x95\x1c\x4c\xc7\xd8\xed\x0f\x26\x33\xe8\x47\x2f\x6f\x86\x13\x29\x47\x93\x61\xbf\x7b\x32\x83\xc1\xd1\xf9\xe9\x60\x94\x0c\x27\x93\x69\x77\x7a\xf6\xda\x40\xa8\xe4\x74\x9a\x4c\xa7\xdd\xc1\xe4\x0d\x94\x1c\x4c\xfa\xaa\x3f\x45\x86\xea\x97\xc7\x4f\xcd\xc7\x66\x83\x07\x8e\xf2\x20\xd6\x6b\x87\x6b\x41\x58\x56\x2d\x30\x0e\x2f\x12\x1e\x16\x71\xb3\xc1\xcf\x33\x78\x7c\x8a\x9a\xc1\x46\x8a\x2c\x5b\x1e\x72\x56\x35\x15\xce\x78\x78\x97\x88\xcc\xe3\xbb\xa0\x0b\x63\x4d\x97\x2f\x78\x1e\x1f\x01\x2f\x47\x7c\xe8\x6a\xa3\x70\x1f\x2e\xf0\x51\xa2\x9d\x27\x1e\xb3\x62\x13\x10\x45\xc2\xd3\xe4\xdd\x56\x64\x05\xbe\x8b\x40\xc7\x18\xc3\x06\x37\x5c\x54\xe1\x28\x6e\x36\x6a\x97\x33\x48\x0a\x53\x56\xca\xe6\x9e\x5c\xe7\xb1\xd9\x68\xf8\x9d\x26\x99\x1e\x1d\x48\xe1\x11\x5a\x17\xf3\xeb\xeb\xd6\x0c\x5e\xfe\x5c\xdc\xfe\x76\xd9\x9a\x35\x1b\x0d\x76\xb9\x16\x2c\x6d\xa5\x5c\x04\x5b\x91\x45\xa5\xbb\xea\xc7\x7f\x0f\x0f\xb6\xa0\xfa\xd7\x7f\x67\xb3\x32\x5e\x18\x9e\x43\xaf\x07\x9e\x84\x7c\x80\x9c\x1c\x90\x2d\xcd\x9a\xcf\xae\x7f\xbb\xbc\xbe\xfc\x3c\x5f\x5e\xbe\xa2\xb0\x58\xce\x97\x57\x17\xe5\xd1\x5f\x49\xfc\x1f\xfe\x07\x3f\xf3\xdf\x68\x3c\x35\x9f\x6f\x85\x9a\x9c\x37\x1b\x75\xd5\x3c\xf1\x9c\xf2\x3c\x8d\xc2\x18\xd1\x3c\x3c\xb9\x2c\x55\x6b\x86\x3e\xe7\x8e\xe1\x0e\x8a\x9b\x8d\x70\xff\x28\xdf\x5a\x45\xa1\xb9\x42\x86\xb7\xc2\xc1\x03\x1e\xe0\x03\xb4\x5a\xf0\x1e\xc8\x7e\xc1\x7d\x5b\xab\x0e\xbc\x87\x56\x97\x4f\xf8\xe6\x79\xb3\xd1\xa0\x54\xfb\x58\x2b\xff\xaf\x07\x3c\xfc\x1b\x3e\xc0\xeb\xff\xef\xa1\x0f\x7f\xfe\x09\xfd\x57\x34\x31\xe7\x85\xa1\xcd\xd6\x3e\xa0\x0a\x92\xe1\x01\x70\x00\x9b\x4b\xab\xaa\x8d\xc1\x11\xfc\xf3\x77\xc0\x3d\xca\x82\xd0\x07\xba\x98\x1f\xb1\xcd\xec\x3a\x02\xb5\xea\x00\xb3\xed\xf5\x60\xf1\xa0\xf3\xb0\xb8\x4a\x14\x5f\xc2\xf0\x46\x34\x96\x40\x1b\x42\x67\x44\x16\xa4\xed\xab\xf8\x24\xd5\x7c\x6b\xf5\x31\x6a\x6c\xf3\x98\xec\x82\x9c\x36\xeb\x76\xa7\xc3\x31\xea\x04\xda\x7f\x93\x54\xfa\xaa\xd2\x7f\x5e\x15\xe3\xd8\x75\xee\xb0\x2b\xed\x26\x0f\x5f\x19\x66\x6b\x65\xd8\xc3\x3e\x02\x4a\x2d\xef\x6f\x87\xf0\x9f\xc2\x13\x24\xc2\xc8\x67\xa2\x15\xbe\xf6\x77\x0e\x2b\x63\xd5\x26\x3b\x57\xca\xa1\xf7\x81\x51\x50\x42\xcc\x6d\xd6\xee\x77\x5e\xc8\xf5\xcf\x3a\x9d\xce\xcf\x48\x7d\x16\x61\xf7\xbf\x0a\xbc\x5e\x62\x55\xfc\xda\x2c\xbe\xc3\x07\x78\xe3\x41\x12\x57\xad\x13\x87\x5e\xbd\x4d\xda\xcf\x19\x08\xd7\x3f\x7e\x80\x51\xe5\xb2\x84\xb8\x4d\x92\x1f\x61\xbc\xb1\x2f\x65\x12\x14\x17\x22\x62\xd1\xbb\x43\xec\x79\x6d\xb5\x03\x48\x54\x61\xbd\x87\x51\x27\x0a\xd4\xba\xa3\x4e\x15\x4f\x2d\x9d\x44\x14\x19\x1d\x6b\x67\x97\x56\xdf\x07\x42\x52\x21\xb2\x4a\x2e\xfc\xad\x63\x13\x10\xa6\x56\x54\x52\x6e\xee\x46\xb0\xff\xa1\x86\xa0\x76\xe1\xd0\xff\xc8\x07\x27\x8f\xfd\xd4\xe2\x0a\x3b\x7f\x85\xdc\x60\x84\x4e\xf0\x47\x8f\xdd\x56\x2d\x56\x0d\xcd\x00\x57\xce\x42\xce\x7f\x05\x5c\x2d\x2e\xde\x1e\x61\xa9\x36\xca\xf3\x23\x52\x92\xf6\x2f\xa2\xae\x9b\xd9\x16\x3c\x3f\xb9\x86\xdc\xc0\x20\x32\x6f\xab\xaa\x48\xda\xc7\xda\xe4\x05\xc5\x19\x9a\x35\xa5\xc7\x15\x3a\x4a\x7a\x99\xe9\xe7\xcb\x11\x9c\x44\x21\xd1\x6f\xcd\xbb\xa3\xce\xeb\x29\x53\xf7\x73\xd9\xc1\x4f\xcd\xff\x06\x00\x00\xff\xff\x8e\xc8\x27\x72\x75\x0b\x00\x00") +var __4byte_tracerJs = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x8c\x55\x5b\x6f\x22\x39\x13\x7d\x86\x5f\x71\xc4\x13\x68\x9a\x4b\x73\x09\x97\xf9\x32\x12\xdf\x28\x99\x41\xca\x66\x22\x42\x34\x8a\x56\xfb\x60\xda\xd5\xdd\xde\x18\xbb\x65\xbb\xb9\x6c\x26\xff\x7d\x65\x37\xe4\x36\xbb\xda\x79\x02\xec\xaa\x73\xaa\x4e\x1d\x17\xdd\x2e\x3e\xeb\xe2\x60\x44\x96\x3b\xf4\x7b\xf1\x18\xab\x9c\x90\xe9\x36\xb9\x9c\x0c\x95\x1b\xcc\x4b\x97\x6b\x63\xeb\xdd\x2e\x56\xb9\xb0\x48\x85\x24\x08\x8b\x82\x19\x07\x9d\xc2\xbd\x8b\x97\x62\x6d\x98\x39\x74\xea\xdd\x6e\x95\xf3\x8f\xd7\x1e\x21\x35\x44\xb0\x3a\x75\x3b\x66\x68\x86\x83\x2e\x91\x30\x05\x43\x5c\x58\x67\xc4\xba\x74\x04\xe1\xc0\x14\xef\x6a\x83\x8d\xe6\x22\x3d\x78\x48\xe1\x50\x2a\x4e\x26\x50\x3b\x32\x1b\x7b\xaa\xe3\xcb\xf5\x1d\xae\xc8\x5a\x32\xf8\x42\x8a\x0c\x93\xb8\x29\xd7\x52\x24\xb8\x12\x09\x29\x4b\x60\x16\x85\x3f\xb1\x39\x71\xac\x03\x9c\x4f\xbc\xf4\xa5\xdc\x1e\x4b\xc1\xa5\x2e\x15\x67\x4e\x68\x15\x81\x84\xaf\x1c\x5b\x32\x56\x68\x85\xc1\x89\xea\x08\x18\x41\x1b\x0f\xd2\x64\xce\x37\x60\xa0\x0b\x9f\xd7\x02\x53\x07\x48\xe6\x5e\x52\x7f\x41\x90\x97\xbe\x39\x84\x0a\x34\xb9\x2e\x08\x2e\x67\xce\x77\xbd\x13\x52\x62\x4d\x28\x2d\xa5\xa5\x8c\x3c\xda\xba\x74\xf8\xbe\x58\x7d\xfd\x76\xb7\xc2\xfc\xfa\x1e\xdf\xe7\xcb\xe5\xfc\x7a\x75\xff\x11\x3b\xe1\x72\x5d\x3a\xd0\x96\x2a\x28\xb1\x29\xa4\x20\x8e\x1d\x33\x86\x29\x77\x80\x4e\x3d\xc2\x6f\x17\xcb\xcf\x5f\xe7\xd7\xab\xf9\xff\x17\x57\x8b\xd5\x3d\xb4\xc1\xe5\x62\x75\x7d\x71\x7b\x8b\xcb\x6f\x4b\xcc\x71\x33\x5f\xae\x16\x9f\xef\xae\xe6\x4b\xdc\xdc\x2d\x6f\xbe\xdd\x5e\x74\x70\x4b\xbe\x2a\xf2\xf9\xff\xad\x79\x1a\xa6\x67\x08\x9c\x1c\x13\xd2\x9e\x94\xb8\xd7\x25\x6c\xae\x4b\xc9\x91\xb3\x2d\xc1\x50\x42\x62\x4b\x1c\x0c\x89\x2e\x0e\xbf\x3c\x54\x8f\xc5\xa4\x56\x59\xe8\xf9\x5f\x0d\x89\x45\x0a\xa5\x5d\x04\x4b\x84\xff\xe5\xce\x15\xb3\x6e\x77\xb7\xdb\x75\x32\x55\x76\xb4\xc9\xba\xb2\x82\xb3\xdd\x4f\x9d\xba\xc7\x1c\xae\x0f\x8e\x56\x86\x25\x64\x60\x89\x99\x24\x27\x1b\x9a\x09\x17\x6d\xc1\x49\x39\x91\x0a\x32\x36\xf2\x26\x45\xa2\xa5\xa4\xc4\x59\x5f\xc1\x26\x04\x16\xda\xba\x76\x61\x74\x42\xd6\x0a\x95\xf9\xc6\xb1\x70\x6f\x02\xb1\x21\x97\x6b\x6e\xf1\x0a\xee\x7d\x37\x56\xfc\x45\x27\x35\x6c\x59\x54\x63\xe4\xcc\xb1\x08\x56\x87\xee\x61\xc8\xdb\x8c\x38\xac\xc8\x14\x73\xa5\xa1\xf0\x96\xd6\x84\x0d\x73\x89\x37\x3b\xcb\x98\x50\xd6\xfd\x04\xe8\x71\x4e\x13\xb9\xd8\xb3\x4d\x21\x69\xe6\xbf\x03\x9f\xc0\x69\x5d\x66\x1d\xe7\x25\x58\x19\xa6\x2c\x4b\xbc\xb9\x9b\x68\xf4\xf6\xfd\x78\x48\xa3\xe9\x98\x06\x23\xce\x7a\x93\xc1\xd9\xb4\x9f\x8e\x06\x93\xb3\x78\x18\xd3\xd9\x34\x1d\x8e\x69\x3a\x1e\xac\xfb\xc9\xe8\x8c\xc6\x6c\xd2\x1b\x0f\xd6\x31\xb1\xde\x24\xe5\xe3\xd1\x38\xa6\x29\xa7\x46\x84\xc7\x00\x6c\x66\x68\xbc\x52\xba\xf1\xd4\xaa\xd8\x1f\xab\x0f\xa0\xb7\xef\x8f\x79\xd2\x9f\x8e\xa9\x1d\xf7\x27\x33\xc4\xd1\xcb\xcd\x60\x92\x24\xc3\xc9\x20\x6e\xf7\x66\xe8\xbf\x3a\x1f\xf5\x87\xe9\x60\x32\x99\xb6\xa7\x67\x6f\x13\x18\x4f\x47\xd3\x74\x3a\x6d\xf7\x27\xef\xa0\x92\xfe\x24\xe6\xf1\x94\x3c\x54\x5c\x1d\x3f\xd5\x1f\xeb\x35\xbf\x70\xb8\x05\xcb\x32\x43\x19\x73\x54\x4d\x2d\x54\x1c\x2e\x52\xbf\x2c\x3a\xf5\x9a\xff\x3e\xc3\xe3\x53\x54\x0f\x39\xd6\x79\xc7\x5b\xef\xeb\x60\x48\xe1\x9f\xa1\x50\xcf\x43\x0e\x8e\xf1\xda\xfb\x59\x74\xea\xb5\x10\x3f\x43\x5a\xaa\x4a\x63\xc1\xa3\x30\xa6\xd6\x63\xbd\x56\xdb\x32\x83\x07\x3a\xe0\x1c\x8d\x06\x3e\xc0\xe9\xaf\xb4\x6f\x0a\xde\xc2\x07\x34\xda\xfe\xc4\x47\x7e\xac\xd7\x6a\x2e\x17\xb6\x23\xb8\xfd\xfd\x81\x0e\x7f\xe0\x1c\x6f\x7f\x7f\x40\x8c\x1f\x3f\x10\x7f\xac\xd7\x42\x99\xa4\x9c\x97\xff\x99\x33\x35\x6c\x43\x2d\x78\xc6\x6e\x17\xb7\x0f\xa2\x08\x6b\xac\x30\xd4\x4e\xf4\xa6\x08\x8b\x5f\x6d\x75\x12\x56\xa3\x8d\xe0\x72\xed\x57\xaa\x21\xfc\x59\x5a\x87\x94\xa9\xe4\x00\x5d\x24\x9a\x93\xad\xd7\x6a\x22\x45\x53\xd8\x1b\x43\xc7\x64\x5e\x11\x74\x32\x72\x2b\xdd\x6c\xb5\x2a\xa6\x9a\x21\x57\x1a\xe5\xab\x7f\x3a\xb6\x2a\x54\x51\x3a\x9c\xe3\x39\x7c\xe1\x0f\x9a\xad\x13\xa6\xff\xd5\x91\xa4\x32\x97\xe3\xd3\x39\x86\x47\xa0\xd0\x6c\xd0\xb1\x69\xfd\x5b\xae\x02\x23\xf4\x22\x0c\x5b\x11\xde\xa4\xb5\x31\x6c\x1d\x29\x2b\x29\xf6\xc2\xbd\x57\x62\x49\xb6\x94\xae\xf5\x32\xd3\x94\x95\xd2\xf9\x45\xed\x55\x78\xf0\xab\x34\x3f\xee\x56\x96\xb8\x92\x49\xd0\x9e\x92\xd2\x03\xf8\xc7\xc5\xd4\x51\x0b\xa4\xd5\xd6\xab\x85\xfc\x57\x2c\x52\x67\x11\xf8\xfa\x15\x83\x09\x94\x3f\x51\x30\x29\x03\xcd\x51\xdb\x6a\x5d\xae\xc9\x3b\xca\x91\x61\xfe\xff\x42\x6f\x8f\x9e\xaa\xe4\xb4\x01\xce\xe7\xa4\x42\x31\x79\x02\x3e\xbe\x79\xff\xf0\xc2\x3e\xaa\x55\xe7\xaf\x6a\x4a\xdc\xfe\xc5\x01\x27\xf7\xea\xd2\xff\x91\x25\x4c\x4a\xef\x58\x30\x69\xf5\x71\x16\x89\xdb\x77\x7e\x79\x1e\xcf\xc1\xcf\x33\x79\x9f\xde\x1e\xb6\x8e\x3e\xa8\xda\x78\x36\x70\x65\xd9\xa7\xfa\xdf\x01\x00\x00\xff\xff\xf6\xa8\xa1\xb9\xb0\x08\x00\x00") func _4byte_tracerJsBytes() ([]byte, error) { return bindataRead( @@ -94,6 +95,26 @@ func _4byte_tracerJs() (*asset, error) { } info := bindataFileInfo{name: "4byte_tracer.js", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} + a := &asset{bytes: bytes, info: info, digest: [32]uint8{0x6b, 0xa8, 0x46, 0xa2, 0x3a, 0x2b, 0xaa, 0xb9, 0xb9, 0xba, 0xe2, 0x22, 0x10, 0xe, 0xe7, 0x4c, 0x24, 0xfc, 0x4c, 0x85, 0xeb, 0x96, 0x48, 0xe8, 0x7f, 0xc8, 0xe0, 0xd0, 0xd, 0x26, 0xa1, 0xb2}} + return a, nil +} + +var __4byte_tracer_legacyJs = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x94\x56\x5b\x6f\xdb\x4a\x0e\x7e\xb6\x7f\x05\xd7\x2f\xb5\x51\x59\x8e\x2f\x89\x2f\xd9\x16\xf0\xe6\xa4\x6d\x80\x9c\x24\x88\xdd\x3d\x28\x16\xfb\x30\x9e\xa1\xac\xd9\xc8\x33\xc2\x0c\xe5\x4b\x73\xf2\xdf\x17\x1c\x49\x89\x93\xd3\x62\xbb\x4f\x96\x47\xc3\x8f\x1f\xc9\x8f\xa4\x7a\x3d\xb8\xb0\xf9\xc1\xe9\x75\x4a\x30\x38\xe9\x8f\x61\x99\x22\xac\x6d\x17\x29\x45\x87\xc5\x06\xe6\x05\xa5\xd6\xf9\x66\xaf\x07\xcb\x54\x7b\x48\x74\x86\xa0\x3d\xe4\xc2\x11\xd8\x04\xe8\xcd\xfd\x4c\xaf\x9c\x70\x87\xb8\xd9\xeb\x95\x36\x3f\x7c\xcd\x08\x89\x43\x04\x6f\x13\xda\x09\x87\x33\x38\xd8\x02\xa4\x30\xe0\x50\x69\x4f\x4e\xaf\x0a\x42\xd0\x04\xc2\xa8\x9e\x75\xb0\xb1\x4a\x27\x07\x86\xd4\x04\x85\x51\xe8\x82\x6b\x42\xb7\xf1\x35\x8f\xcf\x37\x5f\xe1\x1a\xbd\x47\x07\x9f\xd1\xa0\x13\x19\xdc\x15\xab\x4c\x4b\xb8\xd6\x12\x8d\x47\x10\x1e\x72\x3e\xf1\x29\x2a\x58\x05\x38\x36\xfc\xc4\x54\x16\x15\x15\xf8\x64\x0b\xa3\x04\x69\x6b\x22\x40\xcd\xcc\x61\x8b\xce\x6b\x6b\x60\x58\xbb\xaa\x00\x23\xb0\x8e\x41\xda\x82\x38\x00\x07\x36\x67\xbb\x0e\x08\x73\x80\x4c\xd0\x8b\xe9\x2f\x24\xe4\x25\x6e\x05\xda\x04\x37\xa9\xcd\x11\x28\x15\xc4\x51\xef\x74\x96\xc1\x0a\xa1\xf0\x98\x14\x59\xc4\x68\xab\x82\xe0\x8f\xab\xe5\x97\xdb\xaf\x4b\x98\xdf\x7c\x83\x3f\xe6\xf7\xf7\xf3\x9b\xe5\xb7\x73\xd8\x69\x4a\x6d\x41\x80\x5b\x2c\xa1\xf4\x26\xcf\x34\x2a\xd8\x09\xe7\x84\xa1\x03\xd8\x84\x11\x7e\xbf\xbc\xbf\xf8\x32\xbf\x59\xce\xff\x71\x75\x7d\xb5\xfc\x06\xd6\xc1\xa7\xab\xe5\xcd\xe5\x62\x01\x9f\x6e\xef\x61\x0e\x77\xf3\xfb\xe5\xd5\xc5\xd7\xeb\xf9\x3d\xdc\x7d\xbd\xbf\xbb\x5d\x5c\xc6\xb0\x40\x66\x85\x6c\xff\xbf\x73\x9e\x84\xea\x39\x04\x85\x24\x74\xe6\xeb\x4c\x7c\xb3\x05\xf8\xd4\x16\x99\x82\x54\x6c\x11\x1c\x4a\xd4\x5b\x54\x20\x40\xda\xfc\xf0\xcb\x45\x65\x2c\x91\x59\xb3\x0e\x31\xff\x54\x90\x70\x95\x80\xb1\x14\x81\x47\x84\xbf\xa7\x44\xf9\xac\xd7\xdb\xed\x76\xf1\xda\x14\xb1\x75\xeb\x5e\x56\xc2\xf9\xde\xc7\xb8\xc9\x98\xa3\xd5\x81\x70\xe9\x84\x44\x07\x1e\x85\x93\x29\xfa\x10\x4c\x78\xd1\xd5\x0a\x0d\xe9\x44\xa3\xf3\x11\x8b\x14\xa4\xcd\x32\x94\xe4\x99\xc1\x26\x5c\xcc\xad\xa7\x6e\xee\xac\x44\xef\xb5\x59\x73\xe0\x70\x45\xaf\x2e\xc2\x06\x29\xb5\xca\xc3\x11\xdc\xdb\x68\xbc\xfe\x8e\x75\x36\x7c\x91\x97\x65\x54\x82\x44\x04\xde\x86\xe8\xc1\x21\xcb\x0c\x15\x78\xbd\x36\x82\x0a\x87\xa1\x97\x56\x08\x1b\x41\x92\xc5\x2e\xd6\x42\x1b\x4f\x7f\x01\x64\x9c\xba\x22\x97\x7b\xb1\xc9\x33\x9c\xf1\x33\xc0\x47\x50\xb8\x2a\xd6\x31\x71\x0a\x96\x4e\x18\x2f\x24\x8b\xbb\x0d\xad\x93\xfd\xa0\x3f\xc2\xd3\xe9\x18\x87\xa7\x4a\x9c\x4c\x86\x67\xd3\x41\x72\x3a\x9c\x9c\xf5\x47\x7d\x3c\x9b\x26\xa3\x31\x4e\xc7\xc3\xd5\x40\x9e\x9e\xe1\x58\x4c\x4e\xc6\xc3\x55\x1f\xc5\xc9\x24\x51\xe3\xd3\x71\x1f\xa7\x0a\x5b\x11\x3c\x06\x60\x37\x83\xd6\x51\xa6\x5b\x4f\x9d\xd2\xfb\x63\xf9\x03\x70\xb2\x1f\x8c\x95\x1c\x4c\xc7\xd8\xed\x0f\x26\x33\xe8\x47\x2f\x6f\x86\x13\x29\x47\x93\x61\xbf\x7b\x32\x83\xc1\xd1\xf9\xe9\x60\x94\x0c\x27\x93\x69\x77\x7a\xf6\xda\x40\xa8\xe4\x74\x9a\x4c\xa7\xdd\xc1\xe4\x0d\x94\x1c\x4c\xfa\xaa\x3f\x45\x86\xea\x97\xc7\x4f\xcd\xc7\x66\x83\x07\x8e\xf2\x20\xd6\x6b\x87\x6b\x41\x58\x56\x2d\x30\x0e\x2f\x12\x1e\x16\x71\xb3\xc1\xcf\x33\x78\x7c\x8a\x9a\xc1\x46\x8a\x2c\x5b\x1e\x72\x56\x35\x15\xce\x78\x78\x97\x88\xcc\xe3\xbb\xa0\x0b\x63\x4d\x97\x2f\x78\x1e\x1f\x01\x2f\x47\x7c\xe8\x6a\xa3\x70\x1f\x2e\xf0\x51\xa2\x9d\x27\x1e\xb3\x62\x13\x10\x45\xc2\xd3\xe4\xdd\x56\x64\x05\xbe\x8b\x40\xc7\x18\xc3\x06\x37\x5c\x54\xe1\x28\x6e\x36\x6a\x97\x33\x48\x0a\x53\x56\xca\xe6\x9e\x5c\xe7\xb1\xd9\x68\xf8\x9d\x26\x99\x1e\x1d\x48\xe1\x11\x5a\x17\xf3\xeb\xeb\xd6\x0c\x5e\xfe\x5c\xdc\xfe\x76\xd9\x9a\x35\x1b\x0d\x76\xb9\x16\x2c\x6d\xa5\x5c\x04\x5b\x91\x45\xa5\xbb\xea\xc7\x7f\x0f\x0f\xb6\xa0\xfa\xd7\x7f\x67\xb3\x32\x5e\x18\x9e\x43\xaf\x07\x9e\x84\x7c\x80\x9c\x1c\x90\x2d\xcd\x9a\xcf\xae\x7f\xbb\xbc\xbe\xfc\x3c\x5f\x5e\xbe\xa2\xb0\x58\xce\x97\x57\x17\xe5\xd1\x5f\x49\xfc\x1f\xfe\x07\x3f\xf3\xdf\x68\x3c\x35\x9f\x6f\x85\x9a\x9c\x37\x1b\x75\xd5\x3c\xf1\x9c\xf2\x3c\x8d\xc2\x18\xd1\x3c\x3c\xb9\x2c\x55\x6b\x86\x3e\xe7\x8e\xe1\x0e\x8a\x9b\x8d\x70\xff\x28\xdf\x5a\x45\xa1\xb9\x42\x86\xb7\xc2\xc1\x03\x1e\xe0\x03\xb4\x5a\xf0\x1e\xc8\x7e\xc1\x7d\x5b\xab\x0e\xbc\x87\x56\x97\x4f\xf8\xe6\x79\xb3\xd1\xa0\x54\xfb\x58\x2b\xff\xaf\x07\x3c\xfc\x1b\x3e\xc0\xeb\xff\xef\xa1\x0f\x7f\xfe\x09\xfd\x57\x34\x31\xe7\x85\xa1\xcd\xd6\x3e\xa0\x0a\x92\xe1\x01\x70\x00\x9b\x4b\xab\xaa\x8d\xc1\x11\xfc\xf3\x77\xc0\x3d\xca\x82\xd0\x07\xba\x98\x1f\xb1\xcd\xec\x3a\x02\xb5\xea\x00\xb3\xed\xf5\x60\xf1\xa0\xf3\xb0\xb8\x4a\x14\x5f\xc2\xf0\x46\x34\x96\x40\x1b\x42\x67\x44\x16\xa4\xed\xab\xf8\x24\xd5\x7c\x6b\xf5\x31\x6a\x6c\xf3\x98\xec\x82\x9c\x36\xeb\x76\xa7\xc3\x31\xea\x04\xda\x7f\x93\x54\xfa\xaa\xd2\x7f\x5e\x15\xe3\xd8\x75\xee\xb0\x2b\xed\x26\x0f\x5f\x19\x66\x6b\x65\xd8\xc3\x3e\x02\x4a\x2d\xef\x6f\x87\xf0\x9f\xc2\x13\x24\xc2\xc8\x67\xa2\x15\xbe\xf6\x77\x0e\x2b\x63\xd5\x26\x3b\x57\xca\xa1\xf7\x81\x51\x50\x42\xcc\x6d\xd6\xee\x77\x5e\xc8\xf5\xcf\x3a\x9d\xce\xcf\x48\x7d\x16\x61\xf7\xbf\x0a\xbc\x5e\x62\x55\xfc\xda\x2c\xbe\xc3\x07\x78\xe3\x41\x12\x57\xad\x13\x87\x5e\xbd\x4d\xda\xcf\x19\x08\xd7\x3f\x7e\x80\x51\xe5\xb2\x84\xb8\x4d\x92\x1f\x61\xbc\xb1\x2f\x65\x12\x14\x17\x22\x62\xd1\xbb\x43\xec\x79\x6d\xb5\x03\x48\x54\x61\xbd\x87\x51\x27\x0a\xd4\xba\xa3\x4e\x15\x4f\x2d\x9d\x44\x14\x19\x1d\x6b\x67\x97\x56\xdf\x07\x42\x52\x21\xb2\x4a\x2e\xfc\xad\x63\x13\x10\xa6\x56\x54\x52\x6e\xee\x46\xb0\xff\xa1\x86\xa0\x76\xe1\xd0\xff\xc8\x07\x27\x8f\xfd\xd4\xe2\x0a\x3b\x7f\x85\xdc\x60\x84\x4e\xf0\x47\x8f\xdd\x56\x2d\x56\x0d\xcd\x00\x57\xce\x42\xce\x7f\x05\x5c\x2d\x2e\xde\x1e\x61\xa9\x36\xca\xf3\x23\x52\x92\xf6\x2f\xa2\xae\x9b\xd9\x16\x3c\x3f\xb9\x86\xdc\xc0\x20\x32\x6f\xab\xaa\x48\xda\xc7\xda\xe4\x05\xc5\x19\x9a\x35\xa5\xc7\x15\x3a\x4a\x7a\x99\xe9\xe7\xcb\x11\x9c\x44\x21\xd1\x6f\xcd\xbb\xa3\xce\xeb\x29\x53\xf7\x73\xd9\xc1\x4f\xcd\xff\x06\x00\x00\xff\xff\x8e\xc8\x27\x72\x75\x0b\x00\x00") + +func _4byte_tracer_legacyJsBytes() ([]byte, error) { + return bindataRead( + __4byte_tracer_legacyJs, + "4byte_tracer_legacy.js", + ) +} + +func _4byte_tracer_legacyJs() (*asset, error) { + bytes, err := _4byte_tracer_legacyJsBytes() + if err != nil { + return nil, err + } + + info := bindataFileInfo{name: "4byte_tracer_legacy.js", size: 0, mode: os.FileMode(0), modTime: time.Unix(0, 0)} a := &asset{bytes: bytes, info: info, digest: [32]uint8{0xb4, 0xc5, 0x48, 0x2d, 0xd9, 0x43, 0x95, 0x93, 0x3b, 0x93, 0x2c, 0x47, 0x8c, 0x84, 0x32, 0x3c, 0x8b, 0x2e, 0xf3, 0x72, 0xc4, 0x57, 0xe6, 0x3a, 0xb3, 0xdf, 0x1d, 0xbf, 0x45, 0x3, 0xfc, 0xa}} return a, nil } @@ -369,16 +390,17 @@ func AssetNames() []string { // _bindata is a table, holding each asset generator, mapped to its name. var _bindata = map[string]func() (*asset, error){ - "4byte_tracer.js": _4byte_tracerJs, - "bigram_tracer.js": bigram_tracerJs, - "call_tracer.js": call_tracerJs, - "call_tracer_legacy.js": call_tracer_legacyJs, - "evmdis_tracer.js": evmdis_tracerJs, - "noop_tracer.js": noop_tracerJs, - "opcount_tracer.js": opcount_tracerJs, - "prestate_tracer.js": prestate_tracerJs, - "trigram_tracer.js": trigram_tracerJs, - "unigram_tracer.js": unigram_tracerJs, + "4byte_tracer.js": _4byte_tracerJs, + "4byte_tracer_legacy.js": _4byte_tracer_legacyJs, + "bigram_tracer.js": bigram_tracerJs, + "call_tracer.js": call_tracerJs, + "call_tracer_legacy.js": call_tracer_legacyJs, + "evmdis_tracer.js": evmdis_tracerJs, + "noop_tracer.js": noop_tracerJs, + "opcount_tracer.js": opcount_tracerJs, + "prestate_tracer.js": prestate_tracerJs, + "trigram_tracer.js": trigram_tracerJs, + "unigram_tracer.js": unigram_tracerJs, } // AssetDebug is true if the assets were built with the debug flag enabled. @@ -425,16 +447,17 @@ type bintree struct { } var _bintree = &bintree{nil, map[string]*bintree{ - "4byte_tracer.js": {_4byte_tracerJs, map[string]*bintree{}}, - "bigram_tracer.js": {bigram_tracerJs, map[string]*bintree{}}, - "call_tracer.js": {call_tracerJs, map[string]*bintree{}}, - "call_tracer_legacy.js": {call_tracer_legacyJs, map[string]*bintree{}}, - "evmdis_tracer.js": {evmdis_tracerJs, map[string]*bintree{}}, - "noop_tracer.js": {noop_tracerJs, map[string]*bintree{}}, - "opcount_tracer.js": {opcount_tracerJs, map[string]*bintree{}}, - "prestate_tracer.js": {prestate_tracerJs, map[string]*bintree{}}, - "trigram_tracer.js": {trigram_tracerJs, map[string]*bintree{}}, - "unigram_tracer.js": {unigram_tracerJs, map[string]*bintree{}}, + "4byte_tracer.js": {_4byte_tracerJs, map[string]*bintree{}}, + "4byte_tracer_legacy.js": {_4byte_tracer_legacyJs, map[string]*bintree{}}, + "bigram_tracer.js": {bigram_tracerJs, map[string]*bintree{}}, + "call_tracer.js": {call_tracerJs, map[string]*bintree{}}, + "call_tracer_legacy.js": {call_tracer_legacyJs, map[string]*bintree{}}, + "evmdis_tracer.js": {evmdis_tracerJs, map[string]*bintree{}}, + "noop_tracer.js": {noop_tracerJs, map[string]*bintree{}}, + "opcount_tracer.js": {opcount_tracerJs, map[string]*bintree{}}, + "prestate_tracer.js": {prestate_tracerJs, map[string]*bintree{}}, + "trigram_tracer.js": {trigram_tracerJs, map[string]*bintree{}}, + "unigram_tracer.js": {unigram_tracerJs, map[string]*bintree{}}, }} // RestoreAsset restores an asset under the given directory. From 443afc975c7a4016bf9ed1b057b8cc186cb089e1 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Tue, 28 Sep 2021 10:48:07 +0200 Subject: [PATCH 65/73] core/state: move state account to core/types + abstracted "write account to trie" (#23567) * core/state: abstracted "write account to trie" method * fix appveyor build * Apply suggestions from code review Co-authored-by: Martin Holst Swende * review feedback * core/state/accounts: move Account to core/types * core/types: rename Account -> StateAccount * core/state: restore EncodeRLP for stateObject * core/types: add the missing file * more review feedback * more review feedback Co-authored-by: Martin Holst Swende Co-authored-by: Felix Lange --- cmd/geth/snapshot.go | 5 +++-- core/state/database.go | 4 ++++ core/state/dump.go | 3 ++- core/state/iterator.go | 3 ++- core/state/pruner/pruner.go | 3 +-- core/state/state_object.go | 16 ++++------------ core/state/statedb.go | 17 ++++++----------- core/state/sync.go | 3 ++- core/state/sync_test.go | 3 ++- core/types/state_account.go | 32 ++++++++++++++++++++++++++++++++ eth/protocols/snap/handler.go | 4 ++-- eth/protocols/snap/sync.go | 11 ++++++----- eth/protocols/snap/sync_test.go | 12 ++++++------ les/server_handler.go | 16 ++++++++-------- light/trie.go | 12 ++++++++++++ trie/secure_trie.go | 17 +++++++++++++++++ trie/trie.go | 10 ++++++++++ trie/trie_test.go | 12 +++--------- 18 files changed, 122 insertions(+), 61 deletions(-) create mode 100644 core/types/state_account.go diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index 9976db49f..d3903e0af 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -29,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/state/pruner" "github.com/ethereum/go-ethereum/core/state/snapshot" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" @@ -287,7 +288,7 @@ func traverseState(ctx *cli.Context) error { accIter := trie.NewIterator(t.NodeIterator(nil)) for accIter.Next() { accounts += 1 - var acc state.Account + var acc types.StateAccount if err := rlp.DecodeBytes(accIter.Value, &acc); err != nil { log.Error("Invalid account encountered during traversal", "err", err) return err @@ -393,7 +394,7 @@ func traverseRawState(ctx *cli.Context) error { // dig into the storage trie further. if accIter.Leaf() { accounts += 1 - var acc state.Account + var acc types.StateAccount if err := rlp.DecodeBytes(accIter.LeafBlob(), &acc); err != nil { log.Error("Invalid account encountered during traversal", "err", err) return errors.New("invalid account") diff --git a/core/state/database.go b/core/state/database.go index e25d059c0..bbcd2358e 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -23,6 +23,7 @@ import ( "github.com/VictoriaMetrics/fastcache" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/trie" lru "github.com/hashicorp/golang-lru" @@ -70,6 +71,9 @@ type Trie interface { // trie.MissingNodeError is returned. TryGet(key []byte) ([]byte, error) + // TryUpdateAccount abstract an account write in the trie. + TryUpdateAccount(key []byte, account *types.StateAccount) error + // TryUpdate associates key with value in the trie. If value has length zero, any // existing value is deleted from the trie. The value bytes must not be modified // by the caller while they are stored in the trie. If a node was not found in the diff --git a/core/state/dump.go b/core/state/dump.go index 00faa4ed6..bfcc03543 100644 --- a/core/state/dump.go +++ b/core/state/dump.go @@ -23,6 +23,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" @@ -140,7 +141,7 @@ func (s *StateDB) DumpToCollector(c DumpCollector, conf *DumpConfig) (nextKey [] it := trie.NewIterator(s.trie.NodeIterator(conf.Start)) for it.Next() { - var data Account + var data types.StateAccount if err := rlp.DecodeBytes(it.Value, &data); err != nil { panic(err) } diff --git a/core/state/iterator.go b/core/state/iterator.go index 6a5c73d3d..611df5243 100644 --- a/core/state/iterator.go +++ b/core/state/iterator.go @@ -21,6 +21,7 @@ import ( "fmt" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" ) @@ -104,7 +105,7 @@ func (it *NodeIterator) step() error { return nil } // Otherwise we've reached an account node, initiate data iteration - var account Account + var account types.StateAccount if err := rlp.Decode(bytes.NewReader(it.stateIt.LeafBlob()), &account); err != nil { return err } diff --git a/core/state/pruner/pruner.go b/core/state/pruner/pruner.go index 4d6e41551..37772ca35 100644 --- a/core/state/pruner/pruner.go +++ b/core/state/pruner/pruner.go @@ -29,7 +29,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/state/snapshot" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" @@ -426,7 +425,7 @@ func extractGenesis(db ethdb.Database, stateBloom *stateBloom) error { // If it's a leaf node, yes we are touching an account, // dig into the storage trie further. if accIter.Leaf() { - var acc state.Account + var acc types.StateAccount if err := rlp.DecodeBytes(accIter.LeafBlob(), &acc); err != nil { return err } diff --git a/core/state/state_object.go b/core/state/state_object.go index 7b366d64c..73e9cb78e 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -24,6 +24,7 @@ import ( "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/rlp" @@ -65,7 +66,7 @@ func (s Storage) Copy() Storage { type stateObject struct { address common.Address addrHash common.Hash // hash of ethereum address of the account - data Account + data types.StateAccount db *StateDB // DB error. @@ -97,17 +98,8 @@ func (s *stateObject) empty() bool { return s.data.Nonce == 0 && s.data.Balance.Sign() == 0 && bytes.Equal(s.data.CodeHash, emptyCodeHash) } -// Account is the Ethereum consensus representation of accounts. -// These objects are stored in the main account trie. -type Account struct { - Nonce uint64 - Balance *big.Int - Root common.Hash // merkle root of the storage trie - CodeHash []byte -} - // newObject creates a state object. -func newObject(db *StateDB, address common.Address, data Account) *stateObject { +func newObject(db *StateDB, address common.Address, data types.StateAccount) *stateObject { if data.Balance == nil { data.Balance = new(big.Int) } @@ -130,7 +122,7 @@ func newObject(db *StateDB, address common.Address, data Account) *stateObject { // EncodeRLP implements rlp.Encoder. func (s *stateObject) EncodeRLP(w io.Writer) error { - return rlp.Encode(w, s.data) + return rlp.Encode(w, &s.data) } // setError remembers the first non-nil error it is called with. diff --git a/core/state/statedb.go b/core/state/statedb.go index a6d47179b..e3541339e 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -460,12 +460,7 @@ func (s *StateDB) updateStateObject(obj *stateObject) { } // Encode the account and update the account trie addr := obj.Address() - - data, err := rlp.EncodeToBytes(obj) - if err != nil { - panic(fmt.Errorf("can't encode object at %x: %v", addr[:], err)) - } - if err = s.trie.TryUpdate(addr[:], data); err != nil { + if err := s.trie.TryUpdateAccount(addr[:], &obj.data); err != nil { s.setError(fmt.Errorf("updateStateObject (%x) error: %v", addr[:], err)) } @@ -512,7 +507,7 @@ func (s *StateDB) getDeletedStateObject(addr common.Address) *stateObject { } // If no live objects are available, attempt to use snapshots var ( - data *Account + data *types.StateAccount err error ) if s.snap != nil { @@ -524,7 +519,7 @@ func (s *StateDB) getDeletedStateObject(addr common.Address) *stateObject { if acc == nil { return nil } - data = &Account{ + data = &types.StateAccount{ Nonce: acc.Nonce, Balance: acc.Balance, CodeHash: acc.CodeHash, @@ -551,7 +546,7 @@ func (s *StateDB) getDeletedStateObject(addr common.Address) *stateObject { if len(enc) == 0 { return nil } - data = new(Account) + data = new(types.StateAccount) if err := rlp.DecodeBytes(enc, data); err != nil { log.Error("Failed to decode state object", "addr", addr, "err", err) return nil @@ -588,7 +583,7 @@ func (s *StateDB) createObject(addr common.Address) (newobj, prev *stateObject) s.snapDestructs[prev.addrHash] = struct{}{} } } - newobj = newObject(s, addr, Account{}) + newobj = newObject(s, addr, types.StateAccount{}) if prev == nil { s.journal.append(createObjectChange{account: &addr}) } else { @@ -942,7 +937,7 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { } // The onleaf func is called _serially_, so we can reuse the same account // for unmarshalling every time. - var account Account + var account types.StateAccount root, accountCommitted, err := s.trie.Commit(func(_ [][]byte, _ []byte, leaf []byte, parent common.Hash) error { if err := rlp.DecodeBytes(leaf, &account); err != nil { return nil diff --git a/core/state/sync.go b/core/state/sync.go index 7a5852fb1..734961d9c 100644 --- a/core/state/sync.go +++ b/core/state/sync.go @@ -20,6 +20,7 @@ import ( "bytes" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" @@ -43,7 +44,7 @@ func NewStateSync(root common.Hash, database ethdb.KeyValueReader, bloom *trie.S return err } } - var obj Account + var obj types.StateAccount if err := rlp.Decode(bytes.NewReader(leaf), &obj); err != nil { return err } diff --git a/core/state/sync_test.go b/core/state/sync_test.go index a13fcf56a..beb8fcfd9 100644 --- a/core/state/sync_test.go +++ b/core/state/sync_test.go @@ -23,6 +23,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/ethdb/memorydb" @@ -203,7 +204,7 @@ func testIterativeStateSync(t *testing.T, count int, commit bool, bypath bool) { } results[len(hashQueue)+i] = trie.SyncResult{Hash: crypto.Keccak256Hash(data), Data: data} } else { - var acc Account + var acc types.StateAccount if err := rlp.DecodeBytes(srcTrie.Get(path[0]), &acc); err != nil { t.Fatalf("failed to decode account on path %x: %v", path, err) } diff --git a/core/types/state_account.go b/core/types/state_account.go new file mode 100644 index 000000000..68804bf31 --- /dev/null +++ b/core/types/state_account.go @@ -0,0 +1,32 @@ +// Copyright 2021 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package types + +import ( + "math/big" + + "github.com/ethereum/go-ethereum/common" +) + +// StateAccount is the Ethereum consensus representation of accounts. +// These objects are stored in the main account trie. +type StateAccount struct { + Nonce uint64 + Balance *big.Int + Root common.Hash // merkle root of the storage trie + CodeHash []byte +} diff --git a/eth/protocols/snap/handler.go b/eth/protocols/snap/handler.go index 3d668a2eb..6f345716b 100644 --- a/eth/protocols/snap/handler.go +++ b/eth/protocols/snap/handler.go @@ -23,7 +23,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/light" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" @@ -319,7 +319,7 @@ func handleMessage(backend Backend, peer *Peer) error { if err != nil { return p2p.Send(peer.rw, StorageRangesMsg, &StorageRangesPacket{ID: req.ID}) } - var acc state.Account + var acc types.StateAccount if err := rlp.DecodeBytes(accTrie.Get(account[:]), &acc); err != nil { return p2p.Send(peer.rw, StorageRangesMsg, &StorageRangesPacket{ID: req.ID}) } diff --git a/eth/protocols/snap/sync.go b/eth/protocols/snap/sync.go index 646df0388..9ef9d7571 100644 --- a/eth/protocols/snap/sync.go +++ b/eth/protocols/snap/sync.go @@ -32,6 +32,7 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/state/snapshot" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" @@ -125,8 +126,8 @@ type accountRequest struct { type accountResponse struct { task *accountTask // Task which this request is filling - hashes []common.Hash // Account hashes in the returned range - accounts []*state.Account // Expanded accounts in the returned range + hashes []common.Hash // Account hashes in the returned range + accounts []*types.StateAccount // Expanded accounts in the returned range cont bool // Whether the account range has a continuation } @@ -2274,9 +2275,9 @@ func (s *Syncer) OnAccounts(peer SyncPeer, id uint64, hashes []common.Hash, acco s.scheduleRevertAccountRequest(req) return err } - accs := make([]*state.Account, len(accounts)) + accs := make([]*types.StateAccount, len(accounts)) for i, account := range accounts { - acc := new(state.Account) + acc := new(types.StateAccount) if err := rlp.DecodeBytes(account, acc); err != nil { panic(err) // We created these blobs, we must be able to decode them } @@ -2740,7 +2741,7 @@ func (s *Syncer) onHealByteCodes(peer SyncPeer, id uint64, bytecodes [][]byte) e // Note it's not concurrent safe, please handle the concurrent issue outside. func (s *Syncer) onHealState(paths [][]byte, value []byte) error { if len(paths) == 1 { - var account state.Account + var account types.StateAccount if err := rlp.DecodeBytes(value, &account); err != nil { return nil } diff --git a/eth/protocols/snap/sync_test.go b/eth/protocols/snap/sync_test.go index 023fc8ee0..47ab1f026 100644 --- a/eth/protocols/snap/sync_test.go +++ b/eth/protocols/snap/sync_test.go @@ -29,7 +29,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/light" @@ -1349,7 +1349,7 @@ func makeAccountTrieNoStorage(n int) (*trie.Trie, entrySlice) { accTrie, _ := trie.New(common.Hash{}, db) var entries entrySlice for i := uint64(1); i <= uint64(n); i++ { - value, _ := rlp.EncodeToBytes(state.Account{ + value, _ := rlp.EncodeToBytes(types.StateAccount{ Nonce: i, Balance: big.NewInt(int64(i)), Root: emptyRoot, @@ -1394,7 +1394,7 @@ func makeBoundaryAccountTrie(n int) (*trie.Trie, entrySlice) { } // Fill boundary accounts for i := 0; i < len(boundaries); i++ { - value, _ := rlp.EncodeToBytes(state.Account{ + value, _ := rlp.EncodeToBytes(types.StateAccount{ Nonce: uint64(0), Balance: big.NewInt(int64(i)), Root: emptyRoot, @@ -1406,7 +1406,7 @@ func makeBoundaryAccountTrie(n int) (*trie.Trie, entrySlice) { } // Fill other accounts if required for i := uint64(1); i <= uint64(n); i++ { - value, _ := rlp.EncodeToBytes(state.Account{ + value, _ := rlp.EncodeToBytes(types.StateAccount{ Nonce: i, Balance: big.NewInt(int64(i)), Root: emptyRoot, @@ -1442,7 +1442,7 @@ func makeAccountTrieWithStorageWithUniqueStorage(accounts, slots int, code bool) stTrie, stEntries := makeStorageTrieWithSeed(uint64(slots), i, db) stRoot := stTrie.Hash() stTrie.Commit(nil) - value, _ := rlp.EncodeToBytes(state.Account{ + value, _ := rlp.EncodeToBytes(types.StateAccount{ Nonce: i, Balance: big.NewInt(int64(i)), Root: stRoot, @@ -1489,7 +1489,7 @@ func makeAccountTrieWithStorage(accounts, slots int, code, boundary bool) (*trie if code { codehash = getCodeHash(i) } - value, _ := rlp.EncodeToBytes(state.Account{ + value, _ := rlp.EncodeToBytes(types.StateAccount{ Nonce: i, Balance: big.NewInt(int64(i)), Root: stRoot, diff --git a/les/server_handler.go b/les/server_handler.go index fa20fd7b3..f36a87a51 100644 --- a/les/server_handler.go +++ b/les/server_handler.go @@ -27,7 +27,7 @@ import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/forkid" "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/les/flowcontrol" "github.com/ethereum/go-ethereum/light" @@ -358,20 +358,20 @@ func (h *serverHandler) AddTxsSync() bool { } // getAccount retrieves an account from the state based on root. -func getAccount(triedb *trie.Database, root, hash common.Hash) (state.Account, error) { +func getAccount(triedb *trie.Database, root, hash common.Hash) (types.StateAccount, error) { trie, err := trie.New(root, triedb) if err != nil { - return state.Account{}, err + return types.StateAccount{}, err } blob, err := trie.TryGet(hash[:]) if err != nil { - return state.Account{}, err + return types.StateAccount{}, err } - var account state.Account - if err = rlp.DecodeBytes(blob, &account); err != nil { - return state.Account{}, err + var acc types.StateAccount + if err = rlp.DecodeBytes(blob, &acc); err != nil { + return types.StateAccount{}, err } - return account, nil + return acc, nil } // getHelperTrie returns the post-processed trie root for the given trie ID and section index diff --git a/light/trie.go b/light/trie.go index 39e928bbe..4ab6f4ace 100644 --- a/light/trie.go +++ b/light/trie.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" ) @@ -111,6 +112,17 @@ func (t *odrTrie) TryGet(key []byte) ([]byte, error) { return res, err } +func (t *odrTrie) TryUpdateAccount(key []byte, acc *types.StateAccount) error { + key = crypto.Keccak256(key) + value, err := rlp.EncodeToBytes(acc) + if err != nil { + return fmt.Errorf("decoding error in account update: %w", err) + } + return t.do(key, func() error { + return t.trie.TryUpdate(key, value) + }) +} + func (t *odrTrie) TryUpdate(key, value []byte) error { key = crypto.Keccak256(key) return t.do(key, func() error { diff --git a/trie/secure_trie.go b/trie/secure_trie.go index 4d0050bc5..18be12d34 100644 --- a/trie/secure_trie.go +++ b/trie/secure_trie.go @@ -20,7 +20,9 @@ import ( "fmt" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rlp" ) // SecureTrie wraps a trie with key hashing. In a secure trie, all @@ -85,6 +87,21 @@ func (t *SecureTrie) TryGetNode(path []byte) ([]byte, int, error) { return t.trie.TryGetNode(path) } +// TryUpdate account will abstract the write of an account to the +// secure trie. +func (t *SecureTrie) TryUpdateAccount(key []byte, acc *types.StateAccount) error { + hk := t.hashKey(key) + data, err := rlp.EncodeToBytes(acc) + if err != nil { + return err + } + if err := t.trie.TryUpdate(hk, data); err != nil { + return err + } + t.getSecKeyCache()[string(hk)] = common.CopyBytes(key) + return nil +} + // Update associates key with value in the trie. Subsequent calls to // Get will return value. If value has length zero, any existing value // is deleted from the trie and calls to Get will return nil. diff --git a/trie/trie.go b/trie/trie.go index 7ea7efa83..915e6f87d 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -24,8 +24,10 @@ import ( "sync" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/rlp" ) var ( @@ -246,6 +248,14 @@ func (t *Trie) Update(key, value []byte) { } } +func (t *Trie) TryUpdateAccount(key []byte, acc *types.StateAccount) error { + data, err := rlp.EncodeToBytes(acc) + if err != nil { + return fmt.Errorf("can't encode object at %x: %w", key[:], err) + } + return t.TryUpdate(key, data) +} + // TryUpdate associates key with value in the trie. Subsequent calls to // Get will return value. If value has length zero, any existing value // is deleted from the trie and calls to Get will return nil. diff --git a/trie/trie_test.go b/trie/trie_test.go index ae1871e42..be0df8a54 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -32,6 +32,7 @@ import ( "github.com/davecgh/go-spew/spew" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/ethdb/leveldb" @@ -553,13 +554,6 @@ func BenchmarkHash(b *testing.B) { trie.Hash() } -type account struct { - Nonce uint64 - Balance *big.Int - Root common.Hash - Code []byte -} - // Benchmarks the trie Commit following a Hash. Since the trie caches the result of any operation, // we cannot use b.N as the number of hashing rouns, since all rounds apart from // the first one will be NOOP. As such, we'll use b.N as the number of account to @@ -568,7 +562,7 @@ func BenchmarkCommitAfterHash(b *testing.B) { b.Run("no-onleaf", func(b *testing.B) { benchmarkCommitAfterHash(b, nil) }) - var a account + var a types.StateAccount onleaf := func(paths [][]byte, hexpath []byte, leaf []byte, parent common.Hash) error { rlp.DecodeBytes(leaf, &a) return nil @@ -664,7 +658,7 @@ func makeAccounts(size int) (addresses [][20]byte, accounts [][]byte) { balanceBytes := make([]byte, numBytes) random.Read(balanceBytes) balance := new(big.Int).SetBytes(balanceBytes) - data, _ := rlp.EncodeToBytes(&account{nonce, balance, root, code}) + data, _ := rlp.EncodeToBytes(&types.StateAccount{Nonce: nonce, Balance: balance, Root: root, CodeHash: code}) accounts[i] = data } return addresses, accounts From ab2caaee1167a8bef9760d84b45716f1bddcc381 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 28 Sep 2021 12:53:11 +0200 Subject: [PATCH 66/73] eth/tracers: implement debug.intermediateRoots (#23594) This PR implements a new debug method, which I've talked briefly about to some other client developers. It allows the caller to obtain the intermediate state roots for a block (which might be either a canon block or a 'bad' block). --- eth/tracers/api.go | 75 ++++++++++++++++++++++++++++++++----- internal/web3ext/web3ext.go | 6 +++ 2 files changed, 71 insertions(+), 10 deletions(-) diff --git a/eth/tracers/api.go b/eth/tracers/api.go index 682659328..39b95264d 100644 --- a/eth/tracers/api.go +++ b/eth/tracers/api.go @@ -445,12 +445,11 @@ func (api *API) TraceBlockFromFile(ctx context.Context, file string, config *Tra // EVM against a block pulled from the pool of bad ones and returns them as a JSON // object. func (api *API) TraceBadBlock(ctx context.Context, hash common.Hash, config *TraceConfig) ([]*txTraceResult, error) { - for _, block := range rawdb.ReadAllBadBlocks(api.backend.ChainDb()) { - if block.Hash() == hash { - return api.traceBlock(ctx, block, config) - } + block := rawdb.ReadBadBlock(api.backend.ChainDb(), hash) + if block == nil { + return nil, fmt.Errorf("bad block %#x not found", hash) } - return nil, fmt.Errorf("bad block %#x not found", hash) + return api.traceBlock(ctx, block, config) } // StandardTraceBlockToFile dumps the structured logs created during the @@ -464,16 +463,72 @@ func (api *API) StandardTraceBlockToFile(ctx context.Context, hash common.Hash, return api.standardTraceBlockToFile(ctx, block, config) } +// IntermediateRoots executes a block (bad- or canon- or side-), and returns a list +// of intermediate roots: the stateroot after each transaction. +func (api *API) IntermediateRoots(ctx context.Context, hash common.Hash, config *TraceConfig) ([]common.Hash, error) { + block, _ := api.blockByHash(ctx, hash) + if block == nil { + // Check in the bad blocks + block = rawdb.ReadBadBlock(api.backend.ChainDb(), hash) + } + if block == nil { + return nil, fmt.Errorf("block %#x not found", hash) + } + if block.NumberU64() == 0 { + return nil, errors.New("genesis is not traceable") + } + parent, err := api.blockByNumberAndHash(ctx, rpc.BlockNumber(block.NumberU64()-1), block.ParentHash()) + if err != nil { + return nil, err + } + reexec := defaultTraceReexec + if config != nil && config.Reexec != nil { + reexec = *config.Reexec + } + statedb, err := api.backend.StateAtBlock(ctx, parent, reexec, nil, true) + if err != nil { + return nil, err + } + var ( + roots []common.Hash + signer = types.MakeSigner(api.backend.ChainConfig(), block.Number()) + chainConfig = api.backend.ChainConfig() + vmctx = core.NewEVMBlockContext(block.Header(), api.chainContext(ctx), nil) + deleteEmptyObjects = chainConfig.IsEIP158(block.Number()) + ) + for i, tx := range block.Transactions() { + var ( + msg, _ = tx.AsMessage(signer, block.BaseFee()) + txContext = core.NewEVMTxContext(msg) + vmenv = vm.NewEVM(vmctx, txContext, statedb, chainConfig, vm.Config{}) + ) + statedb.Prepare(tx.Hash(), i) + if _, err := core.ApplyMessage(vmenv, msg, new(core.GasPool).AddGas(msg.Gas())); err != nil { + log.Warn("Tracing intermediate roots did not complete", "txindex", i, "txhash", tx.Hash(), "err", err) + // We intentionally don't return the error here: if we do, then the RPC server will not + // return the roots. Most likely, the caller already knows that a certain transaction fails to + // be included, but still want the intermediate roots that led to that point. + // It may happen the tx_N causes an erroneous state, which in turn causes tx_N+M to not be + // executable. + // N.B: This should never happen while tracing canon blocks, only when tracing bad blocks. + return roots, nil + } + // calling IntermediateRoot will internally call Finalize on the state + // so any modifications are written to the trie + roots = append(roots, statedb.IntermediateRoot(deleteEmptyObjects)) + } + return roots, nil +} + // StandardTraceBadBlockToFile dumps the structured logs created during the // execution of EVM against a block pulled from the pool of bad ones to the // local file system and returns a list of files to the caller. func (api *API) StandardTraceBadBlockToFile(ctx context.Context, hash common.Hash, config *StdTraceConfig) ([]string, error) { - for _, block := range rawdb.ReadAllBadBlocks(api.backend.ChainDb()) { - if block.Hash() == hash { - return api.standardTraceBlockToFile(ctx, block, config) - } + block := rawdb.ReadBadBlock(api.backend.ChainDb(), hash) + if block == nil { + return nil, fmt.Errorf("bad block %#x not found", hash) } - return nil, fmt.Errorf("bad block %#x not found", hash) + return api.standardTraceBlockToFile(ctx, block, config) } // traceBlock configures a new tracer according to the provided configuration, and diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go index fe15cb050..ad459362a 100644 --- a/internal/web3ext/web3ext.go +++ b/internal/web3ext/web3ext.go @@ -391,6 +391,12 @@ web3._extend({ params: 2, inputFormatter: [null, null] }), + new web3._extend.Method({ + name: 'intermediateRoots', + call: 'debug_intermediateRoots', + params: 2, + inputFormatter: [null, null] + }), new web3._extend.Method({ name: 'standardTraceBlockToFile', call: 'debug_standardTraceBlockToFile', From 783e97ef1f51f4a94afbad6e07253bcdb78da2d7 Mon Sep 17 00:00:00 2001 From: Sina Mahmoodi <1591639+s1na@users.noreply.github.com> Date: Tue, 28 Sep 2021 12:54:49 +0200 Subject: [PATCH 67/73] core/rawdb: avoid unnecessary receipt processing for log filtering (#23147) * core/types: rm extranous check in test * core/rawdb: add lightweight types for block logs * core/rawdb,eth: use lightweight accessor for log filtering * core/rawdb: add bench for decoding into rlpLogs --- core/rawdb/accessors_chain.go | 81 +++++++++ core/rawdb/accessors_chain_test.go | 213 ++++++++++++++++++++++++ core/rawdb/testdata/stored_receipts.bin | Bin 0 -> 99991 bytes core/types/receipt_test.go | 3 - eth/api_backend.go | 13 +- 5 files changed, 301 insertions(+), 9 deletions(-) create mode 100644 core/rawdb/testdata/stored_receipts.bin diff --git a/core/rawdb/accessors_chain.go b/core/rawdb/accessors_chain.go index 58226fb04..ed1c71e20 100644 --- a/core/rawdb/accessors_chain.go +++ b/core/rawdb/accessors_chain.go @@ -19,6 +19,7 @@ package rawdb import ( "bytes" "encoding/binary" + "errors" "fmt" "math/big" "sort" @@ -663,6 +664,86 @@ func DeleteReceipts(db ethdb.KeyValueWriter, hash common.Hash, number uint64) { } } +// storedReceiptRLP is the storage encoding of a receipt. +// Re-definition in core/types/receipt.go. +type storedReceiptRLP struct { + PostStateOrStatus []byte + CumulativeGasUsed uint64 + Logs []*types.LogForStorage +} + +// ReceiptLogs is a barebone version of ReceiptForStorage which only keeps +// the list of logs. When decoding a stored receipt into this object we +// avoid creating the bloom filter. +type receiptLogs struct { + Logs []*types.Log +} + +// DecodeRLP implements rlp.Decoder. +func (r *receiptLogs) DecodeRLP(s *rlp.Stream) error { + var stored storedReceiptRLP + if err := s.Decode(&stored); err != nil { + return err + } + r.Logs = make([]*types.Log, len(stored.Logs)) + for i, log := range stored.Logs { + r.Logs[i] = (*types.Log)(log) + } + return nil +} + +// DeriveLogFields fills the logs in receiptLogs with information such as block number, txhash, etc. +func deriveLogFields(receipts []*receiptLogs, hash common.Hash, number uint64, txs types.Transactions) error { + logIndex := uint(0) + if len(txs) != len(receipts) { + return errors.New("transaction and receipt count mismatch") + } + for i := 0; i < len(receipts); i++ { + txHash := txs[i].Hash() + // The derived log fields can simply be set from the block and transaction + for j := 0; j < len(receipts[i].Logs); j++ { + receipts[i].Logs[j].BlockNumber = number + receipts[i].Logs[j].BlockHash = hash + receipts[i].Logs[j].TxHash = txHash + receipts[i].Logs[j].TxIndex = uint(i) + receipts[i].Logs[j].Index = logIndex + logIndex++ + } + } + return nil +} + +// ReadLogs retrieves the logs for all transactions in a block. The log fields +// are populated with metadata. In case the receipts or the block body +// are not found, a nil is returned. +func ReadLogs(db ethdb.Reader, hash common.Hash, number uint64) [][]*types.Log { + // Retrieve the flattened receipt slice + data := ReadReceiptsRLP(db, hash, number) + if len(data) == 0 { + return nil + } + receipts := []*receiptLogs{} + if err := rlp.DecodeBytes(data, &receipts); err != nil { + log.Error("Invalid receipt array RLP", "hash", hash, "err", err) + return nil + } + + body := ReadBody(db, hash, number) + if body == nil { + log.Error("Missing body but have receipt", "hash", hash, "number", number) + return nil + } + if err := deriveLogFields(receipts, hash, number, body.Transactions); err != nil { + log.Error("Failed to derive block receipts fields", "hash", hash, "number", number, "err", err) + return nil + } + logs := make([][]*types.Log, len(receipts)) + for i, receipt := range receipts { + logs[i] = receipt.Logs + } + return logs +} + // ReadBlock retrieves an entire block corresponding to the hash, assembling it // back from the stored header and body. If either the header or body could not // be retrieved nil is returned. diff --git a/core/rawdb/accessors_chain_test.go b/core/rawdb/accessors_chain_test.go index 58d7645e5..4b173c55e 100644 --- a/core/rawdb/accessors_chain_test.go +++ b/core/rawdb/accessors_chain_test.go @@ -670,3 +670,216 @@ func makeTestReceipts(n int, nPerBlock int) []types.Receipts { } return allReceipts } + +type fullLogRLP struct { + Address common.Address + Topics []common.Hash + Data []byte + BlockNumber uint64 + TxHash common.Hash + TxIndex uint + BlockHash common.Hash + Index uint +} + +func newFullLogRLP(l *types.Log) *fullLogRLP { + return &fullLogRLP{ + Address: l.Address, + Topics: l.Topics, + Data: l.Data, + BlockNumber: l.BlockNumber, + TxHash: l.TxHash, + TxIndex: l.TxIndex, + BlockHash: l.BlockHash, + Index: l.Index, + } +} + +// Tests that logs associated with a single block can be retrieved. +func TestReadLogs(t *testing.T) { + db := NewMemoryDatabase() + + // Create a live block since we need metadata to reconstruct the receipt + tx1 := types.NewTransaction(1, common.HexToAddress("0x1"), big.NewInt(1), 1, big.NewInt(1), nil) + tx2 := types.NewTransaction(2, common.HexToAddress("0x2"), big.NewInt(2), 2, big.NewInt(2), nil) + + body := &types.Body{Transactions: types.Transactions{tx1, tx2}} + + // Create the two receipts to manage afterwards + receipt1 := &types.Receipt{ + Status: types.ReceiptStatusFailed, + CumulativeGasUsed: 1, + Logs: []*types.Log{ + {Address: common.BytesToAddress([]byte{0x11})}, + {Address: common.BytesToAddress([]byte{0x01, 0x11})}, + }, + TxHash: tx1.Hash(), + ContractAddress: common.BytesToAddress([]byte{0x01, 0x11, 0x11}), + GasUsed: 111111, + } + receipt1.Bloom = types.CreateBloom(types.Receipts{receipt1}) + + receipt2 := &types.Receipt{ + PostState: common.Hash{2}.Bytes(), + CumulativeGasUsed: 2, + Logs: []*types.Log{ + {Address: common.BytesToAddress([]byte{0x22})}, + {Address: common.BytesToAddress([]byte{0x02, 0x22})}, + }, + TxHash: tx2.Hash(), + ContractAddress: common.BytesToAddress([]byte{0x02, 0x22, 0x22}), + GasUsed: 222222, + } + receipt2.Bloom = types.CreateBloom(types.Receipts{receipt2}) + receipts := []*types.Receipt{receipt1, receipt2} + + hash := common.BytesToHash([]byte{0x03, 0x14}) + // Check that no receipt entries are in a pristine database + if rs := ReadReceipts(db, hash, 0, params.TestChainConfig); len(rs) != 0 { + t.Fatalf("non existent receipts returned: %v", rs) + } + // Insert the body that corresponds to the receipts + WriteBody(db, hash, 0, body) + + // Insert the receipt slice into the database and check presence + WriteReceipts(db, hash, 0, receipts) + + logs := ReadLogs(db, hash, 0) + if len(logs) == 0 { + t.Fatalf("no logs returned") + } + if have, want := len(logs), 2; have != want { + t.Fatalf("unexpected number of logs returned, have %d want %d", have, want) + } + if have, want := len(logs[0]), 2; have != want { + t.Fatalf("unexpected number of logs[0] returned, have %d want %d", have, want) + } + if have, want := len(logs[1]), 2; have != want { + t.Fatalf("unexpected number of logs[1] returned, have %d want %d", have, want) + } + + // Fill in log fields so we can compare their rlp encoding + if err := types.Receipts(receipts).DeriveFields(params.TestChainConfig, hash, 0, body.Transactions); err != nil { + t.Fatal(err) + } + for i, pr := range receipts { + for j, pl := range pr.Logs { + rlpHave, err := rlp.EncodeToBytes(newFullLogRLP(logs[i][j])) + if err != nil { + t.Fatal(err) + } + rlpWant, err := rlp.EncodeToBytes(newFullLogRLP(pl)) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(rlpHave, rlpWant) { + t.Fatalf("receipt #%d: receipt mismatch: have %s, want %s", i, hex.EncodeToString(rlpHave), hex.EncodeToString(rlpWant)) + } + } + } +} + +func TestDeriveLogFields(t *testing.T) { + // Create a few transactions to have receipts for + to2 := common.HexToAddress("0x2") + to3 := common.HexToAddress("0x3") + txs := types.Transactions{ + types.NewTx(&types.LegacyTx{ + Nonce: 1, + Value: big.NewInt(1), + Gas: 1, + GasPrice: big.NewInt(1), + }), + types.NewTx(&types.LegacyTx{ + To: &to2, + Nonce: 2, + Value: big.NewInt(2), + Gas: 2, + GasPrice: big.NewInt(2), + }), + types.NewTx(&types.AccessListTx{ + To: &to3, + Nonce: 3, + Value: big.NewInt(3), + Gas: 3, + GasPrice: big.NewInt(3), + }), + } + // Create the corresponding receipts + receipts := []*receiptLogs{ + { + Logs: []*types.Log{ + {Address: common.BytesToAddress([]byte{0x11})}, + {Address: common.BytesToAddress([]byte{0x01, 0x11})}, + }, + }, + { + Logs: []*types.Log{ + {Address: common.BytesToAddress([]byte{0x22})}, + {Address: common.BytesToAddress([]byte{0x02, 0x22})}, + }, + }, + { + Logs: []*types.Log{ + {Address: common.BytesToAddress([]byte{0x33})}, + {Address: common.BytesToAddress([]byte{0x03, 0x33})}, + }, + }, + } + + // Derive log metadata fields + number := big.NewInt(1) + hash := common.BytesToHash([]byte{0x03, 0x14}) + if err := deriveLogFields(receipts, hash, number.Uint64(), txs); err != nil { + t.Fatal(err) + } + + // Iterate over all the computed fields and check that they're correct + logIndex := uint(0) + for i := range receipts { + for j := range receipts[i].Logs { + if receipts[i].Logs[j].BlockNumber != number.Uint64() { + t.Errorf("receipts[%d].Logs[%d].BlockNumber = %d, want %d", i, j, receipts[i].Logs[j].BlockNumber, number.Uint64()) + } + if receipts[i].Logs[j].BlockHash != hash { + t.Errorf("receipts[%d].Logs[%d].BlockHash = %s, want %s", i, j, receipts[i].Logs[j].BlockHash.String(), hash.String()) + } + if receipts[i].Logs[j].TxHash != txs[i].Hash() { + t.Errorf("receipts[%d].Logs[%d].TxHash = %s, want %s", i, j, receipts[i].Logs[j].TxHash.String(), txs[i].Hash().String()) + } + if receipts[i].Logs[j].TxIndex != uint(i) { + t.Errorf("receipts[%d].Logs[%d].TransactionIndex = %d, want %d", i, j, receipts[i].Logs[j].TxIndex, i) + } + if receipts[i].Logs[j].Index != logIndex { + t.Errorf("receipts[%d].Logs[%d].Index = %d, want %d", i, j, receipts[i].Logs[j].Index, logIndex) + } + logIndex++ + } + } +} + +func BenchmarkDecodeRLPLogs(b *testing.B) { + // Encoded receipts from block 0x14ee094309fbe8f70b65f45ebcc08fb33f126942d97464aad5eb91cfd1e2d269 + buf, err := ioutil.ReadFile("testdata/stored_receipts.bin") + if err != nil { + b.Fatal(err) + } + b.Run("ReceiptForStorage", func(b *testing.B) { + b.ReportAllocs() + var r []*types.ReceiptForStorage + for i := 0; i < b.N; i++ { + if err := rlp.DecodeBytes(buf, &r); err != nil { + b.Fatal(err) + } + } + }) + b.Run("rlpLogs", func(b *testing.B) { + b.ReportAllocs() + var r []*receiptLogs + for i := 0; i < b.N; i++ { + if err := rlp.DecodeBytes(buf, &r); err != nil { + b.Fatal(err) + } + } + }) +} diff --git a/core/rawdb/testdata/stored_receipts.bin b/core/rawdb/testdata/stored_receipts.bin new file mode 100644 index 0000000000000000000000000000000000000000..8204fae09bdef1db96cc7ababca9b938910f8a75 GIT binary patch literal 99991 zcmdRX2|QHY|NqQbv&$M0*>^3Hq(r17$cCw^swPZ^}D6(aVRQ7}xOJs{I zks>>pZ2vps!M(a;p6Qw2^L+o~^>RP=oXdIQL?9MkC9EbI}|`LWLTVwATKWIKgZ~6!6tr2vXRSd6$EY1Q81#zZgoQj^?if zA(=xjT0tN}V?BiBjAj!nd6K{0@m5LnId$v=L}5{R(JL@Wqae5oCsfdvk%`+EFZo@9 zOB;gj8S?jwpdozF)8EcwmtugcUcX(13s}q|I3BK31;@k4NP@Y5xoAFUq6MB^Z}hF3 zS^YQJOJ773Tdg?fd4GLu?GvOQi<)ZL70R~u+6|Y`^GSe?5OyI7aJ)bi;4+S{JBFtY zgag5|C0E$t)kh}$u1(Yc@2{lsXAjOpA=z65w+J3TIHCr4qy)DJlSKX4-z5e=wTqAt zi^Yn!gC}a7L6A8(+Zd+nNH6*~TQN*N07p%Q?!niky98qRj#GxI-7Qq|;(bl*AXp^A zIprQNs}a{ko$ZQ3$$jtjozI}brbPsTceg0Fe`GZDNGn-oC2-zwO$|f=tPIHUuD0P= z*N3Z9#DZhaVH?O9^IWy(>n+Nfee|B14tK{Pwuu3Ux;V4x3npOLgjCECB8DLPCCrJ8 z3HqC0kgJ2>E(EA6;d8|7IkCUKTfkV1W49ni^sRUqXRp~z17aVu@rw!z;+TP>E(g!l zYj)ZXS=9;pgS76V0WVUBJ8)&NQ*}Qi9orQ~2`{?tH1@$=w36c;Xj= z@w+ThJuO`3&vEgaI?uJ4C+`AHSHMF zR6gf3W{O>u3f~Nv)&^c&IpK|_eY^MWIq6-EL1)%RH3yG9UBBIg1@Xm-{_Co+?OWxU z#n4OAvrUP%S2%9w?8bR-9nS^#?7*c8OY{*4bf@7>6%H|DZ8ozj`un@d9vqVty10GR z_nFvs>CerNX?vd`3&0n_Q9w*QNSFNCgRL6hsy#qKzlk`4ehVdO0HM({(u9CH!BKnD zG_Vd_w+oIZQvNM{xG~3aX_BZ-1WYXNUvR#G#sh4j@bOnoqDJU{$M*8*mjK0i9*LME z2##*k!K=;BgJSMq;rpbnulMRo&)sHx%pvZzeY=;GdaeQh_nd)az_o@clSTW$oI1B<)71G5!E z$}M_HPZL2?_RJ68f=gV zuRylcCfB5#i+wQjZ`CJ)=s`JIk#Jgb^0tMw8FttYcOl?7|J|^EX05~lDV+dABNPZS zSX_V)4xE<-J!z)PHas5G>QbAU4}%`pWTJ+jj<#@33eAu3;g1$IByuc*enW6JyqE#x z)JgsowqtKjPfcKgn3Ph)z3|OU5TwE>nc2+x4xE=kmcFc0Mpv`@iBC?JvcaIF2 zthqixcI1xE&+x}aH&=8_hd{vD2FDZXdEwe)@b!e!2kQbdZXUlR?61?stYc6! zNH1opy%^LuiHgX8;~KrOEO)BSNc%h8M&FI{8x1jNFpN0oD&)p=A6oI%`o45gQPGI$ zkChMwkg{%fg-iRD54hWza04S^yDPXF3b9QPJqR2!{dMU|{zFU?Bw~&rF$B?rz%4LG zs)niSj*1ga-#4kwAN^}M+t^$veA&zhQkMhKfP)9$V9d^FZC3cv`fz|6bj^Y&cur3` zRY5p>L(PuKg+Pe#w;QG`3?;B2$Ertxp(NFU9Ngj$8jNmrqwq+(jI)jChRLrYPOp>> zggfqoD1em#nd~C}NF#|lzi5OlLg#@Y0)a7ogf$cOs0KP#B#0giZd1vRQBN+ujQ}{r zqwLLJbY~k5J^V4K_;!YGhYUj+v9*&RF_9ZbR{{Ktbyvb_Bh%JmGV_yMngOZbny0_D}$zhw_K z>nbTJusM*tp~PhUP7wtu>@E*D(zTa=ZPoP`Z!$f&|pMxH4krp zgqUy2@bpL6;3>aWIuWpi1(}@EN!-{|MvhGX{6g3g;cu>w>te{{kzQ;a+88n!5^Y4; z@bz16@wSe0^<_MrQx|1QbK&7Iym>DaKm_Z5QO<&Hoh?M}ZPH zIvB2K=Ue4@QuFVC3uD~4O8vof7FCMdf$B>-IcF`AF2r^>1Z^AB_VaH$TKqG+Li`@B z&rZv`8mJO{;85WVxgCZ_f|-DZ1uo$~Zdee=_O)aeFZ@h9>$^}&P+{981J7L7lKOt2 zVSzJEY{NpZ(BOE%*RVjL!kx{#99$d_wm>YN_=Vt7q!;_?kOJ(XAPRfnSQ%e{VL&3i zcv<-|^cWIVCxTPZ1wep+dD{^E3K+wZs5%j}o$lI}E1sYKneC)g;C*uL1I*XPa?xNw zV0rDt$9Uuv020X)WRib7>B>yk!vN8rPcI@TD<4L3_5rsLg6K5mb`Q{`Vh&M#5*e5B zt6hmA%MtUYwPf6K-s;usjbR95yG@+Dqo5!x?U#zQ*%XB<1(`xe&5vudl9Ed98*o9v6-} zlSZnF2V1Ye#=o4bK;7bq)VB5ClhmF-B{jXW=_C4|^mxcVDJsABU=uL12}~F`=-;Y% ze?|`&DO8@zJc;gc!4(bITtCbY91k~;2FDXhAFMk+xdMkl<-f8X3rqpal_u`tZ+wW_ z$hG>^jGFo0j1rEYKRLn@p4d!3T$POm8-}}?h;L)&!g>=hVO_(mhMl*rvDGS=05J3M zpTc2=yKhMBGf(IkM<^eyk z;H;fWY**AIkG{_n3+}}< z7?Rr4VK~O_art}c6g25IRWj3bfO+#M~ zhtKR=41-EcMVTW3#h-{?w|F?yD#?J|;g`qQDITQ{C|H=x0Y z;MjpEfR(qf?U+JbSjU39@IQ4d98es*B}Kh(`s#Zg*{omd z)g@U1PPad&b5UAVH`a3Lc_oka@9}dXHzo$`buDU_A4{OdKH^`jB z7bGy!j_ZO1AJLGk3sSc`IyO|vqBZbh^I=)P)F9O(yX(~+jxhB?AYD2Acyvd4zo8+8-t>O)8R#K?b7|jtHa0Fn&@n#I1GWLFql*u zB;P3apsYEXvk}!z>!7kHfsn=9ut9!x~hA z<@V`~a*=d}hHbON^34B$w>|Y0Pf*do+eE%Q$xNc6`nw^J-i}O`lbyR)klbH8{q2?f zb7tPJKOhQAG3yBIvH^+SP2g zr-CRfphR-LEdFk67|ITer2v$YLPtr)Wk|?al$L#me4h7gtR>^g|gd|R}h7p+1*#L zQaNoic*z@rhhUsLp!8_J;Cxp0$8D0KBv}8SZ%xrHexcVL&tM9TQtkw zC-0%0Tn;62!bo=qK&hP1G>+0u3@>nCTQXDx?{@RD=qY}u42`6^%6vip6|tQa!Savo zXmjQLA>rG9#gVzvlbxpB#QHoQuyjZ2;@gyt0kK$0H`Q;(q1>#N+thl)c9uMp&ys!( z>q`Z)xGG{%5-j*Nj4#(nGbbS!`>4)N=I@N}tx1JWGZ70v@q*-6>4agtOPwVtKY%Mk%&y0sl99&Ae4>8#0X)~W3uHY3oR+0=X!R85b z+UgS{+Tj(cAhLWIrt_T|xdh!axpMoi#@ZjT8u_mt93YL*e{{*j6{Xq0!!ePR4U*>J^?W>cO-pG~4qb zAY8`-8s4)f?Rsg!+t*PjM*ge*_!L9|vG7AasbFXhQ7xNmbyYat{Cg}#a&4&F}_)wlMUtO@Cb>c6SeRIvN9Hol(?dc8FJ5P?H>KffCCOR`yNzUbw z*Y9SPHXMlPbDI*_T2fX%>|Yg*2PI;YTzT@VRv;Ez;;L0*I4&bXcB!(jPu-b^YN-*t z_#>!#Pf{YuQGR4lPItSQN%nXu^Y$(CZ>LPCDkbruWIpu8^OoCCBk(BmtI-MRvb#TeP3}XNpWAQE3zh{GjFG9 zeI&JK&E_{fu4ad!AYvTtdGApvQp6`m39i^rmvL0bVk2IZ0F4Niu8~n}iK}5UaVSaA z8{F28hyEn<;J17kfV$POuZFVDoC+?*ZG$C((zOI5-j> zhQ{4*a)zV;$~BVhhWZMYO56)vVbK%$>RW>7{|p@93mRXeNanEu;WpiA+0`GNP7A45 zW%++<%+ptfFhsP+(voEqU_EyC2f5|iD5CEF^7OtMYIJ4xbGe_FAO5^;t)qnvLQYx>+)=Od5S+TXBN6XLzR4)6`l{SqfcmIv2sX-(TedzSTvP zo_69zk6UfPNX;<_F5!Q$F@p^`>=?QcoLPzeLxON!xV(^yh&O=VZ)kHI_T#(f|56m;>& zpuynxux_;T%3(ObUa?M&MvC7~yx5!`q5xLL%8KC?*r$E)fFr6!CuqB|{XNI9*&4wV zuMjS>uJhAxV=fnTg z`ZNy$-;qQ+b>Nss$bu%LU+DNeQ z7D<{FXZFX{IyU5MqaE{|;% zMzjNT8}<5s>8l;Bprb=h$%-@2B#XO4AqanBs<(Zyqz!}X4(;3jzbJgLiNi>Tqi zhQrMdJd6%iYgE+Ptaq6_##blz1(Gee8bWwD0FbYrPq!ROqQ(U#Sv9@q-C1hHgc0bF z_|lc3(GFq|l$wz@lrwg#Rk}Pb-}hEc+f(#%vKB-E0sV0g-blleZ@|ooW!B)tYTO}9 z5PLXCC5by6-iV|7(0(40Wkh`=EuR;a>A(lk^$)eqTBYCby?Y3P({3-b=EY?5sK+ig z&M7973d%QC-$4|h`9L7gyg| z5QQb#LavlPjlfo6BkQ>v0c%2)A00h1me0d+93%8H`I8C~<4x?utv>TrF4~;4OfqK! zr~AarW~#(I4+scxy@)-SK83V(DowVA~n^x?U;iX0ZbQ`_o!t#8S z5x1l1s3~qo)6tCe*uY~HTX2ViDg_vnD$>goJ&8dnPDBRz47N!=yOX%){w`)2Ph*{^ zp|~h1V4;qv+RGL^Gj^5#1xu&n^i7ejcW&8V3YNc7jLiVv9AP6BBYp^Z;73PPttE&q zf8UX7n{yb7?WB}tB;U1Lw{!O|-~G>_n+|gBp*piKHa3}>%1DU%i|5C3^=b)#yAq~Q z)wb+9)-MHt%$mO2Z@69$R|}ArtmB?JV13AgvK*3g(xb!jmt%tczufOpvei%RmwjAf zwFMU^-xse@WX$3B-<%yEVJoYDIf*w3-?b>#o{a(8bGLMkAEd(7-CRezA^Q6i>wv#lg-90{G zN3s@%M?`_cP)V@!UW-6UkyJ}tJYSU^1m$C+lNZFJJS{>Te1_C0)=!4i)o4Q$z*z!h zA?Nd=%2<>q1`d-9rNf|FN<)u{aNUsrXK^=bv1wWs=*I1j_tzGhJm@mAP*(f2&*g3V zji_hK?~C4x$X0oPiahwma8%IcLLZ!5AUjbtZcwx3y9$E6z4gx%k?_=~Fy zPAd$r&I^?6sdh0moPIBXfV#w@>YQYEq*@thkIW)yUR=gLY+&%wSf zhYbPB4A1XgC%f$jd>Lsx!ExsZFWTgDE5*q#>@HkPm-Ttgo-({)tqV)25~$U|YX(aO z0zqH-jBHMV`)|3gI6VRyVfVt++?UP%$mP1PHwP)Lr=C2!>Ic>M?F{`}2qXYi06IPS z;mjSn{`VG&eIuCWW3PJ=NRK#jP&}N-1Q&=4vl#VB&_vqcjkl!R#uIA14u8D#{Vh#- zc%u=P|1^F>ng1rdJre?9xv`u7o2)2a00iTIFF#0HmM2*rCAhRe!oVj4tPpkd%M%ZjXB0t(BNPi<`WGa2v|P}$1)eJo`fH0<{?CR!@g3JN4810y{( zf}D(wW~mCWWCdHhPN0!1rq1YBDqG%1O(xc-x_^D1bWMrh*Mw%ubsJMfDe5eGX;RR0 z`bq){1V)Xf@>-+#!BHIc)BX@^(ky!}d+MsqYe(Oq)AE;@T5H$&au|-QPeDf;5MY1R zYXaa1pvQp&+Iwr%TuVxd_WpI@c$eCtcT(aL%=%vmRU<^7rO3a z(8w2VB}z3eG9NA9-36)-ooyi5z+nUZqmK2(^-4B__y0YVO`cA{_Ah&4-bp%acv;eJ z!Zcbmvn)!nXCk=ai0xO>4L}KUsUt_JAFGDEs(3lr@Tp)+@s4eX<*S{_u2RHA9MbE| zI>U)5V9ND5JQ^tzd#|wO0cB_Ru^~rS3FGI$q=u*|7pSNfLN7q{8@XQi zsMr1Tm*hB>f2II|lv?S5g=y!k{yz;~Mb{oGH`TCyj*Vms=k6-wY zOtc~oL)d(L{*kNJsVa$v6JeRY9y30uB%a$4g@r;tP8Lx?S=P}4CPtZ=$u{O z^o~h;`oz)6LZ6R2K4PKFGOh5j=7#e9J7Zuhr}?4;9pa6+*{EC zd+(bR#Y)gNB_Z@678i;Y`&>WVg@Dr$_3|>d$GL$BR9}*#O6bbDW;#N(_#vyb1kEMe$XAhyS2G!YPDdF)kx zKg8!rao-LGmv|Zz2$sU3TOlmDwmTwexmrJXptwh`_R~TX02aZ&&0NMzJXlJdLU-d( zzLYzqkUw0}bmNYOu2h2~t37|E1FoO$p=qV4ue_AQMkwtxJXiw0X=$CUT( zj7jRPkN#`c@Z74Oh#E3fhA4oPsret%?_>>~TeNnzDZ`;$Yx3(=m!9~=EwS1!WU$gDr2>~Hu+4n?jwdH4;VMWUGc6_(HZc1c9 z6hJ6(BE?uc0?Va~EkDWJQ;;t~?lrZ1u`|1hsRLiF=&s@C4G<93KW0QPwby8TIi|Nm zWK;OXz^XQHh{95oTs7xCa45&#<4}@fRZt`Fz1G#y564eOerujS8}otf;dyA&{K~BV zKL3wvQa9Wj3VhDfAKPo3nuiZ1x$)bJE~nLS?UL)RkGM+YrollnO8xm-*NIxMUTdIv z5EbOPGm_Rz7rKt>@5RuQAQWVJDZ_}2z9~L+96{5J4=j#dY{3IH-YtAh0rt=*f<3rz zlbO3+cljmR-0eE;E3$1F66AS_2M>0oJV?RVbvnuE(Qtu_VMuB0?K(W@h?+YRM5o{Wu_H$=zH}F2tJPzEvIl2VE-c%a z0wA&INOQaH{{r3Lxn1{)KVZNR$tac0aGFoKNsp}d&>Ed}yoAw5>^rr`KPEG9O&r(Z|=+O9)`O%r@05?C24Yew|B zPjn!?d{(7TG*2`Jxovs*t+9tOrW!4Iwe9;iZh@_>a_#6~p;+A*wn3TIM`pf0_7yEu z{v1#Fj!d1RQ`2w%@b~~<4ti<2(DN$?!RO-Y!?TT1s~!b8bd0aHYkOdrPh|j(|6SgX ze|7m(P`$P)^a|D$d5v19(p~rlR24KdHVt0?*eir>H+?z`U8l-G(>!rbL^~W#!@U)`Weu(4`dEUF>-7mo5ZjQTjHKZtDy^ z6;+7Bf#J!xdsSy7RF1q?&b$MiErxQhK1dfst5|G*-H18;gbDJ^<`guPP3YO-;4xgY ze&yBNlTmmEazt^-_*G)VBhPP{Uug4I@6Hu`>p`Ud zOIelut!+qtf%>=-FCf8-yO`uZeJFY~nwO||!c%FBdQ)&>DCrpA01z)bWx0SG< z^rQ+I{B_BO38H{l{2`45S4OWdUiWCC8|ihmPG;isL`zVq>ko$4{@zg>7rCXcxydSO zCEL~ly-abPr{$}Zb9c}2P%0VbE16%?Q#q$PTHqzf=#-Qw#dsWIbef{5nPk1i*0SNgqY?HwyY4l7`>Js%Ri(b1SB`$oMGk5hs3Ng$ULOi%_GJbCU&j z-U~KGeAW)wg0F{>@nlUf=G);a+4UX8RoEW!7K01P-0a$;Edw5kH#58{x33W9&W?Ar zzY1M1KVCVOyxJr*QiFQ+om5uC_Mufp*U&J?=Yvo;<5RFxh+8J!1sJ3oAPmxDkPZ{i zg5Wk3wxds9t>d7u6J14N(!xhKp6@8C;>)62%^`34`N8q_!VQDaXCrDKYf-3$?KF6M zSo@uk@v5g)yV6h0TV8rZ-atJ;$+RsEUV3h_G+^mD03FV3JK8FPO`|m-BbzGo^T@#u z!kynX`U`%Ir>5b*cDl8{0)Lm+7{+v|MfpWO5 z!iJs0-45a@n7EAex+5+!;S4-xV{HEO=?V*Pjon&b$}aG~8-G9aZGAgBc9p4N*>r4~ z!wHXL9S;sW*`6(|b-2wFdS^tr(foz;5VYTuAK=dtH0x<^e!nZ`=@{jT5_^lj`5Y`c zl1(3F=MjplLSX&(PU}vR`R~~o!K276U9N!|v*fA)T?5u?>$=cjsPEJY76+n^QwX9Oq`c9Rr+>i3I0@>$B9@the#L*L5`$FDa1s+PSMS6w$3}eXN0wcb3kq8T`%3Lc}_Cd-O zPmb>_sMO>mcEsoT=}Mc295>sc`xkpRYToHS{-{J44F;6%f|S_w^Q zD&24W=R7C8u^$S9Vf>GuUev$K8#(N;KPk-l9n|2$8O?fhE%%!1^nBP4g~6BxJc_hW zEyM8w?dBG+Ar@U~F^sg*0`X4pUZH{}>Q&xwnM314ykHyX(85HtN!Fq9ZhO{WDjl#d zQTt2>%kdm?CuQk+2#Dz?Rv}&-x^)}YOt2J%2fnu4h)#wmEJa)v?M;P2dLg|c;xQPc zGyeOd?Xy!Qca+Gp4v*G^`%l?4k@p(XZ&jz#DN>d+GAd_^gjnGcUmY0G*{ATd_AJZm zV8g_FwR*!41uzVh;rJ@+YGD0uItcDMpfk1i_S271=gOzAQMFf^Dreu;CmqMHXH|_X zXe=K%6hx0VX`C6h`Ywoq$N6LXlu#uu?`zLhtZo*2!~QG27NP*xh~$_cK)>?vQ=!Ld zDhda7m{Ekhio=YyM~#7^;Y3!rKy27T_5ojsJo^jhD|^;5n1rMsOV05+^kygJwe=c8 z>KBOZ0QPSdy=%Kxhb9??OmZu`9LUcj(1Zqq;ONWHDqVT0`LN<_%7s0zk5Uh8y9rSM zDU+Q-g}JXV$32D@S%SiLTld&$a0g4=+u!fy;vVdYqQgDd6Lm}+H|M>t&q6Za&uTEr zcgg6dH^_L-=W1M%BcnRT41voWPj(?Xtz4~C-H+!^_~)&%WTi(T3ZVR8MWk_JGj7Ov zzeVHUW8L@GzV&JKkJ=w=m7pb3+%en2r8eM0NzUXi_PvAcEe<8RcHO_nWn%}IAnSqd zH=%JVyT3^!8qunzx8XyHo3pvDKDe-l9c4v%lU@ytH8%fax5%-m)nGO9?ehx^E0#OD&bj{-dOmW= zk$2MWX?ca4Aa>vtJs5^{-58Z<+#x6dcj$hCJv`X_40pIzh&y~L#~xy(t8xAuCnJt4 z13U}4L=CIYo}^Qz$n`09dH?Dt-Oj-7GhZ)!fPmK&*<^Y48C{avU{4SV{oIMy$qfIS zTd=uRRLE1tmO;FX7s*f(Y(qY8@%b`J{vD+GOi>)oPY+~zSi&ETRwT0%uY$Xo6;+! z!BLZX7NP)BCQooRIid*Mkq5v&G>Zi`_Gux#5*ckU6lx^uC5akRkoi>xtbehYRG%^? zXgjmr7HQcfC0FrI($ptY#j{Ve{1ZN^5Fl`*s2T)qmt8COi`Rd~ZPz*~Qg+c_OLdt1 zI$m(tGTq$$FcKJn!zKL3BXDfG2y<}*udgR@;&1;bRaC%b$A%mc(UBeSugI*{~wetR|YC2lkE}MS3Nj_r;*}f%U56 z?^4yxnf>b7$8BCICcrtZ>%Q;J~~iQzt$-yS@uP!E;*v`Imhm_6eUM z3Si|qnW6x@n)W}7uJC^vU6BV+$>6waM>AT4#+f*7(JOv2OAYtUYX7>b=N zt~>Y@EPbGB%5Vkuds#t3$S@SFNUs!Ebqoa^2B{P@%6x0<3%iF}Yj4pU?LH;dbaKDF zKHy-KSvy)1?dj8>&}h*$6-Qj(L@qQ%CF}F6FrrMD)xz-#8k{KX_a(s!8X*e&^&D^ zykr6&yb$WKApM~LwBxY!5&729KVk_|JPl1XWfBv)In8m%Xs2Y3doehf4j_ zPKA*HJs_NEVtYV>*@jd2(;jfqJ4-*TzkD!&ch_|3;$Df3Sx(62C^T;c6Hy>4yV3xP9?hRltXpf z5Clf-6R&_Y-87oYEm1p6LeA!v;su8*&wXwcK>G%l z@IPqZuqg{w$oo0rO#JqbJmKlKZ|!W#ha79L+1|I!6;Axg{D#sj->mJyy#GL(lVKAl|c?Q9@fHiMzxc39%R#M|LAvZIKE#&|4zv;~_20&wk|Z-?I-Tj5NI! zW@wA2;vr{W?DpOf5G7ZJz*3q;D}kf*E6c@imU7B06Gn&Kr5pT4ln^Gvqr5+#-(S5+e!9u9sQJk zu9ZFXr&@x~RBVJ$MJ&qpc6&J-%BNPeBtxn3#@KOtB7KZ%`RuL2eWqlSAqQUnBb0$+ zJm@);EssftQaSZJOTf?LA_j?jtEepaax&M2)cqrr+uQtJ<4|TFI7l*-A)Wn$A8&o9 zY87(McOEWaW}rCr{-2;k8oG?Um_u3GOEQ%Ac$$SMv`&-NB-F8uJS}D`BPWNh&=c#I z8_ANL5ZIO|M{(|ZsidgmH?Jt!c}@<-hafU!S;qp9G5w1@+9N( zq-lctwjZaSXvOT;Ly=8oz9!rE?Vq4Tu)kM7fkXLB@D0gOTK>Ed((@`cPR9NlXKl;r z!N+BvR{tZEtF$SNaV7pN%$j5<%fjO6_GIXzZ9{{7*f1q@S6&9w{v(up5z)X7J-{z> z8E8p{QX?k+w6(RtzK~VibcOZ>rm;1wGXDsrAXRn59Lmd&Nrp0Po2_YVX;*540)>&z z+v)f3OICjUM<{8{YcdS z_E{?fQ?r`^c2E8hN|c(=IvmOu`fEsrGLTxoJ$Td5`{TZoUzt|VWI6=L?E6P3>62eI z&!H^OCmG6%hx;Ck9(ev?bElT$mmizO>&_?^{Uek(E`f=~OYYa$y$&yJD{+VX`nfIo zdDx$Y$l-j`Q?>*)yxJuT$NTT`Y&(c`h+P>gZWwcbu1Ry3{9ecaKwt$HvtI zy*{CP#Abh@@#f~Q2m%VULD;)cc+e4b2ah1SD1T)x=5xB?ce<62M=~Jxgpz%RwkN-5 z1&fX}JI?&0E>xmrioolB)Ibjju(83iHoHM#!O0if%u3Z72T_T2ovrmc-Dy<*oq;TCeM?j}9y?kR5{6zt0tnA5uh*Zy z;h6o4sbPZk`bn-3IR$*J-P~ogJE%SZU|;)w!(x8jFoZpniDPgFS0wIGxdMAA=V!*{ zr@T)3-^~_Rjp93`qW0c@DZI{_*s)6H_cF0#_Ow27#*FK%$bETc&Rp5OEDS6Z0t)n5 z*s%&8bVQ9+2%D*7P^2qzIy@hH>S1TE-J}C55rS%U6$@|y<({TUJ4GS(i!^iTZ&7mGQw`yv}>N&Z_ja{ z5g3jUg)+Y;=)?8fHt4~e;JyysD87euyM16`fY}7g z3-3ZB&7;f&ZJU)>Cip%7n*P~>YeenE#IIX>d-3f<1->jY|Dv{tpq`svEZtmp-xprt z?R0=^%r8t(*$jQm3b0AT?E_15FqvSxGIxrQ3GfncT7#uJ7r(*M^27*jVae;W; z@=|k!*WB__|G_OmhRAHcXQD1`$I+FAhTo=riFJ@sW4BdgB|P32o0OH_bXWUWQG1?5 zgRyJyPFFFhD*fOJWFU(4RSx1#{22%Hj-_mBWRm`4|83AV$qG_aULf6 zR4wIQYd8-;(1UZrNJjChjZ6=_yabo(kw6+zSVH8wc4toR92QAHP;sq@jXPd*gfzn2rw2Y&p(>8=H~h#sFQGK}w!NM)`nIp8v@kcaA=xk#a?Y4Qs_U$5P2C zZ~ZJH|KwURgHtY-48El$6z{6G^4xS~;PYwJ#z?QWujet;2Z8EZ(#jd0AN0yy!MgU9 z_!Gro?cRX~VtW{GQ^W24(rJu*XZ!ueA7AmYRAlag;5^@YRQt{NHiX!j#-keqtnUUP zPp*R~AQpezTwgwR4D84CFnGWbHNYhZ4t5y6hzBI?@o+YMNa1{J^J?t#!+5|Ewd_TZ znC>y&t9CHo0w7Y&Nq9vMLg7;B8lz-#u~#fq zCrF+aP8Nw>2*M-$Tdgefj~~6`iUSNWHC1O{PTV}Jx2<#%jRzBAYxWwQDScckOA42T zV;*9gU%sE(-N5OuBgbES-Mx`{;p#*p_FVke=GO~7T9;7S^X<^s~I zTci_1V+;5h9-Kt=<^*vPEbGOh#7QVU+ArAN91pnvmc;4Dqukg>a`Av8iaCPBY^<+O z9HXfX(zMIqOG@7x{7Y)U1>%THncG*J&xaozQOpqp*E&?t+L-rxDB6O0mDhUZL6P)h zv&*>OFBi`Yhg$$RQp^zu5>s5YO`Cj8?YqqZdQlxQ;Zgf9_(3-_gP1E2Ub8-HX4gj+ zpvQwt_>X%$oDIEKX5!JVAajc%cf2^pi8AO5r-c;IzKK=mb^oU@il7a;dUwX$%TLxT|$Z0W$tCyFed)cdVC1b78p!hh@) z2xME{4#pBbQA=sCIgq@~=OD|b6n`#yz$@TPliVxdc){xxAfWK#e5u06A`tV$;)|aT z$KSmX zeG_C`%0;eepuvMPO>Bcl&|%jcm-VM&|sVOQX0RK z62B@+-wc#~CcrrLSwj%|NyQ7^;E-E=QG zYeK=)rBI1w&(EfvW>YJnuijc@0Jtymky2QstN?-g-cql$_T2ZDlxpu=e5-r0_t?pe z89!;K?-f3Md7jw%`@MVGHGo5TWGHQ0xRIRW)*6#%@sN^Y65mlR|3hPqOLD=5)V-{E zbzdvlAqpUz$gZ4UH#?Gm8w*ZBa0vt7Te9;W+jWw&p_`U@fZlaGw6vQDC!_d|Cw#zR#>0ZQIdUoi-RRdP{@n2fJvBfB(9NN;!H{0eF#br0Sh8T0WO6c zj@xYgW5cQC_EZryR`D1Xi65iCFv9EA<5_d6MvB!<2Ku&N z0fcm{ptEuK7mDKt1rOcqde7t!eP74|#J_|?@+buMzTJZ`ez-@_IFp%%yyuK23z`-cZATV+=HR0-0GpOsxaVMEd?C)__|MG-JROW=# z4h7c^^ipJhol`hIVr{2658SsS_eFXQW}d>3yJ3*6LF8s4zpU@K=tpl(hxTrJ8oBx2CPG32f217FXqV_-LqN0EKYbPw2Ad_OH7)_ZPxyQ=-MoP3NXWhip6lj=`Q`;YMi z|G%?G$b0zy|IQvE=V!uwWoOu+YVIpL*%ZQ;S?U6Fru{xl4`8;;8ojI}`kln0CJ&oT z?YO%38PXPHTRpxuiGc;TP}i3TEA-aiD*W-SADH35CH%)T z9K=?^pbRGUVmS}aB^KvTQ8y`jI@XtJhypVlIMc+=a0q4_P66)>XThcB;gb5eK>UC3 zm7N9Kz#L+@czC%v1W~IJDE=eLd$P{ZYdor>>y+IQ#`LJh4vmjEwG+25?;HgNIb=>q zuhF-87&3bjEl?c2AhTX#AZ>FPo8w*S+Y+d8E+yy(T)OV4IMMWdlj{7@zlO7o&4t33 z4R70mptPss?~seT2Y_2Ix5ef*tKzhc>n26L)5jBHG{TAgyci`*M?u#OPhPnI2H`AD zmHsA*6zCh{2%-lO%5{0`w0IscG{+B)sD_UqI9TT38@?HzSW`01%O7l1Fv!(GaJd3% zcnO~)X3vTJ_1yx-VjQ~#F`{q9%Q$<@W*QLtn2ldlSU|i7);Nd_1e^liUHJ>#!NZ@O z#ath0<*7Jw0&HPkruxx%(&2h%V0yB~y7@5ZakDAk#rk7CT(lkd(#M6*3|=A81RgH) zXeg$|vw5cmCT(N9rE~Aen79``_nwS>FF$OJ_IeaIA*t%cGsfzMZnq93_SsXv7Wvi8 zk!96%8tJ3nTeL1ou1B=vMKh0!fCBGP?9)zo&=EDMB#5qyBHq|T`;A(N4jrFrD!vuh*pCQZw7J~0 z$rfOPL)PuF!edjao~xm1$x4S`hUg9>hWYN0U#*ZhEN^84-S4d)dfSyz*7uMIW2*Vx zEur?$?yy0>Y$%+dJD8ti$n49TR>zwrIcUyxr~$oz3SK~BP&<*{NTj&)T+cMMs*R>(9_h@du8`$9_>S+?!~0>uZ+x z9kQeTAtxdr}ckX zUeNb@O0-q?@UM|RV0P`Q4WIRiQl1nA=*zM=27r)MqHmuBh z56Tjp$O;z-ge_(tTTE2Pb#b>-rnfg{2j-SCxYI=3&pzC|O^D3v4Y8dCI*2isf=;Zw z=r4w>&LpS+0nA+NDBtD*++KYd zB>H9K;kb^NX0VwDtrT)Z=W1Mj9)ON%`;U81k0q@tX|Nj&&pyD|rWiKlb>nbI#i(19 zs;t)?2s%jG*b_RW6htBZmv+`qQ~T;mAGyiX`aZ0BRb7xs_?`|QHYxl+B;v|L&o!De ztq(rzNM<+iNRhQhCVqd*cXX|TSS6cl;{-qcYjH8k1me-)csPQ=@%=DAa6F7`0uu%f z;sc*RlT&c#pX^10E&B(^?#gq|fA2W* zW3pvWE`P?CG3eXr!OB>ff@xM62b0Dz7zY!lKqpPLuQZ%~QtWQ7jjJ>-;BW8A56+xB zx90NRB71pLD*+wRLe9*48#Op~xY#mXXFGNo9r&)-&i{hir;|6Hi6R}?@2%cY+3@2G zxfKoVTfc#46hVge$M)s4^Ef?Y|=jFn=qO<66D-Nb!J~|Y%(>Kkr4G4&yVHm)e^w% ze4PW2F^9(u)Q`&rNFT^X2}NAWebs+S1wFV*zT0l#tt&d_^VHg>NgwG;X3h4bnw3dy z*(jcEo+|m1{zKD|eLYob9vDZ)} zgQzjvn-`(nbs2}M!UNUqP1!yq`K_&@h2*xk;CppMR^*Q%1LN`b=%LIDY`za63Lt9u z`*+Lmq6N!T@h=+@wPs7yve7CFW`nyLku?A@a3cd^i6Lm4AafvM5cA-$%Z+ds0_aG) z+z3}}IryFeXk~s{)o$D@k7wUAVgWulOY<^)xx$WOcdwNu{;?+WVbCM_zVP>fO8mh> z)XOe6&T{9y_V7kBFy|Lwdqaj3uM2RspOIq-SNjd#(ie#6sa%!GYf<(JY;&BD@682x zBWkAi(#%51=K9E{^*?9dp^Q0Qxr~5~%a>v*Y2F0nya#fv!D8N-f=unTO zL-W6~k0GxC{DyUv?%dl-Nb&Kibi6UtG&F-~aRX4n!Sjl)im8`?iaV6)S#t&|5cO}fyFJL)Zo#RAatJih#*hrmoK(mTS-1*#l!l1XPD@Y&Sp7cJ1)4) zLFggq)1WNbZ=R8VD|shm1n>&Dg#Xwp=p@^EYc~cw=?H2BnFEWIXDN4{vGyiQ1iS*y zG|9aJju!%60mi#j{;O1%2LE-wKrEj4g!GPL`e4+S6lPEr=j z7rB=uO!Q0n50zgxP&H?%j!VTP$5I6%te$jM955tN{AAb ziYOG4Kc8Y}9( z{|1Gu9!@hjFh5lE;L0ScCm0sojTAR5IME>)76wCtH;dxOpPV?Z3vSXBO+)8w8xgtX z?0nm@sj?ZwBGsNDhGTB|vpj!dVvNBMPs0n0UOXN2cp45skKzZJwbn7|M9#%{7C|? zX_I9n(}Sxa+1ZdL^G_e`76}n8xa&X}&pM{@rjGwE zlG_adr|hvWb@ONKO0J}P_U%#CIr^*<<&X++B7*wcnLav6$yRqy+&pIzY`ma@fxVs& zDe^}W3(9q%`Gg1v-$*#c78gt7D(*zz)64(MSZPgQv}T!Y()&V4I|ZH@tJaVB$7gPZ``X8 z^~~akIrrFKfRT~4Haf5t({S5-saKF6otu@~g5iR)(>W0MU7qrf*~WLC_O;`#YrQ2O zbdN?%1yTV7n?w@f3w+T8;^lL!^1QyERZ2_du>1<8^)Ut7SRS5Y{2i`37d31QHEsO#m|z^U1)|ZUwwXT+!aM5BX25zx($kZSlmsN9*Z6Y62r2 zS)UTD4V-^C$+Cq**-PF-`&xC*fajVvh#pX1tY(0v{V2XKfkOUvM04DI+yu|-BUs+C zHk1A{k%gTtzYfkaQL6jKRXPGV1j&(Q;;52CMGq%AfSSI!5?CGnxdPDizzcx{6w)(XlIukBsL*Y#D7!hDHEN~7m2*Pg{6q^D2j-2cTu@~xjU4(% zhjywhp^(+XX$Arg6+H+XhGg{w!-Bh!;)Vq$I>a}Xy4wb$k|CKTU0}#yngkzQMxr@l z$5`!f+XJ#AyZT9G(4(SW@!y$`bG8eAM>5>Wa>bg-8Cs&heR77DzVXT@Iq-y>+k|uR zmH^pj*rDz<0xW>v1R^8 zAV7a86mo!6=sKYlf1h9hJ-|T&a^(gyJnH2} zp6Y0ZuRK04ydmP~(d3O2>W0F(4jZK6dGGE1ZEzT*a2!_9?R#m7G!S2*J_zVR=7A*T zKK@OdPrKq`&&0P_+Sk%IrHwQ~(@#Ki-ce^kun?EggzmX&xnu(%MNw1UxZLhR@vPm8 zMsDps?^wa|QI=fyVBnsl%UkArJ%KkF${3{xWxBzj6o{Bn(SyMOuY(P$SL_Juya?dP z^4L+q!AcH!{u`mXp4T-&a1Y>GMRE544&7{i=cMeA;J^EJFy1Jovl=RI!Wnfh4}RCc z(DnMbQ2PGRe_x+0;Vels6PE1x8|gZ+oNAd8auz39yLZ7g9*4x6WYmOAzzu~P%0~$w zy=)YtpY4lHve5LWM_c-EFodkB>Q=$;FDkpH?b(QkhFYg$tX;SqLtHA{_-P(-tqmcp zNR~a1%+uiGZ_zB<;nEY(dOE{qSD!~D1O;pk6OuuY%JI}20cpvVBoMwVO*m&ZP_{(Q zsq{)!um|?bPPP33*XehVJ1XpS+WjIf+`EK>%TDmVgv_0bGG!VNpx);q)Rw2M?qt26 z7^m2fxnT3+J@$|a80e2NP2QVQp(#|}uu_ELnLeZYepsZt-cSKZb%J~FFQ+<9ayTx{ zqC`xCnPxamX@6xIqEUOs3Zy#0u?Cbz*#tN^xy+jCMAX_eUdmEag8u{CjZE<-j5G4; z%PBU_DBN@N##D%uZJ4OqEG5+B-dC_gABQBjrFUH&^s4}$gVr5@p=f*&Piqf!U~?a- zU}4i67NTl*6W1}O)UOP_sX}fCw%y&vbv9CDNn944^2VK7YSLF;B|vcKtzvD zFcO<Tn0mqu+RskoMS*>CkC;~;$D#9gVF+s4iT_;>4dUi^SE>X<0`%)tA zKA3~s8ITZCv7$6=onLXPEe?zV)*U8!a7f3=l{yu7mLIXSH)(e@mxUN~FSTpFe<9zr zoxb(~xhr)twxWA|Zx()1I_1H7_12r$S{Ijog#ZPECebJr=+%j_G>t+y<&(K-dBe7Y*}JAYN?5}G1O={ zFenw~mk>H70y?rfB`P{Q_Qdpi*L%{Hv?qKLuT>>P>@}A833Qa{l>Y^~pXrpYyzTCq z+&S!?yb8}=Y)Dkt#qGEvD^Kgx%}uGkeMh`%U`&Ttgnhj|mlpV{h_HOV?P>wUl7p;bdxf?1MQpH6%)!X_ch3B?^ju7G$vq4nX zXw;{`+N-KYlZy=6S`Yuw_ElIG+KJj+-EqLnM;O7#vT4{i46w?w8HV-bGA%K8u9=@c zFgz+#zVR~;qL4t~x<*9Hh1OiocP;Vi>EZs~xkaT{GvC#;ZMY` zZ2YWoDT^g?4HCFJrJG+%p8ygh0cR8L!N1{bQbkpc_)uBBTn!f76pej+@_kVj*6~Bw zWx&~lV@+{q6HYG3&L&A~Q-~=`Obecnaj4x?O0gauN~a=N_OmKZa$?tH zrY#$tu~vS^WPl+c^`4+dyH$)@Uqw5+Gq1*@?7sA+a~}^)%MKp? z#6*hNxp7>Wf9r zjGEvM=!v~8pPXm)pFQ|P^DH&v`A14zee36Ho=r*W?2U=K`D9+*6`O`{3CnY}0$g!m z^?_*+xNis5?UcohF|Fo>Pvts2C4~1^nvpvKpz?MwEA3?7r)9iuG{v>aZ}dZr-pL3g za0c6QLNn|8i1RGmgMVY5U9BnFH^R1N2;mzK7cQC;*)@%Wyo7Bc;aF3Ao`sVO@;pn@ z8d#GQq^ATwxepg{)ZiM2Bk>>{QEhsNbue&yNt=WeCy~CQx|QIZ4A_`5PaV!YU#JvBw8XQ&H&osk`sjHTBi;U+cQZ+s4ku+eX}JpSv63nZPs$t35K; z_EE)Cf0@st!?;AUaxViH)rv=|yA>Frp9#V=5KG+9(e#woZ*Azv!|;U0z}kXwi>#Eq z-9HjHcicFCE-c_}CQHi8Z=8A=&w~i>dZw=o$_Ym^Nz-yXjUhl;bk+W$f8)iRe7T?2 zgpX2wDe51)(iD~niuh=@th!&(aqF9swnO8KcQ_bzOcqUpJC)xU`MO15{ejU&zVfz1 zh2xhh$WDww<&FBstRSF@v|nY_kAxkO0^B3G2mi8rM5Dc+sO_fIT4mKF5IZnAzwitF z`G>>oM*;T;jy1XN5vtPQkzL0om#7GSZCE!dS^6k<}VMi;S8 zQ%J*y+!qU%sPC};N5=5LC?1f{AGs9@ohVG|Mv|IbsBbY;zc2wJu8?#4lP|iRDY!N zcsis4T&xfJEjvZG6#dbBJXRf0!``<$@z_m(St?G%>S;U_?V^5;;~n8Ert$B(xQX1n z8<8McubVFKYo7LR zWR$$m@8etu4VJ*5FAl;p=>c+dTNH+8QUgZnLV<7{>Q(buO7FeS&MxB&--WxPK<+36 z!v~rP(P~8V*6eWb=kL$dC6h0thWH+nb(&3%RHrj?2%Zlmz@hS14CP-r7HhbxMt<}* z=a6IRXLZI|!C~ZV`3}t$IN3qvfZRicr@9!A@;0|?eMneFeQq|%VFc%C2n2crq&4OIJiPm8Mb_H?_7af{yzKV%MMvST$0V*kq7n?Y7M=8vx1vT$&qby z7}K2Uxsn|DHpepHJ6GUHrUX9DuGTfr?B)ubT1HdhT4wuvH?Hd{MzNBctL5Ic&kk-5 zd+Yz(Tr%gXAv3)p!Rx&crJ&9bU|S>6kSCA|r;fAQ99da7sO}o7!#4%W@K$46$!9LHR@lCajEeu|j5EKrYL#&R@-)h`H zWbZBzSXC4#IUx$EFb$JPQ8_W`GvTS50i2#p*YVVKfZAp5V835-wG6IFq_r$MRS}=^ zFfNnaw=*57{rp@T*;(aRGWJ^wyVyc4w(RHX+w4_3aE`Epud zW2Mq|(OeuD5Q(NbdTg|#HgDVS@yXEVsG+!CVkl4*Q(#f>* k$z>i|%?5N^IMx*Jws3O!^>7kNt20_fJOv9D?mD>ae?=8nIRF3v literal 0 HcmV?d00001 diff --git a/core/types/receipt_test.go b/core/types/receipt_test.go index 22a316c23..492493d5c 100644 --- a/core/types/receipt_test.go +++ b/core/types/receipt_test.go @@ -273,9 +273,6 @@ func TestDeriveFields(t *testing.T) { if receipts[i].Logs[j].TxHash != txs[i].Hash() { t.Errorf("receipts[%d].Logs[%d].TxHash = %s, want %s", i, j, receipts[i].Logs[j].TxHash.String(), txs[i].Hash().String()) } - if receipts[i].Logs[j].TxHash != txs[i].Hash() { - t.Errorf("receipts[%d].Logs[%d].TxHash = %s, want %s", i, j, receipts[i].Logs[j].TxHash.String(), txs[i].Hash().String()) - } if receipts[i].Logs[j].TxIndex != uint(i) { t.Errorf("receipts[%d].Logs[%d].TransactionIndex = %d, want %d", i, j, receipts[i].Logs[j].TxIndex, i) } diff --git a/eth/api_backend.go b/eth/api_backend.go index 7b40a7edd..1af33414c 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -181,13 +181,14 @@ func (b *EthAPIBackend) GetReceipts(ctx context.Context, hash common.Hash) (type } func (b *EthAPIBackend) GetLogs(ctx context.Context, hash common.Hash) ([][]*types.Log, error) { - receipts := b.eth.blockchain.GetReceiptsByHash(hash) - if receipts == nil { - return nil, nil + db := b.eth.ChainDb() + number := rawdb.ReadHeaderNumber(db, hash) + if number == nil { + return nil, errors.New("failed to get block number from hash") } - logs := make([][]*types.Log, len(receipts)) - for i, receipt := range receipts { - logs[i] = receipt.Logs + logs := rawdb.ReadLogs(db, hash, *number) + if logs == nil { + return nil, errors.New("failed to get logs for block") } return logs, nil } From 92c5d104d0158ed29a2aa4f583c40b7595b72a77 Mon Sep 17 00:00:00 2001 From: Yihau Chen Date: Tue, 28 Sep 2021 18:56:03 +0800 Subject: [PATCH 68/73] accounts/abi/bind: check event signature before parsing (#23230) * accounts/abi/bind: check event signature before parsing * remove redundant break line --- accounts/abi/bind/base.go | 6 ++++++ accounts/abi/bind/base_test.go | 10 +++++----- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/accounts/abi/bind/base.go b/accounts/abi/bind/base.go index 354632b25..6d99517e6 100644 --- a/accounts/abi/bind/base.go +++ b/accounts/abi/bind/base.go @@ -431,6 +431,9 @@ func (c *BoundContract) WatchLogs(opts *WatchOpts, name string, query ...[]inter // UnpackLog unpacks a retrieved log into the provided output structure. func (c *BoundContract) UnpackLog(out interface{}, event string, log types.Log) error { + if log.Topics[0] != c.abi.Events[event].ID { + return fmt.Errorf("event signature mismatch") + } if len(log.Data) > 0 { if err := c.abi.UnpackIntoInterface(out, event, log.Data); err != nil { return err @@ -447,6 +450,9 @@ func (c *BoundContract) UnpackLog(out interface{}, event string, log types.Log) // UnpackLogIntoMap unpacks a retrieved log into the provided map. func (c *BoundContract) UnpackLogIntoMap(out map[string]interface{}, event string, log types.Log) error { + if log.Topics[0] != c.abi.Events[event].ID { + return fmt.Errorf("event signature mismatch") + } if len(log.Data) > 0 { if err := c.abi.UnpackIntoMap(out, event, log.Data); err != nil { return err diff --git a/accounts/abi/bind/base_test.go b/accounts/abi/bind/base_test.go index c4740f68b..7b023e3b4 100644 --- a/accounts/abi/bind/base_test.go +++ b/accounts/abi/bind/base_test.go @@ -110,7 +110,7 @@ const hexData = "0x000000000000000000000000376c47978271565f56deb45495afa69e59c16 func TestUnpackIndexedStringTyLogIntoMap(t *testing.T) { hash := crypto.Keccak256Hash([]byte("testName")) topics := []common.Hash{ - common.HexToHash("0x0"), + crypto.Keccak256Hash([]byte("received(string,address,uint256,bytes)")), hash, } mockLog := newMockLog(topics, common.HexToHash("0x0")) @@ -135,7 +135,7 @@ func TestUnpackIndexedSliceTyLogIntoMap(t *testing.T) { } hash := crypto.Keccak256Hash(sliceBytes) topics := []common.Hash{ - common.HexToHash("0x0"), + crypto.Keccak256Hash([]byte("received(string[],address,uint256,bytes)")), hash, } mockLog := newMockLog(topics, common.HexToHash("0x0")) @@ -160,7 +160,7 @@ func TestUnpackIndexedArrayTyLogIntoMap(t *testing.T) { } hash := crypto.Keccak256Hash(arrBytes) topics := []common.Hash{ - common.HexToHash("0x0"), + crypto.Keccak256Hash([]byte("received(address[2],address,uint256,bytes)")), hash, } mockLog := newMockLog(topics, common.HexToHash("0x0")) @@ -187,7 +187,7 @@ func TestUnpackIndexedFuncTyLogIntoMap(t *testing.T) { var functionTy [24]byte copy(functionTy[:], functionTyBytes[0:24]) topics := []common.Hash{ - common.HexToHash("0x99b5620489b6ef926d4518936cfec15d305452712b88bd59da2d9c10fb0953e8"), + crypto.Keccak256Hash([]byte("received(function,address,uint256,bytes)")), common.BytesToHash(functionTyBytes), } mockLog := newMockLog(topics, common.HexToHash("0x5c698f13940a2153440c6d19660878bc90219d9298fdcf37365aa8d88d40fc42")) @@ -208,7 +208,7 @@ func TestUnpackIndexedBytesTyLogIntoMap(t *testing.T) { bytes := []byte{1, 2, 3, 4, 5} hash := crypto.Keccak256Hash(bytes) topics := []common.Hash{ - common.HexToHash("0x99b5620489b6ef926d4518936cfec15d305452712b88bd59da2d9c10fb0953e8"), + crypto.Keccak256Hash([]byte("received(bytes,address,uint256,bytes)")), hash, } mockLog := newMockLog(topics, common.HexToHash("0x5c698f13940a2153440c6d19660878bc90219d9298fdcf37365aa8d88d40fc42")) From 3531ca2246db3474b5d1e61d18adc7e98725175b Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 28 Sep 2021 13:06:19 +0200 Subject: [PATCH 69/73] eth/tracers: avoid unsyncronized mutations on trie database (#23632) This PR fixes an issue in traceChain, where the statedb Commit operation was performed asynchronously with dereference-operations agains the underlying trie.Database instance. Due to how the reference counting works within the trie database (where parent count is recursively updated when new parents are added), doing dereferencing in the middle of Commit can cause the refcount to become wrong, leading to an inconsistent state. This was fixed by doing Commit/Deref from the same routine. --- eth/state_accessor.go | 3 ++- eth/tracers/api.go | 23 +++++++++++++++++------ 2 files changed, 19 insertions(+), 7 deletions(-) diff --git a/eth/state_accessor.go b/eth/state_accessor.go index eb178311f..ca2002b60 100644 --- a/eth/state_accessor.go +++ b/eth/state_accessor.go @@ -119,7 +119,8 @@ func (eth *Ethereum) stateAtBlock(block *types.Block, reexec uint64, base *state // Finalize the state so any modifications are written to the trie root, err := statedb.Commit(eth.blockchain.Config().IsEIP158(current.Number())) if err != nil { - return nil, err + return nil, fmt.Errorf("stateAtBlock commit failed, number %d root %v: %w", + current.NumberU64(), current.Root().Hex(), err) } statedb, err = state.New(root, database, nil) if err != nil { diff --git a/eth/tracers/api.go b/eth/tracers/api.go index 39b95264d..5019fb6f7 100644 --- a/eth/tracers/api.go +++ b/eth/tracers/api.go @@ -290,7 +290,11 @@ func (api *API) traceChain(ctx context.Context, start, end *types.Block, config }() } // Start a goroutine to feed all the blocks into the tracers - begin := time.Now() + var ( + begin = time.Now() + derefTodo []common.Hash // list of hashes to dereference from the db + derefsMu sync.Mutex // mutex for the derefs + ) go func() { var ( @@ -324,6 +328,14 @@ func (api *API) traceChain(ctx context.Context, start, end *types.Block, config return default: } + // clean out any derefs + derefsMu.Lock() + for _, h := range derefTodo { + statedb.Database().TrieDB().Dereference(h) + } + derefTodo = derefTodo[:0] + derefsMu.Unlock() + // Print progress logs if long enough time elapsed if time.Since(logged) > 8*time.Second { logged = time.Now() @@ -382,12 +394,11 @@ func (api *API) traceChain(ctx context.Context, start, end *types.Block, config Hash: res.block.Hash(), Traces: res.results, } + // Schedule any parent tries held in memory by this task for dereferencing done[uint64(result.Block)] = result - - // Dereference any parent tries held in memory by this task - if res.statedb.Database().TrieDB() != nil { - res.statedb.Database().TrieDB().Dereference(res.rootref) - } + derefsMu.Lock() + derefTodo = append(derefTodo, res.rootref) + derefsMu.Unlock() // Stream completed traces to the user, aborting on the first error for result, ok := done[next]; ok; result, ok = done[next] { if len(result.Traces) > 0 || next == end.NumberU64() { From a541fbea182edcb0313d3b578f939ce2c7045aed Mon Sep 17 00:00:00 2001 From: Ferran Borreguero Date: Tue, 28 Sep 2021 13:44:07 +0200 Subject: [PATCH 70/73] eth/protocols/eth: simplify peer known block/txs caches (#23649) * Simplify peer known block/txns cache * Address minor changes * Add more minor comments * Minor changes from review --- eth/protocols/eth/peer.go | 97 ++++++++++++++++------------------ eth/protocols/eth/peer_test.go | 27 ++++++++++ 2 files changed, 72 insertions(+), 52 deletions(-) diff --git a/eth/protocols/eth/peer.go b/eth/protocols/eth/peer.go index 98273ccfc..1b4cfeb3d 100644 --- a/eth/protocols/eth/peer.go +++ b/eth/protocols/eth/peer.go @@ -75,12 +75,12 @@ type Peer struct { head common.Hash // Latest advertised head block hash td *big.Int // Latest advertised head block total difficulty - knownBlocks mapset.Set // Set of block hashes known to be known by this peer + knownBlocks *knownCache // Set of block hashes known to be known by this peer queuedBlocks chan *blockPropagation // Queue of blocks to broadcast to the peer queuedBlockAnns chan *types.Block // Queue of blocks to announce to the peer txpool TxPool // Transaction pool used by the broadcasters for liveness checks - knownTxs mapset.Set // Set of transaction hashes known to be known by this peer + knownTxs *knownCache // Set of transaction hashes known to be known by this peer txBroadcast chan []common.Hash // Channel used to queue transaction propagation requests txAnnounce chan []common.Hash // Channel used to queue transaction announcement requests @@ -96,8 +96,8 @@ func NewPeer(version uint, p *p2p.Peer, rw p2p.MsgReadWriter, txpool TxPool) *Pe Peer: p, rw: rw, version: version, - knownTxs: mapset.NewSet(), - knownBlocks: mapset.NewSet(), + knownTxs: newKnownCache(maxKnownTxs), + knownBlocks: newKnownCache(maxKnownBlocks), queuedBlocks: make(chan *blockPropagation, maxQueuedBlocks), queuedBlockAnns: make(chan *types.Block, maxQueuedBlockAnns), txBroadcast: make(chan []common.Hash), @@ -162,9 +162,6 @@ func (p *Peer) KnownTransaction(hash common.Hash) bool { // never be propagated to this particular peer. func (p *Peer) markBlock(hash common.Hash) { // If we reached the memory allowance, drop a previously known block hash - for p.knownBlocks.Cardinality() >= maxKnownBlocks { - p.knownBlocks.Pop() - } p.knownBlocks.Add(hash) } @@ -172,9 +169,6 @@ func (p *Peer) markBlock(hash common.Hash) { // will never be propagated to this particular peer. func (p *Peer) markTransaction(hash common.Hash) { // If we reached the memory allowance, drop a previously known transaction hash - for p.knownTxs.Cardinality() >= maxKnownTxs { - p.knownTxs.Pop() - } p.knownTxs.Add(hash) } @@ -189,9 +183,6 @@ func (p *Peer) markTransaction(hash common.Hash) { // tests that directly send messages without having to do the asyn queueing. func (p *Peer) SendTransactions(txs types.Transactions) error { // Mark all the transactions as known, but ensure we don't overflow our limits - for p.knownTxs.Cardinality() > max(0, maxKnownTxs-len(txs)) { - p.knownTxs.Pop() - } for _, tx := range txs { p.knownTxs.Add(tx.Hash()) } @@ -205,12 +196,7 @@ func (p *Peer) AsyncSendTransactions(hashes []common.Hash) { select { case p.txBroadcast <- hashes: // Mark all the transactions as known, but ensure we don't overflow our limits - for p.knownTxs.Cardinality() > max(0, maxKnownTxs-len(hashes)) { - p.knownTxs.Pop() - } - for _, hash := range hashes { - p.knownTxs.Add(hash) - } + p.knownTxs.Add(hashes...) case <-p.term: p.Log().Debug("Dropping transaction propagation", "count", len(hashes)) } @@ -224,12 +210,7 @@ func (p *Peer) AsyncSendTransactions(hashes []common.Hash) { // not be managed directly. func (p *Peer) sendPooledTransactionHashes(hashes []common.Hash) error { // Mark all the transactions as known, but ensure we don't overflow our limits - for p.knownTxs.Cardinality() > max(0, maxKnownTxs-len(hashes)) { - p.knownTxs.Pop() - } - for _, hash := range hashes { - p.knownTxs.Add(hash) - } + p.knownTxs.Add(hashes...) return p2p.Send(p.rw, NewPooledTransactionHashesMsg, NewPooledTransactionHashesPacket(hashes)) } @@ -240,12 +221,7 @@ func (p *Peer) AsyncSendPooledTransactionHashes(hashes []common.Hash) { select { case p.txAnnounce <- hashes: // Mark all the transactions as known, but ensure we don't overflow our limits - for p.knownTxs.Cardinality() > max(0, maxKnownTxs-len(hashes)) { - p.knownTxs.Pop() - } - for _, hash := range hashes { - p.knownTxs.Add(hash) - } + p.knownTxs.Add(hashes...) case <-p.term: p.Log().Debug("Dropping transaction announcement", "count", len(hashes)) } @@ -254,12 +230,8 @@ func (p *Peer) AsyncSendPooledTransactionHashes(hashes []common.Hash) { // ReplyPooledTransactionsRLP is the eth/66 version of SendPooledTransactionsRLP. func (p *Peer) ReplyPooledTransactionsRLP(id uint64, hashes []common.Hash, txs []rlp.RawValue) error { // Mark all the transactions as known, but ensure we don't overflow our limits - for p.knownTxs.Cardinality() > max(0, maxKnownTxs-len(hashes)) { - p.knownTxs.Pop() - } - for _, hash := range hashes { - p.knownTxs.Add(hash) - } + p.knownTxs.Add(hashes...) + // Not packed into PooledTransactionsPacket to avoid RLP decoding return p2p.Send(p.rw, PooledTransactionsMsg, PooledTransactionsRLPPacket66{ RequestId: id, @@ -271,12 +243,8 @@ func (p *Peer) ReplyPooledTransactionsRLP(id uint64, hashes []common.Hash, txs [ // a hash notification. func (p *Peer) SendNewBlockHashes(hashes []common.Hash, numbers []uint64) error { // Mark all the block hashes as known, but ensure we don't overflow our limits - for p.knownBlocks.Cardinality() > max(0, maxKnownBlocks-len(hashes)) { - p.knownBlocks.Pop() - } - for _, hash := range hashes { - p.knownBlocks.Add(hash) - } + p.knownBlocks.Add(hashes...) + request := make(NewBlockHashesPacket, len(hashes)) for i := 0; i < len(hashes); i++ { request[i].Hash = hashes[i] @@ -292,9 +260,6 @@ func (p *Peer) AsyncSendNewBlockHash(block *types.Block) { select { case p.queuedBlockAnns <- block: // Mark all the block hash as known, but ensure we don't overflow our limits - for p.knownBlocks.Cardinality() >= maxKnownBlocks { - p.knownBlocks.Pop() - } p.knownBlocks.Add(block.Hash()) default: p.Log().Debug("Dropping block announcement", "number", block.NumberU64(), "hash", block.Hash()) @@ -304,9 +269,6 @@ func (p *Peer) AsyncSendNewBlockHash(block *types.Block) { // SendNewBlock propagates an entire block to a remote peer. func (p *Peer) SendNewBlock(block *types.Block, td *big.Int) error { // Mark all the block hash as known, but ensure we don't overflow our limits - for p.knownBlocks.Cardinality() >= maxKnownBlocks { - p.knownBlocks.Pop() - } p.knownBlocks.Add(block.Hash()) return p2p.Send(p.rw, NewBlockMsg, &NewBlockPacket{ Block: block, @@ -320,9 +282,6 @@ func (p *Peer) AsyncSendNewBlock(block *types.Block, td *big.Int) { select { case p.queuedBlocks <- &blockPropagation{block: block, td: td}: // Mark all the block hash as known, but ensure we don't overflow our limits - for p.knownBlocks.Cardinality() >= maxKnownBlocks { - p.knownBlocks.Pop() - } p.knownBlocks.Add(block.Hash()) default: p.Log().Debug("Dropping block propagation", "number", block.NumberU64(), "hash", block.Hash()) @@ -465,3 +424,37 @@ func (p *Peer) RequestTxs(hashes []common.Hash) error { GetPooledTransactionsPacket: hashes, }) } + +// knownCache is a cache for known hashes. +type knownCache struct { + hashes mapset.Set + max int +} + +// newKnownCache creates a new knownCache with a max capacity. +func newKnownCache(max int) *knownCache { + return &knownCache{ + max: max, + hashes: mapset.NewSet(), + } +} + +// Add adds a list of elements to the set. +func (k *knownCache) Add(hashes ...common.Hash) { + for k.hashes.Cardinality() > max(0, k.max-len(hashes)) { + k.hashes.Pop() + } + for _, hash := range hashes { + k.hashes.Add(hash) + } +} + +// Contains returns whether the given item is in the set. +func (k *knownCache) Contains(hash common.Hash) bool { + return k.hashes.Contains(hash) +} + +// Cardinality returns the number of elements in the set. +func (k *knownCache) Cardinality() int { + return k.hashes.Cardinality() +} diff --git a/eth/protocols/eth/peer_test.go b/eth/protocols/eth/peer_test.go index 70e9959f8..fc9344370 100644 --- a/eth/protocols/eth/peer_test.go +++ b/eth/protocols/eth/peer_test.go @@ -21,7 +21,9 @@ package eth import ( "crypto/rand" + "testing" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" ) @@ -59,3 +61,28 @@ func (p *testPeer) close() { p.Peer.Close() p.app.Close() } + +func TestPeerSet(t *testing.T) { + size := 5 + s := newKnownCache(size) + + // add 10 items + for i := 0; i < size*2; i++ { + s.Add(common.Hash{byte(i)}) + } + + if s.Cardinality() != size { + t.Fatalf("wrong size, expected %d but found %d", size, s.Cardinality()) + } + + vals := []common.Hash{} + for i := 10; i < 20; i++ { + vals = append(vals, common.Hash{byte(i)}) + } + + // add item in batch + s.Add(vals...) + if s.Cardinality() < size { + t.Fatalf("bad size") + } +} From 42bc1944a17260d75dc1e33e1571c804b9a4a88a Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Wed, 29 Sep 2021 00:28:17 +0200 Subject: [PATCH 71/73] graphql: add storage slots to access list (#23650) Fixes #23640 --- graphql/graphql.go | 6 +++--- graphql/schema.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/graphql/graphql.go b/graphql/graphql.go index 4dd96c4b9..0da9faa95 100644 --- a/graphql/graphql.go +++ b/graphql/graphql.go @@ -158,14 +158,14 @@ func (l *Log) Data(ctx context.Context) hexutil.Bytes { // AccessTuple represents EIP-2930 type AccessTuple struct { address common.Address - storageKeys *[]common.Hash + storageKeys []common.Hash } func (at *AccessTuple) Address(ctx context.Context) common.Address { return at.address } -func (at *AccessTuple) StorageKeys(ctx context.Context) *[]common.Hash { +func (at *AccessTuple) StorageKeys(ctx context.Context) []common.Hash { return at.storageKeys } @@ -442,7 +442,7 @@ func (t *Transaction) AccessList(ctx context.Context) (*[]*AccessTuple, error) { for _, al := range accessList { ret = append(ret, &AccessTuple{ address: al.Address, - storageKeys: &al.StorageKeys, + storageKeys: al.StorageKeys, }) } return &ret, nil diff --git a/graphql/schema.go b/graphql/schema.go index 811c11f6c..dfd094a42 100644 --- a/graphql/schema.go +++ b/graphql/schema.go @@ -72,7 +72,7 @@ const schema string = ` #EIP-2718 type AccessTuple{ address: Address! - storageKeys : [Bytes32!] + storageKeys : [Bytes32!]! } # Transaction is an Ethereum transaction. From 3a6fe69f2356c50d1819be3417fcb89abc6e9cfe Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Wed, 29 Sep 2021 15:19:40 +0200 Subject: [PATCH 72/73] eth/protocols/snap, trie: better error-handling (#23657) --- eth/protocols/snap/handler.go | 2 +- trie/trie.go | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/eth/protocols/snap/handler.go b/eth/protocols/snap/handler.go index 6f345716b..c62f9cfca 100644 --- a/eth/protocols/snap/handler.go +++ b/eth/protocols/snap/handler.go @@ -469,7 +469,7 @@ func handleMessage(backend Backend, peer *Peer) error { // Storage slots requested, open the storage trie and retrieve from there account, err := snap.Account(common.BytesToHash(pathset[0])) loads++ // always account database reads, even for failures - if err != nil { + if err != nil || account == nil { break } stTrie, err := trie.NewSecure(common.BytesToHash(account.Root), triedb) diff --git a/trie/trie.go b/trie/trie.go index 915e6f87d..13343112b 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -176,6 +176,10 @@ func (t *Trie) TryGetNode(path []byte) ([]byte, int, error) { } func (t *Trie) tryGetNode(origNode node, path []byte, pos int) (item []byte, newnode node, resolved int, err error) { + // If non-existent path requested, abort + if origNode == nil { + return nil, nil, 0, nil + } // If we reached the requested path, return the current node if pos >= len(path) { // Although we most probably have the original node expanded, encoding @@ -195,10 +199,6 @@ func (t *Trie) tryGetNode(origNode node, path []byte, pos int) (item []byte, new } // Path still needs to be traversed, descend into children switch n := (origNode).(type) { - case nil: - // Non-existent path requested, abort - return nil, nil, 0, nil - case valueNode: // Path prematurely ended, abort return nil, nil, 0, nil From eae3b1946a276ac099e0018fc792d9e8c3bfda6d Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 29 Sep 2021 19:59:30 +0200 Subject: [PATCH 73/73] params: release go-ethereum v1.10.9 stable --- params/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/params/version.go b/params/version.go index dd629a100..6b269537a 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 10 // Minor version component of the current release - VersionPatch = 9 // Patch version component of the current release - VersionMeta = "unstable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 10 // Minor version component of the current release + VersionPatch = 9 // Patch version component of the current release + VersionMeta = "stable" // Version metadata to append to the version string ) // Version holds the textual version string.