From d3f018fde839fa7ee3fb5eeb14a9eb7e34870636 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 28 Jun 2021 18:40:56 +0300 Subject: [PATCH 01/17] eth: drop eth/65, the last non-reqid protocol version --- cmd/devp2p/internal/ethtest/suite_test.go | 2 +- eth/downloader/downloader.go | 4 +- eth/downloader/downloader_test.go | 95 +---------- eth/downloader/peer.go | 8 +- eth/handler.go | 5 +- eth/handler_eth_test.go | 38 +++-- eth/protocols/eth/handler.go | 47 ++---- eth/protocols/eth/handler_test.go | 134 ++++++--------- eth/protocols/eth/handlers.go | 107 ------------ eth/protocols/eth/handshake_test.go | 1 - eth/protocols/eth/peer.go | 194 +++++++--------------- eth/protocols/eth/protocol.go | 34 ++-- eth/sync.go | 102 +----------- eth/sync_test.go | 1 - les/client_handler.go | 2 +- 15 files changed, 172 insertions(+), 602 deletions(-) diff --git a/cmd/devp2p/internal/ethtest/suite_test.go b/cmd/devp2p/internal/ethtest/suite_test.go index 6e3217151..6d14404e6 100644 --- a/cmd/devp2p/internal/ethtest/suite_test.go +++ b/cmd/devp2p/internal/ethtest/suite_test.go @@ -45,7 +45,7 @@ func TestEthSuite(t *testing.T) { if err != nil { t.Fatalf("could not create new test suite: %v", err) } - for _, test := range suite.AllEthTests() { + for _, test := range suite.Eth66Tests() { t.Run(test.Name, func(t *testing.T) { result := utesting.RunTAP([]utesting.Test{{Name: test.Name, Fn: test.Fn}}, os.Stdout) if result[0].Failed { diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index e8a4a76ca..f0fab1b7a 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -448,8 +448,8 @@ func (d *Downloader) syncWithPeer(p *peerConnection, hash common.Hash, td *big.I d.mux.Post(DoneEvent{latest}) } }() - if p.version < eth.ETH65 { - return fmt.Errorf("%w: advertized %d < required %d", errTooOld, p.version, eth.ETH65) + if p.version < eth.ETH66 { + return fmt.Errorf("%w: advertized %d < required %d", errTooOld, p.version, eth.ETH66) } mode := d.getMode() diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index 794160993..17cd3630c 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -522,10 +522,6 @@ func assertOwnForkedChain(t *testing.T, tester *downloadTester, common int, leng } } -func TestCanonicalSynchronisation65Full(t *testing.T) { testCanonSync(t, eth.ETH65, FullSync) } -func TestCanonicalSynchronisation65Fast(t *testing.T) { testCanonSync(t, eth.ETH65, FastSync) } -func TestCanonicalSynchronisation65Light(t *testing.T) { testCanonSync(t, eth.ETH65, LightSync) } - func TestCanonicalSynchronisation66Full(t *testing.T) { testCanonSync(t, eth.ETH66, FullSync) } func TestCanonicalSynchronisation66Fast(t *testing.T) { testCanonSync(t, eth.ETH66, FastSync) } func TestCanonicalSynchronisation66Light(t *testing.T) { testCanonSync(t, eth.ETH66, LightSync) } @@ -549,9 +545,6 @@ func testCanonSync(t *testing.T, protocol uint, mode SyncMode) { // Tests that if a large batch of blocks are being downloaded, it is throttled // until the cached blocks are retrieved. -func TestThrottling65Full(t *testing.T) { testThrottling(t, eth.ETH65, FullSync) } -func TestThrottling65Fast(t *testing.T) { testThrottling(t, eth.ETH65, FastSync) } - func TestThrottling66Full(t *testing.T) { testThrottling(t, eth.ETH66, FullSync) } func TestThrottling66Fast(t *testing.T) { testThrottling(t, eth.ETH66, FastSync) } @@ -634,10 +627,6 @@ func testThrottling(t *testing.T, protocol uint, mode SyncMode) { // Tests that simple synchronization against a forked chain works correctly. In // this test common ancestor lookup should *not* be short circuited, and a full // binary search should be executed. -func TestForkedSync65Full(t *testing.T) { testForkedSync(t, eth.ETH65, FullSync) } -func TestForkedSync65Fast(t *testing.T) { testForkedSync(t, eth.ETH65, FastSync) } -func TestForkedSync65Light(t *testing.T) { testForkedSync(t, eth.ETH65, LightSync) } - func TestForkedSync66Full(t *testing.T) { testForkedSync(t, eth.ETH66, FullSync) } func TestForkedSync66Fast(t *testing.T) { testForkedSync(t, eth.ETH66, FastSync) } func TestForkedSync66Light(t *testing.T) { testForkedSync(t, eth.ETH66, LightSync) } @@ -667,10 +656,6 @@ func testForkedSync(t *testing.T, protocol uint, mode SyncMode) { // Tests that synchronising against a much shorter but much heavyer fork works // corrently and is not dropped. -func TestHeavyForkedSync65Full(t *testing.T) { testHeavyForkedSync(t, eth.ETH65, FullSync) } -func TestHeavyForkedSync65Fast(t *testing.T) { testHeavyForkedSync(t, eth.ETH65, FastSync) } -func TestHeavyForkedSync65Light(t *testing.T) { testHeavyForkedSync(t, eth.ETH65, LightSync) } - func TestHeavyForkedSync66Full(t *testing.T) { testHeavyForkedSync(t, eth.ETH66, FullSync) } func TestHeavyForkedSync66Fast(t *testing.T) { testHeavyForkedSync(t, eth.ETH66, FastSync) } func TestHeavyForkedSync66Light(t *testing.T) { testHeavyForkedSync(t, eth.ETH66, LightSync) } @@ -702,10 +687,6 @@ func testHeavyForkedSync(t *testing.T, protocol uint, mode SyncMode) { // Tests that chain forks are contained within a certain interval of the current // chain head, ensuring that malicious peers cannot waste resources by feeding // long dead chains. -func TestBoundedForkedSync65Full(t *testing.T) { testBoundedForkedSync(t, eth.ETH65, FullSync) } -func TestBoundedForkedSync65Fast(t *testing.T) { testBoundedForkedSync(t, eth.ETH65, FastSync) } -func TestBoundedForkedSync65Light(t *testing.T) { testBoundedForkedSync(t, eth.ETH65, LightSync) } - func TestBoundedForkedSync66Full(t *testing.T) { testBoundedForkedSync(t, eth.ETH66, FullSync) } func TestBoundedForkedSync66Fast(t *testing.T) { testBoundedForkedSync(t, eth.ETH66, FastSync) } func TestBoundedForkedSync66Light(t *testing.T) { testBoundedForkedSync(t, eth.ETH66, LightSync) } @@ -736,16 +717,6 @@ func testBoundedForkedSync(t *testing.T, protocol uint, mode SyncMode) { // Tests that chain forks are contained within a certain interval of the current // chain head for short but heavy forks too. These are a bit special because they // take different ancestor lookup paths. -func TestBoundedHeavyForkedSync65Full(t *testing.T) { - testBoundedHeavyForkedSync(t, eth.ETH65, FullSync) -} -func TestBoundedHeavyForkedSync65Fast(t *testing.T) { - testBoundedHeavyForkedSync(t, eth.ETH65, FastSync) -} -func TestBoundedHeavyForkedSync65Light(t *testing.T) { - testBoundedHeavyForkedSync(t, eth.ETH65, LightSync) -} - func TestBoundedHeavyForkedSync66Full(t *testing.T) { testBoundedHeavyForkedSync(t, eth.ETH66, FullSync) } @@ -800,10 +771,6 @@ func TestInactiveDownloader63(t *testing.T) { } // Tests that a canceled download wipes all previously accumulated state. -func TestCancel65Full(t *testing.T) { testCancel(t, eth.ETH65, FullSync) } -func TestCancel65Fast(t *testing.T) { testCancel(t, eth.ETH65, FastSync) } -func TestCancel65Light(t *testing.T) { testCancel(t, eth.ETH65, LightSync) } - func TestCancel66Full(t *testing.T) { testCancel(t, eth.ETH66, FullSync) } func TestCancel66Fast(t *testing.T) { testCancel(t, eth.ETH66, FastSync) } func TestCancel66Light(t *testing.T) { testCancel(t, eth.ETH66, LightSync) } @@ -833,10 +800,6 @@ func testCancel(t *testing.T, protocol uint, mode SyncMode) { } // Tests that synchronisation from multiple peers works as intended (multi thread sanity test). -func TestMultiSynchronisation65Full(t *testing.T) { testMultiSynchronisation(t, eth.ETH65, FullSync) } -func TestMultiSynchronisation65Fast(t *testing.T) { testMultiSynchronisation(t, eth.ETH65, FastSync) } -func TestMultiSynchronisation65Light(t *testing.T) { testMultiSynchronisation(t, eth.ETH65, LightSync) } - func TestMultiSynchronisation66Full(t *testing.T) { testMultiSynchronisation(t, eth.ETH66, FullSync) } func TestMultiSynchronisation66Fast(t *testing.T) { testMultiSynchronisation(t, eth.ETH66, FastSync) } func TestMultiSynchronisation66Light(t *testing.T) { testMultiSynchronisation(t, eth.ETH66, LightSync) } @@ -863,10 +826,6 @@ func testMultiSynchronisation(t *testing.T, protocol uint, mode SyncMode) { // Tests that synchronisations behave well in multi-version protocol environments // and not wreak havoc on other nodes in the network. -func TestMultiProtoSynchronisation65Full(t *testing.T) { testMultiProtoSync(t, eth.ETH65, FullSync) } -func TestMultiProtoSynchronisation65Fast(t *testing.T) { testMultiProtoSync(t, eth.ETH65, FastSync) } -func TestMultiProtoSynchronisation65Light(t *testing.T) { testMultiProtoSync(t, eth.ETH65, LightSync) } - func TestMultiProtoSynchronisation66Full(t *testing.T) { testMultiProtoSync(t, eth.ETH66, FullSync) } func TestMultiProtoSynchronisation66Fast(t *testing.T) { testMultiProtoSync(t, eth.ETH66, FastSync) } func TestMultiProtoSynchronisation66Light(t *testing.T) { testMultiProtoSync(t, eth.ETH66, LightSync) } @@ -881,8 +840,8 @@ func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { chain := testChainBase.shorten(blockCacheMaxItems - 15) // Create peers of every type - tester.newPeer("peer 65", eth.ETH65, chain) tester.newPeer("peer 66", eth.ETH66, chain) + //tester.newPeer("peer 65", eth.ETH67, chain) // Synchronise with the requested peer and make sure all blocks were retrieved if err := tester.sync(fmt.Sprintf("peer %d", protocol), nil, mode); err != nil { @@ -891,7 +850,7 @@ func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { assertOwnChain(t, tester, chain.len()) // Check that no peers have been dropped off - for _, version := range []int{65, 66} { + for _, version := range []int{66} { peer := fmt.Sprintf("peer %d", version) if _, ok := tester.peers[peer]; !ok { t.Errorf("%s dropped", peer) @@ -901,10 +860,6 @@ func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { // Tests that if a block is empty (e.g. header only), no body request should be // made, and instead the header should be assembled into a whole block in itself. -func TestEmptyShortCircuit65Full(t *testing.T) { testEmptyShortCircuit(t, eth.ETH65, FullSync) } -func TestEmptyShortCircuit65Fast(t *testing.T) { testEmptyShortCircuit(t, eth.ETH65, FastSync) } -func TestEmptyShortCircuit65Light(t *testing.T) { testEmptyShortCircuit(t, eth.ETH65, LightSync) } - func TestEmptyShortCircuit66Full(t *testing.T) { testEmptyShortCircuit(t, eth.ETH66, FullSync) } func TestEmptyShortCircuit66Fast(t *testing.T) { testEmptyShortCircuit(t, eth.ETH66, FastSync) } func TestEmptyShortCircuit66Light(t *testing.T) { testEmptyShortCircuit(t, eth.ETH66, LightSync) } @@ -955,10 +910,6 @@ func testEmptyShortCircuit(t *testing.T, protocol uint, mode SyncMode) { // Tests that headers are enqueued continuously, preventing malicious nodes from // stalling the downloader by feeding gapped header chains. -func TestMissingHeaderAttack65Full(t *testing.T) { testMissingHeaderAttack(t, eth.ETH65, FullSync) } -func TestMissingHeaderAttack65Fast(t *testing.T) { testMissingHeaderAttack(t, eth.ETH65, FastSync) } -func TestMissingHeaderAttack65Light(t *testing.T) { testMissingHeaderAttack(t, eth.ETH65, LightSync) } - func TestMissingHeaderAttack66Full(t *testing.T) { testMissingHeaderAttack(t, eth.ETH66, FullSync) } func TestMissingHeaderAttack66Fast(t *testing.T) { testMissingHeaderAttack(t, eth.ETH66, FastSync) } func TestMissingHeaderAttack66Light(t *testing.T) { testMissingHeaderAttack(t, eth.ETH66, LightSync) } @@ -987,10 +938,6 @@ func testMissingHeaderAttack(t *testing.T, protocol uint, mode SyncMode) { // Tests that if requested headers are shifted (i.e. first is missing), the queue // detects the invalid numbering. -func TestShiftedHeaderAttack65Full(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH65, FullSync) } -func TestShiftedHeaderAttack65Fast(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH65, FastSync) } -func TestShiftedHeaderAttack65Light(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH65, LightSync) } - func TestShiftedHeaderAttack66Full(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH66, FullSync) } func TestShiftedHeaderAttack66Fast(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH66, FastSync) } func TestShiftedHeaderAttack66Light(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH66, LightSync) } @@ -1024,7 +971,6 @@ func testShiftedHeaderAttack(t *testing.T, protocol uint, mode SyncMode) { // Tests that upon detecting an invalid header, the recent ones are rolled back // for various failure scenarios. Afterwards a full sync is attempted to make // sure no state was corrupted. -func TestInvalidHeaderRollback65Fast(t *testing.T) { testInvalidHeaderRollback(t, eth.ETH65, FastSync) } func TestInvalidHeaderRollback66Fast(t *testing.T) { testInvalidHeaderRollback(t, eth.ETH66, FastSync) } func testInvalidHeaderRollback(t *testing.T, protocol uint, mode SyncMode) { @@ -1115,16 +1061,6 @@ func testInvalidHeaderRollback(t *testing.T, protocol uint, mode SyncMode) { // Tests that a peer advertising a high TD doesn't get to stall the downloader // afterwards by not sending any useful hashes. -func TestHighTDStarvationAttack65Full(t *testing.T) { - testHighTDStarvationAttack(t, eth.ETH65, FullSync) -} -func TestHighTDStarvationAttack65Fast(t *testing.T) { - testHighTDStarvationAttack(t, eth.ETH65, FastSync) -} -func TestHighTDStarvationAttack65Light(t *testing.T) { - testHighTDStarvationAttack(t, eth.ETH65, LightSync) -} - func TestHighTDStarvationAttack66Full(t *testing.T) { testHighTDStarvationAttack(t, eth.ETH66, FullSync) } @@ -1149,7 +1085,6 @@ func testHighTDStarvationAttack(t *testing.T, protocol uint, mode SyncMode) { } // Tests that misbehaving peers are disconnected, whilst behaving ones are not. -func TestBlockHeaderAttackerDropping65(t *testing.T) { testBlockHeaderAttackerDropping(t, eth.ETH65) } func TestBlockHeaderAttackerDropping66(t *testing.T) { testBlockHeaderAttackerDropping(t, eth.ETH66) } func testBlockHeaderAttackerDropping(t *testing.T, protocol uint) { @@ -1202,10 +1137,6 @@ func testBlockHeaderAttackerDropping(t *testing.T, protocol uint) { // Tests that synchronisation progress (origin block number, current block number // and highest block number) is tracked and updated correctly. -func TestSyncProgress65Full(t *testing.T) { testSyncProgress(t, eth.ETH65, FullSync) } -func TestSyncProgress65Fast(t *testing.T) { testSyncProgress(t, eth.ETH65, FastSync) } -func TestSyncProgress65Light(t *testing.T) { testSyncProgress(t, eth.ETH65, LightSync) } - func TestSyncProgress66Full(t *testing.T) { testSyncProgress(t, eth.ETH66, FullSync) } func TestSyncProgress66Fast(t *testing.T) { testSyncProgress(t, eth.ETH66, FastSync) } func TestSyncProgress66Light(t *testing.T) { testSyncProgress(t, eth.ETH66, LightSync) } @@ -1286,10 +1217,6 @@ func checkProgress(t *testing.T, d *Downloader, stage string, want ethereum.Sync // Tests that synchronisation progress (origin block number and highest block // number) is tracked and updated correctly in case of a fork (or manual head // revertal). -func TestForkedSyncProgress65Full(t *testing.T) { testForkedSyncProgress(t, eth.ETH65, FullSync) } -func TestForkedSyncProgress65Fast(t *testing.T) { testForkedSyncProgress(t, eth.ETH65, FastSync) } -func TestForkedSyncProgress65Light(t *testing.T) { testForkedSyncProgress(t, eth.ETH65, LightSync) } - func TestForkedSyncProgress66Full(t *testing.T) { testForkedSyncProgress(t, eth.ETH66, FullSync) } func TestForkedSyncProgress66Fast(t *testing.T) { testForkedSyncProgress(t, eth.ETH66, FastSync) } func TestForkedSyncProgress66Light(t *testing.T) { testForkedSyncProgress(t, eth.ETH66, LightSync) } @@ -1362,10 +1289,6 @@ func testForkedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { // Tests that if synchronisation is aborted due to some failure, then the progress // origin is not updated in the next sync cycle, as it should be considered the // continuation of the previous sync and not a new instance. -func TestFailedSyncProgress65Full(t *testing.T) { testFailedSyncProgress(t, eth.ETH65, FullSync) } -func TestFailedSyncProgress65Fast(t *testing.T) { testFailedSyncProgress(t, eth.ETH65, FastSync) } -func TestFailedSyncProgress65Light(t *testing.T) { testFailedSyncProgress(t, eth.ETH65, LightSync) } - func TestFailedSyncProgress66Full(t *testing.T) { testFailedSyncProgress(t, eth.ETH66, FullSync) } func TestFailedSyncProgress66Fast(t *testing.T) { testFailedSyncProgress(t, eth.ETH66, FastSync) } func TestFailedSyncProgress66Light(t *testing.T) { testFailedSyncProgress(t, eth.ETH66, LightSync) } @@ -1435,10 +1358,6 @@ func testFailedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { // Tests that if an attacker fakes a chain height, after the attack is detected, // the progress height is successfully reduced at the next sync invocation. -func TestFakedSyncProgress65Full(t *testing.T) { testFakedSyncProgress(t, eth.ETH65, FullSync) } -func TestFakedSyncProgress65Fast(t *testing.T) { testFakedSyncProgress(t, eth.ETH65, FastSync) } -func TestFakedSyncProgress65Light(t *testing.T) { testFakedSyncProgress(t, eth.ETH65, LightSync) } - func TestFakedSyncProgress66Full(t *testing.T) { testFakedSyncProgress(t, eth.ETH66, FullSync) } func TestFakedSyncProgress66Fast(t *testing.T) { testFakedSyncProgress(t, eth.ETH66, FastSync) } func TestFakedSyncProgress66Light(t *testing.T) { testFakedSyncProgress(t, eth.ETH66, LightSync) } @@ -1512,10 +1431,6 @@ func testFakedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { // This test reproduces an issue where unexpected deliveries would // block indefinitely if they arrived at the right time. -func TestDeliverHeadersHang65Full(t *testing.T) { testDeliverHeadersHang(t, eth.ETH65, FullSync) } -func TestDeliverHeadersHang65Fast(t *testing.T) { testDeliverHeadersHang(t, eth.ETH65, FastSync) } -func TestDeliverHeadersHang65Light(t *testing.T) { testDeliverHeadersHang(t, eth.ETH65, LightSync) } - func TestDeliverHeadersHang66Full(t *testing.T) { testDeliverHeadersHang(t, eth.ETH66, FullSync) } func TestDeliverHeadersHang66Fast(t *testing.T) { testDeliverHeadersHang(t, eth.ETH66, FastSync) } func TestDeliverHeadersHang66Light(t *testing.T) { testDeliverHeadersHang(t, eth.ETH66, LightSync) } @@ -1673,12 +1588,6 @@ func TestRemoteHeaderRequestSpan(t *testing.T) { // Tests that peers below a pre-configured checkpoint block are prevented from // being fast-synced from, avoiding potential cheap eclipse attacks. -func TestCheckpointEnforcement65Full(t *testing.T) { testCheckpointEnforcement(t, eth.ETH65, FullSync) } -func TestCheckpointEnforcement65Fast(t *testing.T) { testCheckpointEnforcement(t, eth.ETH65, FastSync) } -func TestCheckpointEnforcement65Light(t *testing.T) { - testCheckpointEnforcement(t, eth.ETH65, LightSync) -} - func TestCheckpointEnforcement66Full(t *testing.T) { testCheckpointEnforcement(t, eth.ETH66, FullSync) } func TestCheckpointEnforcement66Fast(t *testing.T) { testCheckpointEnforcement(t, eth.ETH66, FastSync) } func TestCheckpointEnforcement66Light(t *testing.T) { diff --git a/eth/downloader/peer.go b/eth/downloader/peer.go index 066a36631..863294832 100644 --- a/eth/downloader/peer.go +++ b/eth/downloader/peer.go @@ -413,7 +413,7 @@ func (ps *peerSet) HeaderIdlePeers() ([]*peerConnection, int) { throughput := func(p *peerConnection) int { return p.rates.Capacity(eth.BlockHeadersMsg, time.Second) } - return ps.idlePeers(eth.ETH65, eth.ETH66, idle, throughput) + return ps.idlePeers(eth.ETH66, eth.ETH66, idle, throughput) } // BodyIdlePeers retrieves a flat list of all the currently body-idle peers within @@ -425,7 +425,7 @@ func (ps *peerSet) BodyIdlePeers() ([]*peerConnection, int) { throughput := func(p *peerConnection) int { return p.rates.Capacity(eth.BlockBodiesMsg, time.Second) } - return ps.idlePeers(eth.ETH65, eth.ETH66, idle, throughput) + return ps.idlePeers(eth.ETH66, eth.ETH66, idle, throughput) } // ReceiptIdlePeers retrieves a flat list of all the currently receipt-idle peers @@ -437,7 +437,7 @@ func (ps *peerSet) ReceiptIdlePeers() ([]*peerConnection, int) { throughput := func(p *peerConnection) int { return p.rates.Capacity(eth.ReceiptsMsg, time.Second) } - return ps.idlePeers(eth.ETH65, eth.ETH66, idle, throughput) + return ps.idlePeers(eth.ETH66, eth.ETH66, idle, throughput) } // NodeDataIdlePeers retrieves a flat list of all the currently node-data-idle @@ -449,7 +449,7 @@ func (ps *peerSet) NodeDataIdlePeers() ([]*peerConnection, int) { throughput := func(p *peerConnection) int { return p.rates.Capacity(eth.NodeDataMsg, time.Second) } - return ps.idlePeers(eth.ETH65, eth.ETH66, idle, throughput) + return ps.idlePeers(eth.ETH66, eth.ETH66, idle, throughput) } // idlePeers retrieves a flat list of all currently idle peers satisfying the diff --git a/eth/handler.go b/eth/handler.go index aff4871af..06a8088bf 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -117,7 +117,6 @@ type handler struct { whitelist map[uint64]common.Hash // channels for fetcher, syncer, txsyncLoop - txsyncCh chan *txsync quitSync chan struct{} chainSync *chainSyncer @@ -140,7 +139,6 @@ func newHandler(config *handlerConfig) (*handler, error) { chain: config.Chain, peers: newPeerSet(), whitelist: config.Whitelist, - txsyncCh: make(chan *txsync), quitSync: make(chan struct{}), } if config.Sync == downloader.FullSync { @@ -408,9 +406,8 @@ func (h *handler) Start(maxPeers int) { go h.minedBroadcastLoop() // start sync handlers - h.wg.Add(2) + h.wg.Add(1) go h.chainSync.loop() - go h.txsyncLoop64() // TODO(karalabe): Legacy initial tx echange, drop with eth/64. } func (h *handler) Stop() { diff --git a/eth/handler_eth_test.go b/eth/handler_eth_test.go index 038de4699..039091244 100644 --- a/eth/handler_eth_test.go +++ b/eth/handler_eth_test.go @@ -80,7 +80,6 @@ func (h *testEthHandler) Handle(peer *eth.Peer, packet eth.Packet) error { // Tests that peers are correctly accepted (or rejected) based on the advertised // fork IDs in the protocol handshake. -func TestForkIDSplit65(t *testing.T) { testForkIDSplit(t, eth.ETH65) } func TestForkIDSplit66(t *testing.T) { testForkIDSplit(t, eth.ETH66) } func testForkIDSplit(t *testing.T, protocol uint) { @@ -236,7 +235,6 @@ func testForkIDSplit(t *testing.T, protocol uint) { } // Tests that received transactions are added to the local pool. -func TestRecvTransactions65(t *testing.T) { testRecvTransactions(t, eth.ETH65) } func TestRecvTransactions66(t *testing.T) { testRecvTransactions(t, eth.ETH66) } func testRecvTransactions(t *testing.T, protocol uint) { @@ -294,7 +292,6 @@ func testRecvTransactions(t *testing.T, protocol uint) { } // This test checks that pending transactions are sent. -func TestSendTransactions65(t *testing.T) { testSendTransactions(t, eth.ETH65) } func TestSendTransactions66(t *testing.T) { testSendTransactions(t, eth.ETH66) } func testSendTransactions(t *testing.T, protocol uint) { @@ -306,7 +303,7 @@ func testSendTransactions(t *testing.T, protocol uint) { insert := make([]*types.Transaction, 100) for nonce := range insert { - tx := types.NewTransaction(uint64(nonce), common.Address{}, big.NewInt(0), 100000, big.NewInt(0), make([]byte, txsyncPackSize/10)) + tx := types.NewTransaction(uint64(nonce), common.Address{}, big.NewInt(0), 100000, big.NewInt(0), make([]byte, 10240)) tx, _ = types.SignTx(tx, types.HomesteadSigner{}, testKey) insert[nonce] = tx @@ -380,7 +377,6 @@ func testSendTransactions(t *testing.T, protocol uint) { // Tests that transactions get propagated to all attached peers, either via direct // broadcasts or via announcements/retrievals. -func TestTransactionPropagation65(t *testing.T) { testTransactionPropagation(t, eth.ETH65) } func TestTransactionPropagation66(t *testing.T) { testTransactionPropagation(t, eth.ETH66) } func testTransactionPropagation(t *testing.T, protocol uint) { @@ -521,8 +517,8 @@ func testCheckpointChallenge(t *testing.T, syncmode downloader.SyncMode, checkpo defer p2pLocal.Close() defer p2pRemote.Close() - local := eth.NewPeer(eth.ETH65, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pLocal), p2pLocal, handler.txpool) - remote := eth.NewPeer(eth.ETH65, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pRemote), p2pRemote, handler.txpool) + local := eth.NewPeer(eth.ETH66, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pLocal), p2pLocal, handler.txpool) + remote := eth.NewPeer(eth.ETH66, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pRemote), p2pRemote, handler.txpool) defer local.Close() defer remote.Close() @@ -543,30 +539,39 @@ func testCheckpointChallenge(t *testing.T, syncmode downloader.SyncMode, checkpo if err := remote.Handshake(1, td, head.Hash(), genesis.Hash(), forkid.NewIDWithChain(handler.chain), forkid.NewFilter(handler.chain)); err != nil { t.Fatalf("failed to run protocol handshake") } - // Connect a new peer and check that we receive the checkpoint challenge. if checkpoint { - if err := remote.ExpectRequestHeadersByNumber(response.Number.Uint64(), 1, 0, false); err != nil { - t.Fatalf("challenge mismatch: %v", err) + msg, err := p2pRemote.ReadMsg() + if err != nil { + t.Fatalf("failed to read checkpoint challenge: %v", err) + } + request := new(eth.GetBlockHeadersPacket66) + if err := msg.Decode(request); err != nil { + t.Fatalf("failed to decode checkpoint challenge: %v", err) + } + query := request.GetBlockHeadersPacket + if query.Origin.Number != response.Number.Uint64() || query.Amount != 1 || query.Skip != 0 || query.Reverse { + t.Fatalf("challenge mismatch: have [%d, %d, %d, %v] want [%d, %d, %d, %v]", + query.Origin.Number, query.Amount, query.Skip, query.Reverse, + response.Number.Uint64(), 1, 0, false) } // Create a block to reply to the challenge if no timeout is simulated. if !timeout { if empty { - if err := remote.SendBlockHeaders([]*types.Header{}); err != nil { + if err := remote.ReplyBlockHeaders(request.RequestId, []*types.Header{}); err != nil { t.Fatalf("failed to answer challenge: %v", err) } } else if match { - if err := remote.SendBlockHeaders([]*types.Header{response}); err != nil { + if err := remote.ReplyBlockHeaders(request.RequestId, []*types.Header{response}); err != nil { t.Fatalf("failed to answer challenge: %v", err) } } else { - if err := remote.SendBlockHeaders([]*types.Header{{Number: response.Number}}); err != nil { + if err := remote.ReplyBlockHeaders(request.RequestId, []*types.Header{{Number: response.Number}}); err != nil { t.Fatalf("failed to answer challenge: %v", err) } } } } - // Wait until the test timeout passes to ensure proper cleanup time.Sleep(syncChallengeTimeout + 300*time.Millisecond) @@ -619,8 +624,8 @@ func testBroadcastBlock(t *testing.T, peers, bcasts int) { defer sourcePipe.Close() defer sinkPipe.Close() - sourcePeer := eth.NewPeer(eth.ETH65, p2p.NewPeerPipe(enode.ID{byte(i)}, "", nil, sourcePipe), sourcePipe, nil) - sinkPeer := eth.NewPeer(eth.ETH65, p2p.NewPeerPipe(enode.ID{0}, "", nil, sinkPipe), sinkPipe, nil) + sourcePeer := eth.NewPeer(eth.ETH66, p2p.NewPeerPipe(enode.ID{byte(i)}, "", nil, sourcePipe), sourcePipe, nil) + sinkPeer := eth.NewPeer(eth.ETH66, p2p.NewPeerPipe(enode.ID{0}, "", nil, sinkPipe), sinkPipe, nil) defer sourcePeer.Close() defer sinkPeer.Close() @@ -671,7 +676,6 @@ func testBroadcastBlock(t *testing.T, peers, bcasts int) { // Tests that a propagated malformed block (uncles or transactions don't match // with the hashes in the header) gets discarded and not broadcast forward. -func TestBroadcastMalformedBlock65(t *testing.T) { testBroadcastMalformedBlock(t, eth.ETH65) } func TestBroadcastMalformedBlock66(t *testing.T) { testBroadcastMalformedBlock(t, eth.ETH66) } func testBroadcastMalformedBlock(t *testing.T, protocol uint) { diff --git a/eth/protocols/eth/handler.go b/eth/protocols/eth/handler.go index 6bbaa2f55..828930014 100644 --- a/eth/protocols/eth/handler.go +++ b/eth/protocols/eth/handler.go @@ -171,39 +171,21 @@ type Decoder interface { Time() time.Time } -var eth65 = map[uint64]msgHandler{ - GetBlockHeadersMsg: handleGetBlockHeaders, - BlockHeadersMsg: handleBlockHeaders, - GetBlockBodiesMsg: handleGetBlockBodies, - BlockBodiesMsg: handleBlockBodies, - GetNodeDataMsg: handleGetNodeData, - NodeDataMsg: handleNodeData, - GetReceiptsMsg: handleGetReceipts, - ReceiptsMsg: handleReceipts, - NewBlockHashesMsg: handleNewBlockhashes, - NewBlockMsg: handleNewBlock, - TransactionsMsg: handleTransactions, - NewPooledTransactionHashesMsg: handleNewPooledTransactionHashes, - GetPooledTransactionsMsg: handleGetPooledTransactions, - PooledTransactionsMsg: handlePooledTransactions, -} - var eth66 = map[uint64]msgHandler{ NewBlockHashesMsg: handleNewBlockhashes, NewBlockMsg: handleNewBlock, TransactionsMsg: handleTransactions, NewPooledTransactionHashesMsg: handleNewPooledTransactionHashes, - // eth66 messages with request-id - GetBlockHeadersMsg: handleGetBlockHeaders66, - BlockHeadersMsg: handleBlockHeaders66, - GetBlockBodiesMsg: handleGetBlockBodies66, - BlockBodiesMsg: handleBlockBodies66, - GetNodeDataMsg: handleGetNodeData66, - NodeDataMsg: handleNodeData66, - GetReceiptsMsg: handleGetReceipts66, - ReceiptsMsg: handleReceipts66, - GetPooledTransactionsMsg: handleGetPooledTransactions66, - PooledTransactionsMsg: handlePooledTransactions66, + GetBlockHeadersMsg: handleGetBlockHeaders66, + BlockHeadersMsg: handleBlockHeaders66, + GetBlockBodiesMsg: handleGetBlockBodies66, + BlockBodiesMsg: handleBlockBodies66, + GetNodeDataMsg: handleGetNodeData66, + NodeDataMsg: handleNodeData66, + GetReceiptsMsg: handleGetReceipts66, + ReceiptsMsg: handleReceipts66, + GetPooledTransactionsMsg: handleGetPooledTransactions66, + PooledTransactionsMsg: handlePooledTransactions66, } // handleMessage is invoked whenever an inbound message is received from a remote @@ -219,10 +201,11 @@ func handleMessage(backend Backend, peer *Peer) error { } defer msg.Discard() - var handlers = eth65 - if peer.Version() >= ETH66 { - handlers = eth66 - } + var handlers = eth66 + //if peer.Version() >= ETH67 { // Left in as a sample when new protocol is added + // handlers = eth67 + //} + // Track the amount of time it takes to serve the request and run the handler if metrics.Enabled { h := fmt.Sprintf("%s/%s/%d/%#02x", p2p.HandleHistName, ProtocolName, peer.Version(), msg.Code) diff --git a/eth/protocols/eth/handler_test.go b/eth/protocols/eth/handler_test.go index 473be3f9b..809f17e36 100644 --- a/eth/protocols/eth/handler_test.go +++ b/eth/protocols/eth/handler_test.go @@ -110,7 +110,6 @@ func (b *testBackend) Handle(*Peer, Packet) error { } // Tests that block headers can be retrieved from a remote chain based on user queries. -func TestGetBlockHeaders65(t *testing.T) { testGetBlockHeaders(t, ETH65) } func TestGetBlockHeaders66(t *testing.T) { testGetBlockHeaders(t, ETH66) } func testGetBlockHeaders(t *testing.T, protocol uint) { @@ -254,44 +253,30 @@ func testGetBlockHeaders(t *testing.T, protocol uint) { headers = append(headers, backend.chain.GetBlockByHash(hash).Header()) } // Send the hash request and verify the response - if protocol <= ETH65 { - p2p.Send(peer.app, GetBlockHeadersMsg, tt.query) - if err := p2p.ExpectMsg(peer.app, BlockHeadersMsg, headers); err != nil { - t.Errorf("test %d: headers mismatch: %v", i, err) - } - } else { - p2p.Send(peer.app, GetBlockHeadersMsg, GetBlockHeadersPacket66{ - RequestId: 123, - GetBlockHeadersPacket: tt.query, - }) - if err := p2p.ExpectMsg(peer.app, BlockHeadersMsg, BlockHeadersPacket66{ - RequestId: 123, - BlockHeadersPacket: headers, - }); err != nil { - t.Errorf("test %d: headers mismatch: %v", i, err) - } + p2p.Send(peer.app, GetBlockHeadersMsg, GetBlockHeadersPacket66{ + RequestId: 123, + GetBlockHeadersPacket: tt.query, + }) + if err := p2p.ExpectMsg(peer.app, BlockHeadersMsg, BlockHeadersPacket66{ + RequestId: 123, + BlockHeadersPacket: headers, + }); err != nil { + t.Errorf("test %d: headers mismatch: %v", i, err) } // If the test used number origins, repeat with hashes as the too if tt.query.Origin.Hash == (common.Hash{}) { if origin := backend.chain.GetBlockByNumber(tt.query.Origin.Number); origin != nil { tt.query.Origin.Hash, tt.query.Origin.Number = origin.Hash(), 0 - if protocol <= ETH65 { - p2p.Send(peer.app, GetBlockHeadersMsg, tt.query) - if err := p2p.ExpectMsg(peer.app, BlockHeadersMsg, headers); err != nil { - t.Errorf("test %d: headers mismatch: %v", i, err) - } - } else { - p2p.Send(peer.app, GetBlockHeadersMsg, GetBlockHeadersPacket66{ - RequestId: 456, - GetBlockHeadersPacket: tt.query, - }) - if err := p2p.ExpectMsg(peer.app, BlockHeadersMsg, BlockHeadersPacket66{ - RequestId: 456, - BlockHeadersPacket: headers, - }); err != nil { - t.Errorf("test %d: headers mismatch: %v", i, err) - } + p2p.Send(peer.app, GetBlockHeadersMsg, GetBlockHeadersPacket66{ + RequestId: 456, + GetBlockHeadersPacket: tt.query, + }) + if err := p2p.ExpectMsg(peer.app, BlockHeadersMsg, BlockHeadersPacket66{ + RequestId: 456, + BlockHeadersPacket: headers, + }); err != nil { + t.Errorf("test %d: headers mismatch: %v", i, err) } } } @@ -299,7 +284,6 @@ func testGetBlockHeaders(t *testing.T, protocol uint) { } // Tests that block contents can be retrieved from a remote chain based on their hashes. -func TestGetBlockBodies65(t *testing.T) { testGetBlockBodies(t, ETH65) } func TestGetBlockBodies66(t *testing.T) { testGetBlockBodies(t, ETH66) } func testGetBlockBodies(t *testing.T, protocol uint) { @@ -369,28 +353,20 @@ func testGetBlockBodies(t *testing.T, protocol uint) { } } // Send the hash request and verify the response - if protocol <= ETH65 { - p2p.Send(peer.app, GetBlockBodiesMsg, hashes) - if err := p2p.ExpectMsg(peer.app, BlockBodiesMsg, bodies); err != nil { - t.Errorf("test %d: bodies mismatch: %v", i, err) - } - } else { - p2p.Send(peer.app, GetBlockBodiesMsg, GetBlockBodiesPacket66{ - RequestId: 123, - GetBlockBodiesPacket: hashes, - }) - if err := p2p.ExpectMsg(peer.app, BlockBodiesMsg, BlockBodiesPacket66{ - RequestId: 123, - BlockBodiesPacket: bodies, - }); err != nil { - t.Errorf("test %d: bodies mismatch: %v", i, err) - } + p2p.Send(peer.app, GetBlockBodiesMsg, GetBlockBodiesPacket66{ + RequestId: 123, + GetBlockBodiesPacket: hashes, + }) + if err := p2p.ExpectMsg(peer.app, BlockBodiesMsg, BlockBodiesPacket66{ + RequestId: 123, + BlockBodiesPacket: bodies, + }); err != nil { + t.Errorf("test %d: bodies mismatch: %v", i, err) } } } // Tests that the state trie nodes can be retrieved based on hashes. -func TestGetNodeData65(t *testing.T) { testGetNodeData(t, ETH65) } func TestGetNodeData66(t *testing.T) { testGetNodeData(t, ETH66) } func testGetNodeData(t *testing.T, protocol uint) { @@ -449,14 +425,10 @@ func testGetNodeData(t *testing.T, protocol uint) { } it.Release() - if protocol <= ETH65 { - p2p.Send(peer.app, GetNodeDataMsg, hashes) - } else { - p2p.Send(peer.app, GetNodeDataMsg, GetNodeDataPacket66{ - RequestId: 123, - GetNodeDataPacket: hashes, - }) - } + p2p.Send(peer.app, GetNodeDataMsg, GetNodeDataPacket66{ + RequestId: 123, + GetNodeDataPacket: hashes, + }) msg, err := peer.app.ReadMsg() if err != nil { t.Fatalf("failed to read node data response: %v", err) @@ -464,18 +436,14 @@ func testGetNodeData(t *testing.T, protocol uint) { if msg.Code != NodeDataMsg { t.Fatalf("response packet code mismatch: have %x, want %x", msg.Code, NodeDataMsg) } - var data [][]byte - if protocol <= ETH65 { - if err := msg.Decode(&data); err != nil { - t.Fatalf("failed to decode response node data: %v", err) - } - } else { - var res NodeDataPacket66 - if err := msg.Decode(&res); err != nil { - t.Fatalf("failed to decode response node data: %v", err) - } - data = res.NodeDataPacket + var ( + data [][]byte + res NodeDataPacket66 + ) + if err := msg.Decode(&res); err != nil { + t.Fatalf("failed to decode response node data: %v", err) } + data = res.NodeDataPacket // Verify that all hashes correspond to the requested data, and reconstruct a state tree for i, want := range hashes { if hash := crypto.Keccak256Hash(data[i]); hash != want { @@ -506,7 +474,6 @@ func testGetNodeData(t *testing.T, protocol uint) { } // Tests that the transaction receipts can be retrieved based on hashes. -func TestGetBlockReceipts65(t *testing.T) { testGetBlockReceipts(t, ETH65) } func TestGetBlockReceipts66(t *testing.T) { testGetBlockReceipts(t, ETH66) } func testGetBlockReceipts(t *testing.T, protocol uint) { @@ -566,21 +533,14 @@ func testGetBlockReceipts(t *testing.T, protocol uint) { receipts = append(receipts, backend.chain.GetReceiptsByHash(block.Hash())) } // Send the hash request and verify the response - if protocol <= ETH65 { - p2p.Send(peer.app, GetReceiptsMsg, hashes) - if err := p2p.ExpectMsg(peer.app, ReceiptsMsg, receipts); err != nil { - t.Errorf("receipts mismatch: %v", err) - } - } else { - p2p.Send(peer.app, GetReceiptsMsg, GetReceiptsPacket66{ - RequestId: 123, - GetReceiptsPacket: hashes, - }) - if err := p2p.ExpectMsg(peer.app, ReceiptsMsg, ReceiptsPacket66{ - RequestId: 123, - ReceiptsPacket: receipts, - }); err != nil { - t.Errorf("receipts mismatch: %v", err) - } + p2p.Send(peer.app, GetReceiptsMsg, GetReceiptsPacket66{ + RequestId: 123, + GetReceiptsPacket: hashes, + }) + if err := p2p.ExpectMsg(peer.app, ReceiptsMsg, ReceiptsPacket66{ + RequestId: 123, + ReceiptsPacket: receipts, + }); err != nil { + t.Errorf("receipts mismatch: %v", err) } } diff --git a/eth/protocols/eth/handlers.go b/eth/protocols/eth/handlers.go index d7d993a23..e54838cbc 100644 --- a/eth/protocols/eth/handlers.go +++ b/eth/protocols/eth/handlers.go @@ -27,17 +27,6 @@ import ( "github.com/ethereum/go-ethereum/trie" ) -// handleGetBlockHeaders handles Block header query, collect the requested headers and reply -func handleGetBlockHeaders(backend Backend, msg Decoder, peer *Peer) error { - // Decode the complex header query - var query GetBlockHeadersPacket - if err := msg.Decode(&query); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - response := answerGetBlockHeadersQuery(backend, &query, peer) - return peer.SendBlockHeaders(response) -} - // handleGetBlockHeaders66 is the eth/66 version of handleGetBlockHeaders func handleGetBlockHeaders66(backend Backend, msg Decoder, peer *Peer) error { // Decode the complex header query @@ -135,16 +124,6 @@ func answerGetBlockHeadersQuery(backend Backend, query *GetBlockHeadersPacket, p return headers } -func handleGetBlockBodies(backend Backend, msg Decoder, peer *Peer) error { - // Decode the block body retrieval message - var query GetBlockBodiesPacket - if err := msg.Decode(&query); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - response := answerGetBlockBodiesQuery(backend, query, peer) - return peer.SendBlockBodiesRLP(response) -} - func handleGetBlockBodies66(backend Backend, msg Decoder, peer *Peer) error { // Decode the block body retrieval message var query GetBlockBodiesPacket66 @@ -174,16 +153,6 @@ func answerGetBlockBodiesQuery(backend Backend, query GetBlockBodiesPacket, peer return bodies } -func handleGetNodeData(backend Backend, msg Decoder, peer *Peer) error { - // Decode the trie node data retrieval message - var query GetNodeDataPacket - if err := msg.Decode(&query); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - response := answerGetNodeDataQuery(backend, query, peer) - return peer.SendNodeData(response) -} - func handleGetNodeData66(backend Backend, msg Decoder, peer *Peer) error { // Decode the trie node data retrieval message var query GetNodeDataPacket66 @@ -223,16 +192,6 @@ func answerGetNodeDataQuery(backend Backend, query GetNodeDataPacket, peer *Peer return nodes } -func handleGetReceipts(backend Backend, msg Decoder, peer *Peer) error { - // Decode the block receipts retrieval message - var query GetReceiptsPacket - if err := msg.Decode(&query); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - response := answerGetReceiptsQuery(backend, query, peer) - return peer.SendReceiptsRLP(response) -} - func handleGetReceipts66(backend Backend, msg Decoder, peer *Peer) error { // Decode the block receipts retrieval message var query GetReceiptsPacket66 @@ -312,15 +271,6 @@ func handleNewBlock(backend Backend, msg Decoder, peer *Peer) error { return backend.Handle(peer, ann) } -func handleBlockHeaders(backend Backend, msg Decoder, peer *Peer) error { - // A batch of headers arrived to one of our previous requests - res := new(BlockHeadersPacket) - if err := msg.Decode(res); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - return backend.Handle(peer, res) -} - func handleBlockHeaders66(backend Backend, msg Decoder, peer *Peer) error { // A batch of headers arrived to one of our previous requests res := new(BlockHeadersPacket66) @@ -332,15 +282,6 @@ func handleBlockHeaders66(backend Backend, msg Decoder, peer *Peer) error { return backend.Handle(peer, &res.BlockHeadersPacket) } -func handleBlockBodies(backend Backend, msg Decoder, peer *Peer) error { - // A batch of block bodies arrived to one of our previous requests - res := new(BlockBodiesPacket) - if err := msg.Decode(res); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - return backend.Handle(peer, res) -} - func handleBlockBodies66(backend Backend, msg Decoder, peer *Peer) error { // A batch of block bodies arrived to one of our previous requests res := new(BlockBodiesPacket66) @@ -352,15 +293,6 @@ func handleBlockBodies66(backend Backend, msg Decoder, peer *Peer) error { return backend.Handle(peer, &res.BlockBodiesPacket) } -func handleNodeData(backend Backend, msg Decoder, peer *Peer) error { - // A batch of node state data arrived to one of our previous requests - res := new(NodeDataPacket) - if err := msg.Decode(res); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - return backend.Handle(peer, res) -} - func handleNodeData66(backend Backend, msg Decoder, peer *Peer) error { // A batch of node state data arrived to one of our previous requests res := new(NodeDataPacket66) @@ -372,15 +304,6 @@ func handleNodeData66(backend Backend, msg Decoder, peer *Peer) error { return backend.Handle(peer, &res.NodeDataPacket) } -func handleReceipts(backend Backend, msg Decoder, peer *Peer) error { - // A batch of receipts arrived to one of our previous requests - res := new(ReceiptsPacket) - if err := msg.Decode(res); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - return backend.Handle(peer, res) -} - func handleReceipts66(backend Backend, msg Decoder, peer *Peer) error { // A batch of receipts arrived to one of our previous requests res := new(ReceiptsPacket66) @@ -409,16 +332,6 @@ func handleNewPooledTransactionHashes(backend Backend, msg Decoder, peer *Peer) return backend.Handle(peer, ann) } -func handleGetPooledTransactions(backend Backend, msg Decoder, peer *Peer) error { - // Decode the pooled transactions retrieval message - var query GetPooledTransactionsPacket - if err := msg.Decode(&query); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - hashes, txs := answerGetPooledTransactions(backend, query, peer) - return peer.SendPooledTransactionsRLP(hashes, txs) -} - func handleGetPooledTransactions66(backend Backend, msg Decoder, peer *Peer) error { // Decode the pooled transactions retrieval message var query GetPooledTransactionsPacket66 @@ -477,26 +390,6 @@ func handleTransactions(backend Backend, msg Decoder, peer *Peer) error { return backend.Handle(peer, &txs) } -func handlePooledTransactions(backend Backend, msg Decoder, peer *Peer) error { - // Transactions arrived, make sure we have a valid and fresh chain to handle them - if !backend.AcceptTxs() { - return nil - } - // Transactions can be processed, parse all of them and deliver to the pool - var txs PooledTransactionsPacket - if err := msg.Decode(&txs); err != nil { - return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) - } - for i, tx := range txs { - // Validate and mark the remote transaction - if tx == nil { - return fmt.Errorf("%w: transaction %d is nil", errDecode, i) - } - peer.markTransaction(tx.Hash()) - } - return backend.Handle(peer, &txs) -} - func handlePooledTransactions66(backend Backend, msg Decoder, peer *Peer) error { // Transactions arrived, make sure we have a valid and fresh chain to handle them if !backend.AcceptTxs() { diff --git a/eth/protocols/eth/handshake_test.go b/eth/protocols/eth/handshake_test.go index 3bebda2dc..05d473e05 100644 --- a/eth/protocols/eth/handshake_test.go +++ b/eth/protocols/eth/handshake_test.go @@ -27,7 +27,6 @@ import ( ) // Tests that handshake failures are detected and reported correctly. -func TestHandshake65(t *testing.T) { testHandshake(t, ETH65) } func TestHandshake66(t *testing.T) { testHandshake(t, ETH66) } func testHandshake(t *testing.T, protocol uint) { diff --git a/eth/protocols/eth/peer.go b/eth/protocols/eth/peer.go index e619c183b..98273ccfc 100644 --- a/eth/protocols/eth/peer.go +++ b/eth/protocols/eth/peer.go @@ -108,9 +108,8 @@ func NewPeer(version uint, p *p2p.Peer, rw p2p.MsgReadWriter, txpool TxPool) *Pe // Start up all the broadcasters go peer.broadcastBlocks() go peer.broadcastTransactions() - if version >= ETH65 { - go peer.announceTransactions() - } + go peer.announceTransactions() + return peer } @@ -252,22 +251,6 @@ func (p *Peer) AsyncSendPooledTransactionHashes(hashes []common.Hash) { } } -// SendPooledTransactionsRLP sends requested transactions to the peer and adds the -// hashes in its transaction hash set for future reference. -// -// Note, the method assumes the hashes are correct and correspond to the list of -// transactions being sent. -func (p *Peer) SendPooledTransactionsRLP(hashes []common.Hash, txs []rlp.RawValue) error { - // Mark all the transactions as known, but ensure we don't overflow our limits - for p.knownTxs.Cardinality() > max(0, maxKnownTxs-len(hashes)) { - p.knownTxs.Pop() - } - for _, hash := range hashes { - p.knownTxs.Add(hash) - } - return p2p.Send(p.rw, PooledTransactionsMsg, txs) // Not packed into PooledTransactionsPacket to avoid RLP decoding -} - // ReplyPooledTransactionsRLP is the eth/66 version of SendPooledTransactionsRLP. func (p *Peer) ReplyPooledTransactionsRLP(id uint64, hashes []common.Hash, txs []rlp.RawValue) error { // Mark all the transactions as known, but ensure we don't overflow our limits @@ -346,11 +329,6 @@ func (p *Peer) AsyncSendNewBlock(block *types.Block, td *big.Int) { } } -// SendBlockHeaders sends a batch of block headers to the remote peer. -func (p *Peer) SendBlockHeaders(headers []*types.Header) error { - return p2p.Send(p.rw, BlockHeadersMsg, BlockHeadersPacket(headers)) -} - // ReplyBlockHeaders is the eth/66 version of SendBlockHeaders. func (p *Peer) ReplyBlockHeaders(id uint64, headers []*types.Header) error { return p2p.Send(p.rw, BlockHeadersMsg, BlockHeadersPacket66{ @@ -359,12 +337,6 @@ func (p *Peer) ReplyBlockHeaders(id uint64, headers []*types.Header) error { }) } -// SendBlockBodiesRLP sends a batch of block contents to the remote peer from -// an already RLP encoded format. -func (p *Peer) SendBlockBodiesRLP(bodies []rlp.RawValue) error { - return p2p.Send(p.rw, BlockBodiesMsg, bodies) // Not packed into BlockBodiesPacket to avoid RLP decoding -} - // ReplyBlockBodiesRLP is the eth/66 version of SendBlockBodiesRLP. func (p *Peer) ReplyBlockBodiesRLP(id uint64, bodies []rlp.RawValue) error { // Not packed into BlockBodiesPacket to avoid RLP decoding @@ -374,12 +346,6 @@ func (p *Peer) ReplyBlockBodiesRLP(id uint64, bodies []rlp.RawValue) error { }) } -// SendNodeDataRLP sends a batch of arbitrary internal data, corresponding to the -// hashes requested. -func (p *Peer) SendNodeData(data [][]byte) error { - return p2p.Send(p.rw, NodeDataMsg, NodeDataPacket(data)) -} - // ReplyNodeData is the eth/66 response to GetNodeData. func (p *Peer) ReplyNodeData(id uint64, data [][]byte) error { return p2p.Send(p.rw, NodeDataMsg, NodeDataPacket66{ @@ -388,12 +354,6 @@ func (p *Peer) ReplyNodeData(id uint64, data [][]byte) error { }) } -// SendReceiptsRLP sends a batch of transaction receipts, corresponding to the -// ones requested from an already RLP encoded format. -func (p *Peer) SendReceiptsRLP(receipts []rlp.RawValue) error { - return p2p.Send(p.rw, ReceiptsMsg, receipts) // Not packed into ReceiptsPacket to avoid RLP decoding -} - // ReplyReceiptsRLP is the eth/66 response to GetReceipts. func (p *Peer) ReplyReceiptsRLP(id uint64, receipts []rlp.RawValue) error { return p2p.Send(p.rw, ReceiptsMsg, ReceiptsRLPPacket66{ @@ -406,138 +366,102 @@ func (p *Peer) ReplyReceiptsRLP(id uint64, receipts []rlp.RawValue) error { // single header. It is used solely by the fetcher. func (p *Peer) RequestOneHeader(hash common.Hash) error { p.Log().Debug("Fetching single header", "hash", hash) - query := GetBlockHeadersPacket{ - Origin: HashOrNumber{Hash: hash}, - Amount: uint64(1), - Skip: uint64(0), - Reverse: false, - } - if p.Version() >= ETH66 { - id := rand.Uint64() + id := rand.Uint64() - requestTracker.Track(p.id, p.version, GetBlockHeadersMsg, BlockHeadersMsg, id) - return p2p.Send(p.rw, GetBlockHeadersMsg, &GetBlockHeadersPacket66{ - RequestId: id, - GetBlockHeadersPacket: &query, - }) - } - return p2p.Send(p.rw, GetBlockHeadersMsg, &query) + requestTracker.Track(p.id, p.version, GetBlockHeadersMsg, BlockHeadersMsg, id) + return p2p.Send(p.rw, GetBlockHeadersMsg, &GetBlockHeadersPacket66{ + RequestId: id, + GetBlockHeadersPacket: &GetBlockHeadersPacket{ + Origin: HashOrNumber{Hash: hash}, + Amount: uint64(1), + Skip: uint64(0), + Reverse: false, + }, + }) } // RequestHeadersByHash fetches a batch of blocks' headers corresponding to the // specified header query, based on the hash of an origin block. func (p *Peer) RequestHeadersByHash(origin common.Hash, amount int, skip int, reverse bool) error { p.Log().Debug("Fetching batch of headers", "count", amount, "fromhash", origin, "skip", skip, "reverse", reverse) - query := GetBlockHeadersPacket{ - Origin: HashOrNumber{Hash: origin}, - Amount: uint64(amount), - Skip: uint64(skip), - Reverse: reverse, - } - if p.Version() >= ETH66 { - id := rand.Uint64() + id := rand.Uint64() - requestTracker.Track(p.id, p.version, GetBlockHeadersMsg, BlockHeadersMsg, id) - return p2p.Send(p.rw, GetBlockHeadersMsg, &GetBlockHeadersPacket66{ - RequestId: id, - GetBlockHeadersPacket: &query, - }) - } - return p2p.Send(p.rw, GetBlockHeadersMsg, &query) + requestTracker.Track(p.id, p.version, GetBlockHeadersMsg, BlockHeadersMsg, id) + return p2p.Send(p.rw, GetBlockHeadersMsg, &GetBlockHeadersPacket66{ + RequestId: id, + GetBlockHeadersPacket: &GetBlockHeadersPacket{ + Origin: HashOrNumber{Hash: origin}, + Amount: uint64(amount), + Skip: uint64(skip), + Reverse: reverse, + }, + }) } // RequestHeadersByNumber fetches a batch of blocks' headers corresponding to the // specified header query, based on the number of an origin block. func (p *Peer) RequestHeadersByNumber(origin uint64, amount int, skip int, reverse bool) error { p.Log().Debug("Fetching batch of headers", "count", amount, "fromnum", origin, "skip", skip, "reverse", reverse) - query := GetBlockHeadersPacket{ - Origin: HashOrNumber{Number: origin}, - Amount: uint64(amount), - Skip: uint64(skip), - Reverse: reverse, - } - if p.Version() >= ETH66 { - id := rand.Uint64() + id := rand.Uint64() - requestTracker.Track(p.id, p.version, GetBlockHeadersMsg, BlockHeadersMsg, id) - return p2p.Send(p.rw, GetBlockHeadersMsg, &GetBlockHeadersPacket66{ - RequestId: id, - GetBlockHeadersPacket: &query, - }) - } - return p2p.Send(p.rw, GetBlockHeadersMsg, &query) -} - -// ExpectRequestHeadersByNumber is a testing method to mirror the recipient side -// of the RequestHeadersByNumber operation. -func (p *Peer) ExpectRequestHeadersByNumber(origin uint64, amount int, skip int, reverse bool) error { - req := &GetBlockHeadersPacket{ - Origin: HashOrNumber{Number: origin}, - Amount: uint64(amount), - Skip: uint64(skip), - Reverse: reverse, - } - return p2p.ExpectMsg(p.rw, GetBlockHeadersMsg, req) + requestTracker.Track(p.id, p.version, GetBlockHeadersMsg, BlockHeadersMsg, id) + return p2p.Send(p.rw, GetBlockHeadersMsg, &GetBlockHeadersPacket66{ + RequestId: id, + GetBlockHeadersPacket: &GetBlockHeadersPacket{ + Origin: HashOrNumber{Number: origin}, + Amount: uint64(amount), + Skip: uint64(skip), + Reverse: reverse, + }, + }) } // RequestBodies fetches a batch of blocks' bodies corresponding to the hashes // specified. func (p *Peer) RequestBodies(hashes []common.Hash) error { p.Log().Debug("Fetching batch of block bodies", "count", len(hashes)) - if p.Version() >= ETH66 { - id := rand.Uint64() + id := rand.Uint64() - requestTracker.Track(p.id, p.version, GetBlockBodiesMsg, BlockBodiesMsg, id) - return p2p.Send(p.rw, GetBlockBodiesMsg, &GetBlockBodiesPacket66{ - RequestId: id, - GetBlockBodiesPacket: hashes, - }) - } - return p2p.Send(p.rw, GetBlockBodiesMsg, GetBlockBodiesPacket(hashes)) + requestTracker.Track(p.id, p.version, GetBlockBodiesMsg, BlockBodiesMsg, id) + return p2p.Send(p.rw, GetBlockBodiesMsg, &GetBlockBodiesPacket66{ + RequestId: id, + GetBlockBodiesPacket: hashes, + }) } // RequestNodeData fetches a batch of arbitrary data from a node's known state // data, corresponding to the specified hashes. func (p *Peer) RequestNodeData(hashes []common.Hash) error { p.Log().Debug("Fetching batch of state data", "count", len(hashes)) - if p.Version() >= ETH66 { - id := rand.Uint64() + id := rand.Uint64() - requestTracker.Track(p.id, p.version, GetNodeDataMsg, NodeDataMsg, id) - return p2p.Send(p.rw, GetNodeDataMsg, &GetNodeDataPacket66{ - RequestId: id, - GetNodeDataPacket: hashes, - }) - } - return p2p.Send(p.rw, GetNodeDataMsg, GetNodeDataPacket(hashes)) + requestTracker.Track(p.id, p.version, GetNodeDataMsg, NodeDataMsg, id) + return p2p.Send(p.rw, GetNodeDataMsg, &GetNodeDataPacket66{ + RequestId: id, + GetNodeDataPacket: hashes, + }) } // RequestReceipts fetches a batch of transaction receipts from a remote node. func (p *Peer) RequestReceipts(hashes []common.Hash) error { p.Log().Debug("Fetching batch of receipts", "count", len(hashes)) - if p.Version() >= ETH66 { - id := rand.Uint64() + id := rand.Uint64() - requestTracker.Track(p.id, p.version, GetReceiptsMsg, ReceiptsMsg, id) - return p2p.Send(p.rw, GetReceiptsMsg, &GetReceiptsPacket66{ - RequestId: id, - GetReceiptsPacket: hashes, - }) - } - return p2p.Send(p.rw, GetReceiptsMsg, GetReceiptsPacket(hashes)) + requestTracker.Track(p.id, p.version, GetReceiptsMsg, ReceiptsMsg, id) + return p2p.Send(p.rw, GetReceiptsMsg, &GetReceiptsPacket66{ + RequestId: id, + GetReceiptsPacket: hashes, + }) } // RequestTxs fetches a batch of transactions from a remote node. func (p *Peer) RequestTxs(hashes []common.Hash) error { p.Log().Debug("Fetching batch of transactions", "count", len(hashes)) - if p.Version() >= ETH66 { - id := rand.Uint64() + id := rand.Uint64() - requestTracker.Track(p.id, p.version, GetPooledTransactionsMsg, PooledTransactionsMsg, id) - return p2p.Send(p.rw, GetPooledTransactionsMsg, &GetPooledTransactionsPacket66{ - RequestId: id, - GetPooledTransactionsPacket: hashes, - }) - } - return p2p.Send(p.rw, GetPooledTransactionsMsg, GetPooledTransactionsPacket(hashes)) + requestTracker.Track(p.id, p.version, GetPooledTransactionsMsg, PooledTransactionsMsg, id) + return p2p.Send(p.rw, GetPooledTransactionsMsg, &GetPooledTransactionsPacket66{ + RequestId: id, + GetPooledTransactionsPacket: hashes, + }) } diff --git a/eth/protocols/eth/protocol.go b/eth/protocols/eth/protocol.go index de1b0ed1e..3c3da30fa 100644 --- a/eth/protocols/eth/protocol.go +++ b/eth/protocols/eth/protocol.go @@ -30,7 +30,6 @@ import ( // Constants to match up protocol versions and messages const ( - ETH65 = 65 ETH66 = 66 ) @@ -40,31 +39,28 @@ const ProtocolName = "eth" // ProtocolVersions are the supported versions of the `eth` protocol (first // is primary). -var ProtocolVersions = []uint{ETH66, ETH65} +var ProtocolVersions = []uint{ETH66} // protocolLengths are the number of implemented message corresponding to // different protocol versions. -var protocolLengths = map[uint]uint64{ETH66: 17, ETH65: 17} +var protocolLengths = map[uint]uint64{ETH66: 17} // maxMessageSize is the maximum cap on the size of a protocol message. const maxMessageSize = 10 * 1024 * 1024 const ( - // Protocol messages in eth/64 - StatusMsg = 0x00 - NewBlockHashesMsg = 0x01 - TransactionsMsg = 0x02 - GetBlockHeadersMsg = 0x03 - BlockHeadersMsg = 0x04 - GetBlockBodiesMsg = 0x05 - BlockBodiesMsg = 0x06 - NewBlockMsg = 0x07 - GetNodeDataMsg = 0x0d - NodeDataMsg = 0x0e - GetReceiptsMsg = 0x0f - ReceiptsMsg = 0x10 - - // Protocol messages overloaded in eth/65 + StatusMsg = 0x00 + NewBlockHashesMsg = 0x01 + TransactionsMsg = 0x02 + GetBlockHeadersMsg = 0x03 + BlockHeadersMsg = 0x04 + GetBlockBodiesMsg = 0x05 + BlockBodiesMsg = 0x06 + NewBlockMsg = 0x07 + GetNodeDataMsg = 0x0d + NodeDataMsg = 0x0e + GetReceiptsMsg = 0x0f + ReceiptsMsg = 0x10 NewPooledTransactionHashesMsg = 0x08 GetPooledTransactionsMsg = 0x09 PooledTransactionsMsg = 0x0a @@ -128,7 +124,7 @@ type GetBlockHeadersPacket struct { Reverse bool // Query direction (false = rising towards latest, true = falling towards genesis) } -// GetBlockHeadersPacket represents a block header query over eth/66 +// GetBlockHeadersPacket66 represents a block header query over eth/66 type GetBlockHeadersPacket66 struct { RequestId uint64 *GetBlockHeadersPacket diff --git a/eth/sync.go b/eth/sync.go index ab114b59f..27941158f 100644 --- a/eth/sync.go +++ b/eth/sync.go @@ -18,7 +18,6 @@ package eth import ( "math/big" - "math/rand" "sync/atomic" "time" @@ -28,23 +27,13 @@ import ( "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/log" - "github.com/ethereum/go-ethereum/p2p/enode" ) const ( forceSyncCycle = 10 * time.Second // Time interval to force syncs, even if few peers are available defaultMinSyncPeers = 5 // Amount of peers desired to start syncing - - // This is the target size for the packs of transactions sent by txsyncLoop64. - // A pack can get larger than this if a single transactions exceeds this size. - txsyncPackSize = 100 * 1024 ) -type txsync struct { - p *eth.Peer - txs []*types.Transaction -} - // syncTransactions starts sending all currently pending transactions to the given peer. func (h *handler) syncTransactions(p *eth.Peer) { // Assemble the set of transaction to broadcast or announce to the remote @@ -64,94 +53,11 @@ func (h *handler) syncTransactions(p *eth.Peer) { // The eth/65 protocol introduces proper transaction announcements, so instead // of dripping transactions across multiple peers, just send the entire list as // an announcement and let the remote side decide what they need (likely nothing). - if p.Version() >= eth.ETH65 { - hashes := make([]common.Hash, len(txs)) - for i, tx := range txs { - hashes[i] = tx.Hash() - } - p.AsyncSendPooledTransactionHashes(hashes) - return - } - // Out of luck, peer is running legacy protocols, drop the txs over - select { - case h.txsyncCh <- &txsync{p: p, txs: txs}: - case <-h.quitSync: - } -} - -// txsyncLoop64 takes care of the initial transaction sync for each new -// connection. When a new peer appears, we relay all currently pending -// transactions. In order to minimise egress bandwidth usage, we send -// the transactions in small packs to one peer at a time. -func (h *handler) txsyncLoop64() { - defer h.wg.Done() - - var ( - pending = make(map[enode.ID]*txsync) - sending = false // whether a send is active - pack = new(txsync) // the pack that is being sent - done = make(chan error, 1) // result of the send - ) - - // send starts a sending a pack of transactions from the sync. - send := func(s *txsync) { - if s.p.Version() >= eth.ETH65 { - panic("initial transaction syncer running on eth/65+") - } - // Fill pack with transactions up to the target size. - size := common.StorageSize(0) - pack.p = s.p - pack.txs = pack.txs[:0] - for i := 0; i < len(s.txs) && size < txsyncPackSize; i++ { - pack.txs = append(pack.txs, s.txs[i]) - size += s.txs[i].Size() - } - // Remove the transactions that will be sent. - s.txs = s.txs[:copy(s.txs, s.txs[len(pack.txs):])] - if len(s.txs) == 0 { - delete(pending, s.p.Peer.ID()) - } - // Send the pack in the background. - s.p.Log().Trace("Sending batch of transactions", "count", len(pack.txs), "bytes", size) - sending = true - go func() { done <- pack.p.SendTransactions(pack.txs) }() - } - // pick chooses the next pending sync. - pick := func() *txsync { - if len(pending) == 0 { - return nil - } - n := rand.Intn(len(pending)) + 1 - for _, s := range pending { - if n--; n == 0 { - return s - } - } - return nil - } - - for { - select { - case s := <-h.txsyncCh: - pending[s.p.Peer.ID()] = s - if !sending { - send(s) - } - case err := <-done: - sending = false - // Stop tracking peers that cause send failures. - if err != nil { - pack.p.Log().Debug("Transaction send failed", "err", err) - delete(pending, pack.p.Peer.ID()) - } - // Schedule the next send. - if s := pick(); s != nil { - send(s) - } - case <-h.quitSync: - return - } + hashes := make([]common.Hash, len(txs)) + for i, tx := range txs { + hashes[i] = tx.Hash() } + p.AsyncSendPooledTransactionHashes(hashes) } // chainSyncer coordinates blockchain sync components. diff --git a/eth/sync_test.go b/eth/sync_test.go index a0c6f8602..e96b9ee81 100644 --- a/eth/sync_test.go +++ b/eth/sync_test.go @@ -28,7 +28,6 @@ import ( ) // Tests that fast sync is disabled after a successful sync cycle. -func TestFastSyncDisabling65(t *testing.T) { testFastSyncDisabling(t, eth.ETH65) } func TestFastSyncDisabling66(t *testing.T) { testFastSyncDisabling(t, eth.ETH66) } // Tests that fast sync gets disabled as soon as a real block is successfully diff --git a/les/client_handler.go b/les/client_handler.go index e95996c51..b903b1106 100644 --- a/les/client_handler.go +++ b/les/client_handler.go @@ -472,7 +472,7 @@ func (d *downloaderPeerNotify) registerPeer(p *serverPeer) { handler: h, peer: p, } - h.downloader.RegisterLightPeer(p.id, eth.ETH65, pc) + h.downloader.RegisterLightPeer(p.id, eth.ETH66, pc) } func (d *downloaderPeerNotify) unregisterPeer(p *serverPeer) { From a580f7d6c54812ef47df94c6ffc974c9dbc48245 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Thu, 12 Aug 2021 10:15:49 +0300 Subject: [PATCH 02/17] params: begin v1.10.8 release cycle --- params/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/params/version.go b/params/version.go index 2d3155d35..3c50a9e8d 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 10 // Minor version component of the current release - VersionPatch = 7 // Patch version component of the current release - VersionMeta = "stable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 10 // Minor version component of the current release + VersionPatch = 8 // Patch version component of the current release + VersionMeta = "unstable" // Version metadata to append to the version string ) // Version holds the textual version string. From 5f98020a21d64fa22516c04aaf3bc4ed0629e7cc Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Fri, 13 Aug 2021 10:51:01 +0200 Subject: [PATCH 03/17] core/rawdb: implement sequential reads in freezer_table (#23117) * core/rawdb: implement sequential reads in freezer_table * core/rawdb, ethdb: add sequential reader to db interface * core/rawdb: lint nitpicks * core/rawdb: fix some nitpicks * core/rawdb: fix flaw with deferred reads not being performed * core/rawdb: better documentation --- core/rawdb/database.go | 5 + core/rawdb/freezer.go | 12 ++ core/rawdb/freezer_table.go | 214 +++++++++++++++++++++++-------- core/rawdb/freezer_table_test.go | 117 ++++++++++++++++- core/rawdb/table.go | 6 + ethdb/database.go | 7 + 6 files changed, 309 insertions(+), 52 deletions(-) diff --git a/core/rawdb/database.go b/core/rawdb/database.go index c8bfdbace..90619169a 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -89,6 +89,11 @@ func (db *nofreezedb) Ancient(kind string, number uint64) ([]byte, error) { return nil, errNotSupported } +// ReadAncients returns an error as we don't have a backing chain freezer. +func (db *nofreezedb) ReadAncients(kind string, start, max, maxByteSize uint64) ([][]byte, error) { + return nil, errNotSupported +} + // Ancients returns an error as we don't have a backing chain freezer. func (db *nofreezedb) Ancients() (uint64, error) { return 0, errNotSupported diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index ff8919b59..253de9f7c 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -180,6 +180,18 @@ func (f *freezer) Ancient(kind string, number uint64) ([]byte, error) { return nil, errUnknownTable } +// ReadAncients retrieves multiple items in sequence, starting from the index 'start'. +// It will return +// - at most 'max' items, +// - at least 1 item (even if exceeding the maxByteSize), but will otherwise +// return as many items as fit into maxByteSize. +func (f *freezer) ReadAncients(kind string, start, count, maxBytes uint64) ([][]byte, error) { + if table := f.tables[kind]; table != nil { + return table.RetrieveItems(start, count, maxBytes) + } + return nil, errUnknownTable +} + // Ancients returns the length of the frozen items. func (f *freezer) Ancients() (uint64, error) { return atomic.LoadUint64(&f.frozen), nil diff --git a/core/rawdb/freezer_table.go b/core/rawdb/freezer_table.go index d7bfe18e0..9d052f7cd 100644 --- a/core/rawdb/freezer_table.go +++ b/core/rawdb/freezer_table.go @@ -70,6 +70,19 @@ func (i *indexEntry) marshallBinary() []byte { return b } +// bounds returns the start- and end- offsets, and the file number of where to +// read there data item marked by the two index entries. The two entries are +// assumed to be sequential. +func (start *indexEntry) bounds(end *indexEntry) (startOffset, endOffset, fileId uint32) { + if start.filenum != end.filenum { + // If a piece of data 'crosses' a data-file, + // it's actually in one piece on the second data-file. + // We return a zero-indexEntry for the second file as start + return 0, end.offset, end.filenum + } + return start.offset, end.offset, end.filenum +} + // freezerTable represents a single chained data table within the freezer (e.g. blocks). // It consists of a data file (snappy encoded arbitrary data blobs) and an indexEntry // file (uncompressed 64 bit indices into the data file). @@ -546,84 +559,183 @@ func (t *freezerTable) append(item uint64, encodedBlob []byte, wlock bool) (bool return false, nil } -// getBounds returns the indexes for the item -// returns start, end, filenumber and error -func (t *freezerTable) getBounds(item uint64) (uint32, uint32, uint32, error) { - buffer := make([]byte, indexEntrySize) - var startIdx, endIdx indexEntry - // Read second index - if _, err := t.index.ReadAt(buffer, int64((item+1)*indexEntrySize)); err != nil { - return 0, 0, 0, err +// getIndices returns the index entries for the given from-item, covering 'count' items. +// N.B: The actual number of returned indices for N items will always be N+1 (unless an +// error is returned). +// OBS: This method assumes that the caller has already verified (and/or trimmed) the range +// so that the items are within bounds. If this method is used to read out of bounds, +// it will return error. +func (t *freezerTable) getIndices(from, count uint64) ([]*indexEntry, error) { + // Apply the table-offset + from = from - uint64(t.itemOffset) + // For reading N items, we need N+1 indices. + buffer := make([]byte, (count+1)*indexEntrySize) + if _, err := t.index.ReadAt(buffer, int64(from*indexEntrySize)); err != nil { + return nil, err } - endIdx.unmarshalBinary(buffer) - // Read first index (unless it's the very first item) - if item != 0 { - if _, err := t.index.ReadAt(buffer, int64(item*indexEntrySize)); err != nil { - return 0, 0, 0, err - } - startIdx.unmarshalBinary(buffer) - } else { + var ( + indices []*indexEntry + offset int + ) + for i := from; i <= from+count; i++ { + index := new(indexEntry) + index.unmarshalBinary(buffer[offset:]) + offset += indexEntrySize + indices = append(indices, index) + } + if from == 0 { // Special case if we're reading the first item in the freezer. We assume that // the first item always start from zero(regarding the deletion, we // only support deletion by files, so that the assumption is held). // This means we can use the first item metadata to carry information about // the 'global' offset, for the deletion-case - return 0, endIdx.offset, endIdx.filenum, nil + indices[0].offset = 0 + indices[0].filenum = indices[1].filenum } - if startIdx.filenum != endIdx.filenum { - // If a piece of data 'crosses' a data-file, - // it's actually in one piece on the second data-file. - // We return a zero-indexEntry for the second file as start - return 0, endIdx.offset, endIdx.filenum, nil - } - return startIdx.offset, endIdx.offset, endIdx.filenum, nil + return indices, nil } // Retrieve looks up the data offset of an item with the given number and retrieves // the raw binary blob from the data file. func (t *freezerTable) Retrieve(item uint64) ([]byte, error) { - blob, err := t.retrieve(item) + items, err := t.RetrieveItems(item, 1, 0) if err != nil { return nil, err } - if t.noCompression { - return blob, nil - } - return snappy.Decode(nil, blob) + return items[0], nil } -// retrieve looks up the data offset of an item with the given number and retrieves -// the raw binary blob from the data file. OBS! This method does not decode -// compressed data. -func (t *freezerTable) retrieve(item uint64) ([]byte, error) { +// RetrieveItems returns multiple items in sequence, starting from the index 'start'. +// It will return at most 'max' items, but will abort earlier to respect the +// 'maxBytes' argument. However, if the 'maxBytes' is smaller than the size of one +// item, it _will_ return one element and possibly overflow the maxBytes. +func (t *freezerTable) RetrieveItems(start, count, maxBytes uint64) ([][]byte, error) { + // First we read the 'raw' data, which might be compressed. + diskData, sizes, err := t.retrieveItems(start, count, maxBytes) + if err != nil { + return nil, err + } + var ( + output = make([][]byte, 0, count) + offset int // offset for reading + outputSize int // size of uncompressed data + ) + // Now slice up the data and decompress. + for i, diskSize := range sizes { + item := diskData[offset : offset+diskSize] + offset += diskSize + decompressedSize := diskSize + if !t.noCompression { + decompressedSize, _ = snappy.DecodedLen(item) + } + if i > 0 && uint64(outputSize+decompressedSize) > maxBytes { + break + } + if !t.noCompression { + data, err := snappy.Decode(nil, item) + if err != nil { + return nil, err + } + output = append(output, data) + } else { + output = append(output, item) + } + outputSize += decompressedSize + } + return output, nil +} + +// retrieveItems reads up to 'count' items from the table. It reads at least +// one item, but otherwise avoids reading more than maxBytes bytes. +// It returns the (potentially compressed) data, and the sizes. +func (t *freezerTable) retrieveItems(start, count, maxBytes uint64) ([]byte, []int, error) { t.lock.RLock() defer t.lock.RUnlock() // Ensure the table and the item is accessible if t.index == nil || t.head == nil { - return nil, errClosed + return nil, nil, errClosed } - if atomic.LoadUint64(&t.items) <= item { - return nil, errOutOfBounds + itemCount := atomic.LoadUint64(&t.items) // max number + // Ensure the start is written, not deleted from the tail, and that the + // caller actually wants something + if itemCount <= start || uint64(t.itemOffset) > start || count == 0 { + return nil, nil, errOutOfBounds } - // Ensure the item was not deleted from the tail either - if uint64(t.itemOffset) > item { - return nil, errOutOfBounds + if start+count > itemCount { + count = itemCount - start } - startOffset, endOffset, filenum, err := t.getBounds(item - uint64(t.itemOffset)) + var ( + output = make([]byte, maxBytes) // Buffer to read data into + outputSize int // Used size of that buffer + ) + // readData is a helper method to read a single data item from disk. + readData := func(fileId, start uint32, length int) error { + // In case a small limit is used, and the elements are large, may need to + // realloc the read-buffer when reading the first (and only) item. + if len(output) < length { + output = make([]byte, length) + } + dataFile, exist := t.files[fileId] + if !exist { + return fmt.Errorf("missing data file %d", fileId) + } + if _, err := dataFile.ReadAt(output[outputSize:outputSize+length], int64(start)); err != nil { + return err + } + outputSize += length + return nil + } + // Read all the indexes in one go + indices, err := t.getIndices(start, count) if err != nil { - return nil, err + return nil, nil, err } - dataFile, exist := t.files[filenum] - if !exist { - return nil, fmt.Errorf("missing data file %d", filenum) + var ( + sizes []int // The sizes for each element + totalSize = 0 // The total size of all data read so far + readStart = indices[0].offset // Where, in the file, to start reading + unreadSize = 0 // The size of the as-yet-unread data + ) + + for i, firstIndex := range indices[:len(indices)-1] { + secondIndex := indices[i+1] + // Determine the size of the item. + offset1, offset2, _ := firstIndex.bounds(secondIndex) + size := int(offset2 - offset1) + // Crossing a file boundary? + if secondIndex.filenum != firstIndex.filenum { + // If we have unread data in the first file, we need to do that read now. + if unreadSize > 0 { + if err := readData(firstIndex.filenum, readStart, unreadSize); err != nil { + return nil, nil, err + } + unreadSize = 0 + } + readStart = 0 + } + if i > 0 && uint64(totalSize+size) > maxBytes { + // About to break out due to byte limit being exceeded. We don't + // read this last item, but we need to do the deferred reads now. + if unreadSize > 0 { + if err := readData(secondIndex.filenum, readStart, unreadSize); err != nil { + return nil, nil, err + } + } + break + } + // Defer the read for later + unreadSize += size + totalSize += size + sizes = append(sizes, size) + if i == len(indices)-2 || uint64(totalSize) > maxBytes { + // Last item, need to do the read now + if err := readData(secondIndex.filenum, readStart, unreadSize); err != nil { + return nil, nil, err + } + break + } } - // Retrieve the data itself, decompress and return - blob := make([]byte, endOffset-startOffset) - if _, err := dataFile.ReadAt(blob, int64(startOffset)); err != nil { - return nil, err - } - t.readMeter.Mark(int64(len(blob) + 2*indexEntrySize)) - return blob, nil + return output[:outputSize], sizes, nil } // has returns an indicator whether the specified number data diff --git a/core/rawdb/freezer_table_test.go b/core/rawdb/freezer_table_test.go index 0df28f236..e8a8b5c46 100644 --- a/core/rawdb/freezer_table_test.go +++ b/core/rawdb/freezer_table_test.go @@ -74,7 +74,7 @@ func TestFreezerBasics(t *testing.T) { exp := getChunk(15, y) got, err := f.Retrieve(uint64(y)) if err != nil { - t.Fatal(err) + t.Fatalf("reading item %d: %v", y, err) } if !bytes.Equal(got, exp) { t.Fatalf("test %d, got \n%x != \n%x", y, got, exp) @@ -692,3 +692,118 @@ func TestAppendTruncateParallel(t *testing.T) { } } } + +// TestSequentialRead does some basic tests on the RetrieveItems. +func TestSequentialRead(t *testing.T) { + rm, wm, sg := metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge() + fname := fmt.Sprintf("batchread-%d", rand.Uint64()) + { // Fill table + f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true) + if err != nil { + t.Fatal(err) + } + // Write 15 bytes 30 times + for x := 0; x < 30; x++ { + data := getChunk(15, x) + f.Append(uint64(x), data) + } + f.DumpIndex(0, 30) + f.Close() + } + { // Open it, iterate, verify iteration + f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 50, true) + if err != nil { + t.Fatal(err) + } + items, err := f.RetrieveItems(0, 10000, 100000) + if err != nil { + t.Fatal(err) + } + if have, want := len(items), 30; have != want { + t.Fatalf("want %d items, have %d ", want, have) + } + for i, have := range items { + want := getChunk(15, i) + if !bytes.Equal(want, have) { + t.Fatalf("data corruption: have\n%x\n, want \n%x\n", have, want) + } + } + f.Close() + } + { // Open it, iterate, verify byte limit. The byte limit is less than item + // size, so each lookup should only return one item + f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 40, true) + if err != nil { + t.Fatal(err) + } + items, err := f.RetrieveItems(0, 10000, 10) + if err != nil { + t.Fatal(err) + } + if have, want := len(items), 1; have != want { + t.Fatalf("want %d items, have %d ", want, have) + } + for i, have := range items { + want := getChunk(15, i) + if !bytes.Equal(want, have) { + t.Fatalf("data corruption: have\n%x\n, want \n%x\n", have, want) + } + } + f.Close() + } +} + +// TestSequentialReadByteLimit does some more advanced tests on batch reads. +// These tests check that when the byte limit hits, we correctly abort in time, +// but also properly do all the deferred reads for the previous data, regardless +// of whether the data crosses a file boundary or not. +func TestSequentialReadByteLimit(t *testing.T) { + rm, wm, sg := metrics.NewMeter(), metrics.NewMeter(), metrics.NewGauge() + fname := fmt.Sprintf("batchread-2-%d", rand.Uint64()) + { // Fill table + f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 100, true) + if err != nil { + t.Fatal(err) + } + // Write 10 bytes 30 times, + // Splitting it at every 100 bytes (10 items) + for x := 0; x < 30; x++ { + data := getChunk(10, x) + f.Append(uint64(x), data) + } + f.Close() + } + for i, tc := range []struct { + items uint64 + limit uint64 + want int + }{ + {9, 89, 8}, + {10, 99, 9}, + {11, 109, 10}, + {100, 89, 8}, + {100, 99, 9}, + {100, 109, 10}, + } { + { + f, err := newCustomTable(os.TempDir(), fname, rm, wm, sg, 100, true) + if err != nil { + t.Fatal(err) + } + items, err := f.RetrieveItems(0, tc.items, tc.limit) + if err != nil { + t.Fatal(err) + } + if have, want := len(items), tc.want; have != want { + t.Fatalf("test %d: want %d items, have %d ", i, want, have) + } + for ii, have := range items { + want := getChunk(10, ii) + if !bytes.Equal(want, have) { + t.Fatalf("test %d: data corruption item %d: have\n%x\n, want \n%x\n", i, ii, have, want) + } + } + f.Close() + } + } +} diff --git a/core/rawdb/table.go b/core/rawdb/table.go index d5ef60ae5..586451c06 100644 --- a/core/rawdb/table.go +++ b/core/rawdb/table.go @@ -62,6 +62,12 @@ func (t *table) Ancient(kind string, number uint64) ([]byte, error) { return t.db.Ancient(kind, number) } +// ReadAncients is a noop passthrough that just forwards the request to the underlying +// database. +func (t *table) ReadAncients(kind string, start, count, maxBytes uint64) ([][]byte, error) { + return t.db.ReadAncients(kind, start, count, maxBytes) +} + // Ancients is a noop passthrough that just forwards the request to the underlying // database. func (t *table) Ancients() (uint64, error) { diff --git a/ethdb/database.go b/ethdb/database.go index 0dc14624b..bdc09d5e9 100644 --- a/ethdb/database.go +++ b/ethdb/database.go @@ -76,6 +76,13 @@ type AncientReader interface { // Ancient retrieves an ancient binary blob from the append-only immutable files. Ancient(kind string, number uint64) ([]byte, error) + // ReadAncients retrieves multiple items in sequence, starting from the index 'start'. + // It will return + // - at most 'count' items, + // - at least 1 item (even if exceeding the maxBytes), but will otherwise + // return as many items as fit into maxBytes. + ReadAncients(kind string, start, count, maxBytes uint64) ([][]byte, error) + // Ancients returns the ancient item numbers in the ancient store. Ancients() (uint64, error) From 0a68558e7e025afebf67b81bf48ecb8b0fa7c06d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Fri, 13 Aug 2021 15:39:51 +0300 Subject: [PATCH 04/17] accounts/external: handle 0 chainid as not-set for the Clef API (#23394) * accounts/external: handle 0 chainid as not-set for the Clef API * accounts/external: document SignTx Co-authored-by: Felix Lange --- accounts/external/backend.go | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/accounts/external/backend.go b/accounts/external/backend.go index b2de9e932..e3f754eaf 100644 --- a/accounts/external/backend.go +++ b/accounts/external/backend.go @@ -196,6 +196,10 @@ type signTransactionResult struct { Tx *types.Transaction `json:"tx"` } +// SignTx sends the transaction to the external signer. +// If chainID is nil, or tx.ChainID is zero, the chain ID will be assigned +// by the external signer. For non-legacy transactions, the chain ID of the +// transaction overrides the chainID parameter. func (api *ExternalSigner) SignTx(account accounts.Account, tx *types.Transaction, chainID *big.Int) (*types.Transaction, error) { data := hexutil.Bytes(tx.Data()) var to *common.MixedcaseAddress @@ -218,17 +222,17 @@ func (api *ExternalSigner) SignTx(account accounts.Account, tx *types.Transactio args.MaxFeePerGas = (*hexutil.Big)(tx.GasFeeCap()) args.MaxPriorityFeePerGas = (*hexutil.Big)(tx.GasTipCap()) default: - return nil, fmt.Errorf("Unsupported tx type %d", tx.Type()) + return nil, fmt.Errorf("unsupported tx type %d", tx.Type()) } // We should request the default chain id that we're operating with // (the chain we're executing on) - if chainID != nil { + if chainID != nil && chainID.Sign() != 0 { args.ChainID = (*hexutil.Big)(chainID) } if tx.Type() != types.LegacyTxType { // However, if the user asked for a particular chain id, then we should // use that instead. - if tx.ChainId() != nil { + if tx.ChainId().Sign() != 0 { args.ChainID = (*hexutil.Big)(tx.ChainId()) } accessList := tx.AccessList() From fb4007bb2208a5b76f76287c03001ef906261691 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 17 Aug 2021 17:30:21 +0200 Subject: [PATCH 05/17] tests: update, enable legacy tests, remove vm tests (#23350) * tests: update, enable legacy tests, remove vm tests * tests: minor fixes --- tests/gen_vmexec.go | 90 ---------------------- tests/init_test.go | 9 +-- tests/state_test.go | 7 +- tests/state_test_util.go | 4 + tests/testdata | 2 +- tests/vm_test.go | 39 ---------- tests/vm_test_util.go | 161 --------------------------------------- 7 files changed, 11 insertions(+), 301 deletions(-) delete mode 100644 tests/gen_vmexec.go delete mode 100644 tests/vm_test.go delete mode 100644 tests/vm_test_util.go diff --git a/tests/gen_vmexec.go b/tests/gen_vmexec.go deleted file mode 100644 index 2fe155152..000000000 --- a/tests/gen_vmexec.go +++ /dev/null @@ -1,90 +0,0 @@ -// Code generated by github.com/fjl/gencodec. DO NOT EDIT. - -package tests - -import ( - "encoding/json" - "errors" - "math/big" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/hexutil" - "github.com/ethereum/go-ethereum/common/math" -) - -var _ = (*vmExecMarshaling)(nil) - -// MarshalJSON marshals as JSON. -func (v vmExec) MarshalJSON() ([]byte, error) { - type vmExec struct { - Address common.UnprefixedAddress `json:"address" gencodec:"required"` - Caller common.UnprefixedAddress `json:"caller" gencodec:"required"` - Origin common.UnprefixedAddress `json:"origin" gencodec:"required"` - Code hexutil.Bytes `json:"code" gencodec:"required"` - Data hexutil.Bytes `json:"data" gencodec:"required"` - Value *math.HexOrDecimal256 `json:"value" gencodec:"required"` - GasLimit math.HexOrDecimal64 `json:"gas" gencodec:"required"` - GasPrice *math.HexOrDecimal256 `json:"gasPrice" gencodec:"required"` - } - var enc vmExec - enc.Address = common.UnprefixedAddress(v.Address) - enc.Caller = common.UnprefixedAddress(v.Caller) - enc.Origin = common.UnprefixedAddress(v.Origin) - enc.Code = v.Code - enc.Data = v.Data - enc.Value = (*math.HexOrDecimal256)(v.Value) - enc.GasLimit = math.HexOrDecimal64(v.GasLimit) - enc.GasPrice = (*math.HexOrDecimal256)(v.GasPrice) - return json.Marshal(&enc) -} - -// UnmarshalJSON unmarshals from JSON. -func (v *vmExec) UnmarshalJSON(input []byte) error { - type vmExec struct { - Address *common.UnprefixedAddress `json:"address" gencodec:"required"` - Caller *common.UnprefixedAddress `json:"caller" gencodec:"required"` - Origin *common.UnprefixedAddress `json:"origin" gencodec:"required"` - Code *hexutil.Bytes `json:"code" gencodec:"required"` - Data *hexutil.Bytes `json:"data" gencodec:"required"` - Value *math.HexOrDecimal256 `json:"value" gencodec:"required"` - GasLimit *math.HexOrDecimal64 `json:"gas" gencodec:"required"` - GasPrice *math.HexOrDecimal256 `json:"gasPrice" gencodec:"required"` - } - var dec vmExec - if err := json.Unmarshal(input, &dec); err != nil { - return err - } - if dec.Address == nil { - return errors.New("missing required field 'address' for vmExec") - } - v.Address = common.Address(*dec.Address) - if dec.Caller == nil { - return errors.New("missing required field 'caller' for vmExec") - } - v.Caller = common.Address(*dec.Caller) - if dec.Origin == nil { - return errors.New("missing required field 'origin' for vmExec") - } - v.Origin = common.Address(*dec.Origin) - if dec.Code == nil { - return errors.New("missing required field 'code' for vmExec") - } - v.Code = *dec.Code - if dec.Data == nil { - return errors.New("missing required field 'data' for vmExec") - } - v.Data = *dec.Data - if dec.Value == nil { - return errors.New("missing required field 'value' for vmExec") - } - v.Value = (*big.Int)(dec.Value) - if dec.GasLimit == nil { - return errors.New("missing required field 'gas' for vmExec") - } - v.GasLimit = uint64(*dec.GasLimit) - if dec.GasPrice == nil { - return errors.New("missing required field 'gasPrice' for vmExec") - } - v.GasPrice = (*big.Int)(dec.GasPrice) - return nil -} diff --git a/tests/init_test.go b/tests/init_test.go index 969cf7139..312ad8869 100644 --- a/tests/init_test.go +++ b/tests/init_test.go @@ -34,12 +34,11 @@ import ( ) var ( - baseDir = filepath.Join(".", "testdata") - blockTestDir = filepath.Join(baseDir, "BlockchainTests") - stateTestDir = filepath.Join(baseDir, "GeneralStateTests") - //legacyStateTestDir = filepath.Join(baseDir, "LegacyTests", "Constantinople", "GeneralStateTests") + baseDir = filepath.Join(".", "testdata") + blockTestDir = filepath.Join(baseDir, "BlockchainTests") + stateTestDir = filepath.Join(baseDir, "GeneralStateTests") + legacyStateTestDir = filepath.Join(baseDir, "LegacyTests", "Constantinople", "GeneralStateTests") transactionTestDir = filepath.Join(baseDir, "TransactionTests") - vmTestDir = filepath.Join(baseDir, "VMTests") rlpTestDir = filepath.Join(baseDir, "RLPTests") difficultyTestDir = filepath.Join(baseDir, "BasicTests") ) diff --git a/tests/state_test.go b/tests/state_test.go index 242e9712a..c2ca0e8d6 100644 --- a/tests/state_test.go +++ b/tests/state_test.go @@ -45,8 +45,7 @@ func TestState(t *testing.T) { // Uses 1GB RAM per tested fork st.skipLoad(`^stStaticCall/static_Call1MB`) - // Un-skip this when https://github.com/ethereum/tests/issues/908 is closed - st.skipLoad(`^stQuadraticComplexityTest/QuadraticComplexitySolidity_CallDataCopy`) + // Broken tests: // Expected failures: //st.fails(`^stRevertTest/RevertPrecompiledTouch(_storage)?\.json/Byzantium/0`, "bug in test") @@ -59,9 +58,7 @@ func TestState(t *testing.T) { // For Istanbul, older tests were moved into LegacyTests for _, dir := range []string{ stateTestDir, - // legacy state tests are disabled, due to them not being - // regenerated for the no-sender-eoa change. - //legacyStateTestDir, + legacyStateTestDir, } { st.walk(t, dir, func(t *testing.T, name string, test *StateTest) { for _, subtest := range test.Subtests() { diff --git a/tests/state_test_util.go b/tests/state_test_util.go index 4f57fee89..8b8a6a4fd 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -358,3 +358,7 @@ func rlpHash(x interface{}) (h common.Hash) { hw.Sum(h[:0]) return h } + +func vmTestBlockHash(n uint64) common.Hash { + return common.BytesToHash(crypto.Keccak256([]byte(big.NewInt(int64(n)).String()))) +} diff --git a/tests/testdata b/tests/testdata index 5d534e37b..092a8834d 160000 --- a/tests/testdata +++ b/tests/testdata @@ -1 +1 @@ -Subproject commit 5d534e37b80e9310e8c7751f805ca481a451123e +Subproject commit 092a8834dc445e683103689d6f0e75a5d380a190 diff --git a/tests/vm_test.go b/tests/vm_test.go deleted file mode 100644 index 2150df9e2..000000000 --- a/tests/vm_test.go +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2014 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package tests - -import ( - "testing" - - "github.com/ethereum/go-ethereum/core/vm" -) - -func TestVM(t *testing.T) { - t.Parallel() - vmt := new(testMatcher) - vmt.slow("^vmPerformance") - vmt.fails("^vmSystemOperationsTest.json/createNameRegistrator$", "fails without parallel execution") - - vmt.walk(t, vmTestDir, func(t *testing.T, name string, test *VMTest) { - withTrace(t, test.json.Exec.GasLimit, func(vmconfig vm.Config) error { - return vmt.checkFailure(t, test.Run(vmconfig, false)) - }) - withTrace(t, test.json.Exec.GasLimit, func(vmconfig vm.Config) error { - return vmt.checkFailure(t, test.Run(vmconfig, true)) - }) - }) -} diff --git a/tests/vm_test_util.go b/tests/vm_test_util.go deleted file mode 100644 index 418cc6716..000000000 --- a/tests/vm_test_util.go +++ /dev/null @@ -1,161 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package tests - -import ( - "bytes" - "encoding/json" - "fmt" - "math/big" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/hexutil" - "github.com/ethereum/go-ethereum/common/math" - "github.com/ethereum/go-ethereum/core" - "github.com/ethereum/go-ethereum/core/rawdb" - "github.com/ethereum/go-ethereum/core/state" - "github.com/ethereum/go-ethereum/core/vm" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/params" -) - -// VMTest checks EVM execution without block or transaction context. -// See https://github.com/ethereum/tests/wiki/VM-Tests for the test format specification. -type VMTest struct { - json vmJSON -} - -func (t *VMTest) UnmarshalJSON(data []byte) error { - return json.Unmarshal(data, &t.json) -} - -type vmJSON struct { - Env stEnv `json:"env"` - Exec vmExec `json:"exec"` - Logs common.UnprefixedHash `json:"logs"` - GasRemaining *math.HexOrDecimal64 `json:"gas"` - Out hexutil.Bytes `json:"out"` - Pre core.GenesisAlloc `json:"pre"` - Post core.GenesisAlloc `json:"post"` - PostStateRoot common.Hash `json:"postStateRoot"` -} - -//go:generate gencodec -type vmExec -field-override vmExecMarshaling -out gen_vmexec.go - -type vmExec struct { - Address common.Address `json:"address" gencodec:"required"` - Caller common.Address `json:"caller" gencodec:"required"` - Origin common.Address `json:"origin" gencodec:"required"` - Code []byte `json:"code" gencodec:"required"` - Data []byte `json:"data" gencodec:"required"` - Value *big.Int `json:"value" gencodec:"required"` - GasLimit uint64 `json:"gas" gencodec:"required"` - GasPrice *big.Int `json:"gasPrice" gencodec:"required"` -} - -type vmExecMarshaling struct { - Address common.UnprefixedAddress - Caller common.UnprefixedAddress - Origin common.UnprefixedAddress - Code hexutil.Bytes - Data hexutil.Bytes - Value *math.HexOrDecimal256 - GasLimit math.HexOrDecimal64 - GasPrice *math.HexOrDecimal256 -} - -func (t *VMTest) Run(vmconfig vm.Config, snapshotter bool) error { - snaps, statedb := MakePreState(rawdb.NewMemoryDatabase(), t.json.Pre, snapshotter) - if snapshotter { - preRoot := statedb.IntermediateRoot(false) - defer func() { - if _, err := snaps.Journal(preRoot); err != nil { - panic(err) - } - }() - } - ret, gasRemaining, err := t.exec(statedb, vmconfig) - - if t.json.GasRemaining == nil { - if err == nil { - return fmt.Errorf("gas unspecified (indicating an error), but VM returned no error") - } - if gasRemaining > 0 { - return fmt.Errorf("gas unspecified (indicating an error), but VM returned gas remaining > 0") - } - return nil - } - // Test declares gas, expecting outputs to match. - if !bytes.Equal(ret, t.json.Out) { - return fmt.Errorf("return data mismatch: got %x, want %x", ret, t.json.Out) - } - if gasRemaining != uint64(*t.json.GasRemaining) { - return fmt.Errorf("remaining gas %v, want %v", gasRemaining, *t.json.GasRemaining) - } - for addr, account := range t.json.Post { - for k, wantV := range account.Storage { - if haveV := statedb.GetState(addr, k); haveV != wantV { - return fmt.Errorf("wrong storage value at %x:\n got %x\n want %x", k, haveV, wantV) - } - } - } - // if root := statedb.IntermediateRoot(false); root != t.json.PostStateRoot { - // return fmt.Errorf("post state root mismatch, got %x, want %x", root, t.json.PostStateRoot) - // } - if logs := rlpHash(statedb.Logs()); logs != common.Hash(t.json.Logs) { - return fmt.Errorf("post state logs hash mismatch: got %x, want %x", logs, t.json.Logs) - } - return nil -} - -func (t *VMTest) exec(statedb *state.StateDB, vmconfig vm.Config) ([]byte, uint64, error) { - evm := t.newEVM(statedb, vmconfig) - e := t.json.Exec - return evm.Call(vm.AccountRef(e.Caller), e.Address, e.Data, e.GasLimit, e.Value) -} - -func (t *VMTest) newEVM(statedb *state.StateDB, vmconfig vm.Config) *vm.EVM { - initialCall := true - canTransfer := func(db vm.StateDB, address common.Address, amount *big.Int) bool { - if initialCall { - initialCall = false - return true - } - return core.CanTransfer(db, address, amount) - } - transfer := func(db vm.StateDB, sender, recipient common.Address, amount *big.Int) {} - txContext := vm.TxContext{ - Origin: t.json.Exec.Origin, - GasPrice: t.json.Exec.GasPrice, - } - context := vm.BlockContext{ - CanTransfer: canTransfer, - Transfer: transfer, - GetHash: vmTestBlockHash, - Coinbase: t.json.Env.Coinbase, - BlockNumber: new(big.Int).SetUint64(t.json.Env.Number), - Time: new(big.Int).SetUint64(t.json.Env.Timestamp), - GasLimit: t.json.Env.GasLimit, - Difficulty: t.json.Env.Difficulty, - } - vmconfig.NoRecursion = true - return vm.NewEVM(context, txContext, statedb, params.MainnetChainConfig, vmconfig) -} - -func vmTestBlockHash(n uint64) common.Hash { - return common.BytesToHash(crypto.Keccak256([]byte(big.NewInt(int64(n)).String()))) -} From 690248576704d9dc312d5746400e035c6dccaea6 Mon Sep 17 00:00:00 2001 From: jwasinger Date: Tue, 17 Aug 2021 18:40:14 +0200 Subject: [PATCH 06/17] cmd, metrics: add support for influxdb-v2 (cherry-picking from italoacasas' changes), leave existing support for v1 to maintain backwards-compatibility. (#23194) This PR adds flag to enable InfluxDB v2 (--metrics.influxdbv2), flags for v2-specific features (--metrics.influxdb.token, --metrics.influxdb.bucket), also carries over addition of support for specifying organization (--metrics.influxdb.organization), but still retains backwards compatibility with InfluxDB v1. --- cmd/geth/chaincmd.go | 4 + cmd/geth/config.go | 12 ++ cmd/geth/main.go | 4 + cmd/utils/flags.go | 64 +++++++++- go.mod | 13 +- go.sum | 64 ++++++++-- metrics/config.go | 11 ++ metrics/influxdb/influxdbv2.go | 223 +++++++++++++++++++++++++++++++++ 8 files changed, 377 insertions(+), 18 deletions(-) create mode 100644 metrics/influxdb/influxdbv2.go diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index 9c5d4f9b2..071a3cd65 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -92,11 +92,15 @@ The dumpgenesis command dumps the genesis block configuration in JSON format to utils.MetricsHTTPFlag, utils.MetricsPortFlag, utils.MetricsEnableInfluxDBFlag, + utils.MetricsEnableInfluxDBV2Flag, utils.MetricsInfluxDBEndpointFlag, utils.MetricsInfluxDBDatabaseFlag, utils.MetricsInfluxDBUsernameFlag, utils.MetricsInfluxDBPasswordFlag, utils.MetricsInfluxDBTagsFlag, + utils.MetricsInfluxDBTokenFlag, + utils.MetricsInfluxDBBucketFlag, + utils.MetricsInfluxDBOrganizationFlag, utils.TxLookupLimitFlag, }, Category: "BLOCKCHAIN COMMANDS", diff --git a/cmd/geth/config.go b/cmd/geth/config.go index 117be96ae..604f907b7 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -233,6 +233,18 @@ func applyMetricConfig(ctx *cli.Context, cfg *gethConfig) { if ctx.GlobalIsSet(utils.MetricsInfluxDBTagsFlag.Name) { cfg.Metrics.InfluxDBTags = ctx.GlobalString(utils.MetricsInfluxDBTagsFlag.Name) } + if ctx.GlobalIsSet(utils.MetricsEnableInfluxDBV2Flag.Name) { + cfg.Metrics.EnableInfluxDBV2 = ctx.GlobalBool(utils.MetricsEnableInfluxDBV2Flag.Name) + } + if ctx.GlobalIsSet(utils.MetricsInfluxDBTokenFlag.Name) { + cfg.Metrics.InfluxDBToken = ctx.GlobalString(utils.MetricsInfluxDBTokenFlag.Name) + } + if ctx.GlobalIsSet(utils.MetricsInfluxDBBucketFlag.Name) { + cfg.Metrics.InfluxDBBucket = ctx.GlobalString(utils.MetricsInfluxDBBucketFlag.Name) + } + if ctx.GlobalIsSet(utils.MetricsInfluxDBOrganizationFlag.Name) { + cfg.Metrics.InfluxDBOrganization = ctx.GlobalString(utils.MetricsInfluxDBOrganizationFlag.Name) + } } func deprecated(field string) bool { diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 25b35ca93..642405c50 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -195,6 +195,10 @@ var ( utils.MetricsInfluxDBUsernameFlag, utils.MetricsInfluxDBPasswordFlag, utils.MetricsInfluxDBTagsFlag, + utils.MetricsEnableInfluxDBV2Flag, + utils.MetricsInfluxDBTokenFlag, + utils.MetricsInfluxDBBucketFlag, + utils.MetricsInfluxDBOrganizationFlag, } ) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 7fa3d8b86..453a553eb 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -756,6 +756,29 @@ var ( Value: metrics.DefaultConfig.InfluxDBTags, } + MetricsEnableInfluxDBV2Flag = cli.BoolFlag{ + Name: "metrics.influxdbv2", + Usage: "Enable metrics export/push to an external InfluxDB v2 database", + } + + MetricsInfluxDBTokenFlag = cli.StringFlag{ + Name: "metrics.influxdb.token", + Usage: "Token to authorize access to the database (v2 only)", + Value: metrics.DefaultConfig.InfluxDBToken, + } + + MetricsInfluxDBBucketFlag = cli.StringFlag{ + Name: "metrics.influxdb.bucket", + Usage: "InfluxDB bucket name to push reported metrics to (v2 only)", + Value: metrics.DefaultConfig.InfluxDBBucket, + } + + MetricsInfluxDBOrganizationFlag = cli.StringFlag{ + Name: "metrics.influxdb.organization", + Usage: "InfluxDB organization name (v2 only)", + Value: metrics.DefaultConfig.InfluxDBOrganization, + } + CatalystFlag = cli.BoolFlag{ Name: "catalyst", Usage: "Catalyst mode (eth2 integration testing)", @@ -1739,11 +1762,36 @@ func SetupMetrics(ctx *cli.Context) { log.Info("Enabling metrics collection") var ( - enableExport = ctx.GlobalBool(MetricsEnableInfluxDBFlag.Name) - endpoint = ctx.GlobalString(MetricsInfluxDBEndpointFlag.Name) - database = ctx.GlobalString(MetricsInfluxDBDatabaseFlag.Name) - username = ctx.GlobalString(MetricsInfluxDBUsernameFlag.Name) - password = ctx.GlobalString(MetricsInfluxDBPasswordFlag.Name) + enableExport = ctx.GlobalBool(MetricsEnableInfluxDBFlag.Name) + enableExportV2 = ctx.GlobalBool(MetricsEnableInfluxDBV2Flag.Name) + ) + + if enableExport || enableExportV2 { + CheckExclusive(ctx, MetricsEnableInfluxDBFlag, MetricsEnableInfluxDBV2Flag) + + v1FlagIsSet := ctx.GlobalIsSet(MetricsInfluxDBUsernameFlag.Name) || + ctx.GlobalIsSet(MetricsInfluxDBPasswordFlag.Name) + + v2FlagIsSet := ctx.GlobalIsSet(MetricsInfluxDBTokenFlag.Name) || + ctx.GlobalIsSet(MetricsInfluxDBOrganizationFlag.Name) || + ctx.GlobalIsSet(MetricsInfluxDBBucketFlag.Name) + + if enableExport && v2FlagIsSet { + Fatalf("Flags --influxdb.metrics.organization, --influxdb.metrics.token, --influxdb.metrics.bucket are only available for influxdb-v2") + } else if enableExportV2 && v1FlagIsSet { + Fatalf("Flags --influxdb.metrics.username, --influxdb.metrics.password are only available for influxdb-v1") + } + } + + var ( + endpoint = ctx.GlobalString(MetricsInfluxDBEndpointFlag.Name) + database = ctx.GlobalString(MetricsInfluxDBDatabaseFlag.Name) + username = ctx.GlobalString(MetricsInfluxDBUsernameFlag.Name) + password = ctx.GlobalString(MetricsInfluxDBPasswordFlag.Name) + + token = ctx.GlobalString(MetricsInfluxDBTokenFlag.Name) + bucket = ctx.GlobalString(MetricsInfluxDBBucketFlag.Name) + organization = ctx.GlobalString(MetricsInfluxDBOrganizationFlag.Name) ) if enableExport { @@ -1752,6 +1800,12 @@ func SetupMetrics(ctx *cli.Context) { log.Info("Enabling metrics export to InfluxDB") go influxdb.InfluxDBWithTags(metrics.DefaultRegistry, 10*time.Second, endpoint, database, username, password, "geth.", tagsMap) + } else if enableExportV2 { + tagsMap := SplitTagsFlag(ctx.GlobalString(MetricsInfluxDBTagsFlag.Name)) + + log.Info("Enabling metrics export to InfluxDB (v2)") + + go influxdb.InfluxDBV2WithTags(metrics.DefaultRegistry, 10*time.Second, endpoint, token, bucket, organization, "geth.", tagsMap) } if ctx.GlobalIsSet(MetricsHTTPFlag.Name) { diff --git a/go.mod b/go.mod index 4621bc9ab..7e8407f53 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f github.com/davecgh/go-spew v1.1.1 github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea + github.com/deepmap/oapi-codegen v1.8.2 // indirect github.com/dlclark/regexp2 v1.2.0 // indirect github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf github.com/dop251/goja v0.0.0-20200721192441-a695b0cdd498 @@ -39,13 +40,15 @@ require ( github.com/holiman/uint256 v1.2.0 github.com/huin/goupnp v1.0.2 github.com/influxdata/influxdb v1.8.3 + github.com/influxdata/influxdb-client-go/v2 v2.4.0 + github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097 // indirect github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458 github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e github.com/julienschmidt/httprouter v1.2.0 github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356 github.com/kylelemons/godebug v1.1.0 // indirect - github.com/mattn/go-colorable v0.1.0 - github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035 + github.com/mattn/go-colorable v0.1.8 + github.com/mattn/go-isatty v0.0.12 github.com/naoina/go-stringutil v0.1.0 // indirect github.com/naoina/toml v0.1.2-0.20170918210437-9fafd6967416 github.com/olekukonko/tablewriter v0.0.5 @@ -60,12 +63,14 @@ require ( github.com/tklauser/go-sysconf v0.3.5 // indirect github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 + golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d // indirect golang.org/x/sync v0.0.0-20210220032951-036812b2e83c - golang.org/x/sys v0.0.0-20210420205809-ac73e9fd8988 + golang.org/x/sys v0.0.0-20210423082822-04245dca01da golang.org/x/text v0.3.6 - golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 + golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6 gopkg.in/urfave/cli.v1 v1.20.0 + gopkg.in/yaml.v2 v2.4.0 // indirect gotest.tools v2.2.0+incompatible // indirect ) diff --git a/go.sum b/go.sum index 4c494e170..b4d0b725c 100644 --- a/go.sum +++ b/go.sum @@ -104,6 +104,7 @@ github.com/consensys/bavard v0.1.8-0.20210406032232-f3452dc9b572/go.mod h1:Bpd0/ github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f h1:C43yEtQ6NIf4ftFXD/V55gnGFgPbMQobd//YlnLjUJ8= github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f/go.mod h1:815PAHg3wvysy0SyIqanF8gZ0Y1wjk/hrDHD/iT88+Q= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cyberdelia/templates v0.0.0-20141128023046-ca7fffd4298c/go.mod h1:GyV+0YP4qX0UQ7r2MoYZ+AvYDp12OF5yg4q8rGnyNh4= github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -111,6 +112,9 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea h1:j4317fAZh7X6GqbFowYdYdI0L9bwxL07jyPZIdepyZ0= github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea/go.mod h1:93vsz/8Wt4joVM7c2AVqh+YRMiUSc14yDtF28KmMOgQ= +github.com/deepmap/oapi-codegen v1.6.0/go.mod h1:ryDa9AgbELGeB+YEXE1dR53yAjHwFvE9iAUlWl9Al3M= +github.com/deepmap/oapi-codegen v1.8.2 h1:SegyeYGcdi0jLLrpbCMoJxnUUn8GBXHsvr4rbzjuhfU= +github.com/deepmap/oapi-codegen v1.8.2/go.mod h1:YLgSKSDv/bZQB7N4ws6luhozi3cEdRktEqrX88CvjIw= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-bitstream v0.0.0-20180413035011-3522498ce2c8/go.mod h1:VMaSuZ+SZcx/wljOQKvp5srsbCiKDEb6K2wC4+PiBmQ= @@ -136,8 +140,12 @@ github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWo github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqGNY4FhTFhk+o9oFHGINQ/+vhlm8HFzi6znCI= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= +github.com/getkin/kin-openapi v0.53.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= +github.com/getkin/kin-openapi v0.61.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= +github.com/go-chi/chi/v5 v5.0.0/go.mod h1:BBug9lr0cqtdAhsu6R4AAdvufI0/XBzAQSsUqJpoZOs= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-kit/kit v0.8.0 h1:Wz+5lgoB0kkuqLEc6NVmwRknTKP6dTGbSqvhZtBI/j0= @@ -147,6 +155,8 @@ github.com/go-logfmt/logfmt v0.4.0 h1:MP4Eh7ZCb31lleYCFuwm0oe4/YGak+5l1vA2NOE80n github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-ole/go-ole v1.2.1 h1:2lOsA72HgjxAuMlKpFiCbHTvu44PIVkZ5hqm3RSdI/E= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= github.com/go-sourcemap/sourcemap v2.1.2+incompatible h1:0b/xya7BKGhXuqFESKM4oIiRo9WOt2ebz7KxfreD6ug= github.com/go-sourcemap/sourcemap v2.1.2+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= @@ -178,6 +188,7 @@ github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8l github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/flatbuffers v1.11.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= @@ -200,6 +211,7 @@ github.com/google/uuid v1.1.5/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/graph-gophers/graphql-go v0.0.0-20201113091052-beb923fada29 h1:sezaKhEfPFg8W0Enm61B9Gs911H8iesGY5R8NDPtd1M= @@ -221,8 +233,13 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt github.com/influxdata/flux v0.65.1/go.mod h1:J754/zds0vvpfwuq7Gc2wRdVwEodfpCFM7mYlOw2LqY= github.com/influxdata/influxdb v1.8.3 h1:WEypI1BQFTT4teLM+1qkEcvUi0dAvopAI/ir0vAiBg8= github.com/influxdata/influxdb v1.8.3/go.mod h1:JugdFhsvvI8gadxOI6noqNeeBHvWNTbfYGtiAn+2jhI= +github.com/influxdata/influxdb-client-go/v2 v2.4.0 h1:HGBfZYStlx3Kqvsv1h2pJixbCl/jhnFtxpKFAv9Tu5k= +github.com/influxdata/influxdb-client-go/v2 v2.4.0/go.mod h1:vLNHdxTJkIf2mSLvGrpj8TCcISApPoXkaxP8g9uRlW8= github.com/influxdata/influxql v1.1.1-0.20200828144457-65d3ef77d385/go.mod h1:gHp9y86a/pxhjJ+zMjNXiQAA197Xk9wLxaz+fGG+kWk= github.com/influxdata/line-protocol v0.0.0-20180522152040-32c6aa80de5e/go.mod h1:4kt73NQhadE3daL3WhR5EJ/J2ocX0PZzwxQ0gXJ7oFE= +github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo= +github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097 h1:vilfsDSy7TDxedi9gyBkMvAirat/oRcL0lFdJBf6tdM= +github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo= github.com/influxdata/promql/v2 v2.12.0/go.mod h1:fxOPu+DY0bqCTCECchSRtWfc+0X19ybifQhZoQNF5D8= github.com/influxdata/roaring v0.4.13-0.20180809181101-fc520f41fab6/go.mod h1:bSgUQ7q5ZLSO+bKBGqJiCBGAl+9DxyW63zLTujjUlOE= github.com/influxdata/tdigest v0.0.0-20181121200506-bf2b5ad3c0a9/go.mod h1:Js0mqiSBE6Ffsg94weZZ2c+v/ciT8QRHFOap7EKDrR0= @@ -263,18 +280,27 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= +github.com/labstack/echo/v4 v4.2.1/go.mod h1:AA49e0DZ8kk5jTOOCKNuPR6oTnBS0dYiM4FW1e6jwpg= +github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/matryer/moq v0.0.0-20190312154309-6cfb0558e1bd/go.mod h1:9ELz6aaclSIGnZBoaSLZ3NAl1VTufbOrXBPvtcy6WiQ= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= -github.com/mattn/go-colorable v0.1.0 h1:v2XXALHHh6zHfYTJ+cSkwtyffnaOyR1MXaA91mTrb8o= -github.com/mattn/go-colorable v0.1.0/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= +github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-ieproxy v0.0.0-20190610004146-91bb50d98149/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d h1:oNAwILwmgWKFpuU+dXvI6dl9jG2mAWAZLX3r9s0PPiw= github.com/mattn/go-ieproxy v0.0.0-20190702010315-6dee0af9227d/go.mod h1:31jz6HNzdxOmlERGGEc4v/dMssOfmp2p5bT/okiKFFc= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035 h1:USWjF42jDCSEeikX/G1g40ZWnsPXN5WkZ4jMHZWyBK4= -github.com/mattn/go-isatty v0.0.5-0.20180830101745-3fb116b82035/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= +github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= +github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= @@ -360,6 +386,7 @@ github.com/stretchr/testify v1.2.0/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/syndtr/goleveldb v1.0.1-0.20210305035536-64b5b1c73954 h1:xQdMZ1WLrgkkvOZ/LDQxjVxMLdby7osSh4ZEVa5sIjs= @@ -372,6 +399,9 @@ github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZF github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef h1:wHSqTBrZW24CsNJDfeh9Ex6Pm0Rcpc7qrgKBiL44vF4= github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= +github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -389,6 +419,8 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190909091759-094676da4a83/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200820211705-5c72a883971a/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 h1:It14KIkyBFYkHkwZ7k45minvA9aorojkyjGk9KJ5B/w= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -436,10 +468,13 @@ golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210220033124-5f55cee0dc0d/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= -golang.org/x/net v0.0.0-20210226172049-e18ecbb05110 h1:qWPm9rbaAMKs8Bq/9LRpbMqxWRVUAQwMI9fVrssnTfw= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d h1:20cMwl2fHAzkJMEA+8J4JgqBQcQGzbisXo31MIeenXI= +golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -461,6 +496,7 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -468,36 +504,45 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200107162124-548cf772de50/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200826173525-f9321e4c35a6/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210420205809-ac73e9fd8988 h1:EjgCl+fVlIaPJSori0ikSz3uV0DOHKWOJFpv1sAAhBM= golang.org/x/sys v0.0.0-20210420205809-ac73e9fd8988/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da h1:b3NXsE2LusjYGGjL5bxEVZZORm/YEFFrWFjR8eFrw/c= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20201208040808-7e3f01d25324 h1:Hir2P/De0WpUhtrKGGjvSb2YxUgyZ7EFOSLIcSSpiwE= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba h1:O8mE0/t419eoIwhTFpKVkHiTs/Igowgfkj25AcZrtiE= +golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -593,8 +638,9 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= diff --git a/metrics/config.go b/metrics/config.go index d05d66426..2eb09fb48 100644 --- a/metrics/config.go +++ b/metrics/config.go @@ -28,6 +28,11 @@ type Config struct { InfluxDBUsername string `toml:",omitempty"` InfluxDBPassword string `toml:",omitempty"` InfluxDBTags string `toml:",omitempty"` + + EnableInfluxDBV2 bool `toml:",omitempty"` + InfluxDBToken string `toml:",omitempty"` + InfluxDBBucket string `toml:",omitempty"` + InfluxDBOrganization string `toml:",omitempty"` } // DefaultConfig is the default config for metrics used in go-ethereum. @@ -42,4 +47,10 @@ var DefaultConfig = Config{ InfluxDBUsername: "test", InfluxDBPassword: "test", InfluxDBTags: "host=localhost", + + // influxdbv2-specific flags + EnableInfluxDBV2: false, + InfluxDBToken: "test", + InfluxDBBucket: "geth", + InfluxDBOrganization: "geth", } diff --git a/metrics/influxdb/influxdbv2.go b/metrics/influxdb/influxdbv2.go new file mode 100644 index 000000000..00901f52c --- /dev/null +++ b/metrics/influxdb/influxdbv2.go @@ -0,0 +1,223 @@ +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . +package influxdb + +import ( + "context" + "fmt" + "time" + + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" + influxdb2 "github.com/influxdata/influxdb-client-go/v2" + "github.com/influxdata/influxdb-client-go/v2/api" +) + +type v2Reporter struct { + reg metrics.Registry + interval time.Duration + + endpoint string + token string + bucket string + organization string + namespace string + tags map[string]string + + client influxdb2.Client + write api.WriteAPI + + cache map[string]int64 +} + +// InfluxDBWithTags starts a InfluxDB reporter which will post the from the given metrics.Registry at each d interval with the specified tags +func InfluxDBV2WithTags(r metrics.Registry, d time.Duration, endpoint string, token string, bucket string, organization string, namespace string, tags map[string]string) { + rep := &v2Reporter{ + reg: r, + interval: d, + endpoint: endpoint, + token: token, + bucket: bucket, + organization: organization, + namespace: namespace, + tags: tags, + cache: make(map[string]int64), + } + + rep.client = influxdb2.NewClient(rep.endpoint, rep.token) + defer rep.client.Close() + + // async write client + rep.write = rep.client.WriteAPI(rep.organization, rep.bucket) + errorsCh := rep.write.Errors() + + // have to handle write errors in a separate goroutine like this b/c the channel is unbuffered and will block writes if not read + go func() { + for err := range errorsCh { + log.Warn("write error", "err", err.Error()) + } + }() + rep.run() +} + +func (r *v2Reporter) run() { + intervalTicker := time.Tick(r.interval) + pingTicker := time.Tick(time.Second * 5) + + for { + select { + case <-intervalTicker: + r.send() + case <-pingTicker: + _, err := r.client.Health(context.Background()) + if err != nil { + log.Warn("Got error from influxdb client health check", "err", err.Error()) + } + } + } + +} + +func (r *v2Reporter) send() { + r.reg.Each(func(name string, i interface{}) { + now := time.Now() + namespace := r.namespace + + switch metric := i.(type) { + + case metrics.Counter: + v := metric.Count() + l := r.cache[name] + + measurement := fmt.Sprintf("%s%s.count", namespace, name) + fields := map[string]interface{}{ + "value": v - l, + } + + pt := influxdb2.NewPoint(measurement, r.tags, fields, now) + r.write.WritePoint(pt) + + r.cache[name] = v + + case metrics.Gauge: + ms := metric.Snapshot() + + measurement := fmt.Sprintf("%s%s.gauge", namespace, name) + fields := map[string]interface{}{ + "value": ms.Value(), + } + + pt := influxdb2.NewPoint(measurement, r.tags, fields, now) + r.write.WritePoint(pt) + + case metrics.GaugeFloat64: + ms := metric.Snapshot() + + measurement := fmt.Sprintf("%s%s.gauge", namespace, name) + fields := map[string]interface{}{ + "value": ms.Value(), + } + + pt := influxdb2.NewPoint(measurement, r.tags, fields, now) + r.write.WritePoint(pt) + + case metrics.Histogram: + ms := metric.Snapshot() + + if ms.Count() > 0 { + ps := ms.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999, 0.9999}) + measurement := fmt.Sprintf("%s%s.histogram", namespace, name) + fields := map[string]interface{}{ + "count": ms.Count(), + "max": ms.Max(), + "mean": ms.Mean(), + "min": ms.Min(), + "stddev": ms.StdDev(), + "variance": ms.Variance(), + "p50": ps[0], + "p75": ps[1], + "p95": ps[2], + "p99": ps[3], + "p999": ps[4], + "p9999": ps[5], + } + + pt := influxdb2.NewPoint(measurement, r.tags, fields, now) + r.write.WritePoint(pt) + } + + case metrics.Meter: + ms := metric.Snapshot() + + measurement := fmt.Sprintf("%s%s.meter", namespace, name) + fields := map[string]interface{}{ + "count": ms.Count(), + "m1": ms.Rate1(), + "m5": ms.Rate5(), + "m15": ms.Rate15(), + "mean": ms.RateMean(), + } + + pt := influxdb2.NewPoint(measurement, r.tags, fields, now) + r.write.WritePoint(pt) + + case metrics.Timer: + ms := metric.Snapshot() + ps := ms.Percentiles([]float64{0.5, 0.75, 0.95, 0.99, 0.999, 0.9999}) + + measurement := fmt.Sprintf("%s%s.timer", namespace, name) + fields := map[string]interface{}{ + "count": ms.Count(), + "max": ms.Max(), + "mean": ms.Mean(), + "min": ms.Min(), + "stddev": ms.StdDev(), + "variance": ms.Variance(), + "p50": ps[0], + "p75": ps[1], + "p95": ps[2], + "p99": ps[3], + "p999": ps[4], + "p9999": ps[5], + "m1": ms.Rate1(), + "m5": ms.Rate5(), + "m15": ms.Rate15(), + "meanrate": ms.RateMean(), + } + + pt := influxdb2.NewPoint(measurement, r.tags, fields, now) + r.write.WritePoint(pt) + + case metrics.ResettingTimer: + t := metric.Snapshot() + + if len(t.Values()) > 0 { + ps := t.Percentiles([]float64{50, 95, 99}) + val := t.Values() + + measurement := fmt.Sprintf("%s%s.span", namespace, name) + fields := map[string]interface{}{ + "count": len(val), + "max": val[len(val)-1], + "mean": t.Mean(), + "min": val[0], + "p50": ps[0], + "p95": ps[1], + "p99": ps[2], + } + + pt := influxdb2.NewPoint(measurement, r.tags, fields, now) + r.write.WritePoint(pt) + } + } + }) + + // Force all unwritten data to be sent + r.write.Flush() +} From 85b9bdd6413797f6720c7d9ef46e5fdeb47b0cb6 Mon Sep 17 00:00:00 2001 From: Zachinquarantine Date: Tue, 17 Aug 2021 12:43:25 -0400 Subject: [PATCH 07/17] cmd, core: remove calaveras testnet (#23366) Removes references to the short-lived Calaveras testnet --- cmd/geth/chaincmd.go | 1 - cmd/geth/consolecmd.go | 2 -- cmd/geth/dbcmd.go | 8 -------- cmd/geth/main.go | 4 ---- cmd/geth/usage.go | 1 - cmd/utils/flags.go | 20 +------------------- core/genesis.go | 14 -------------- core/genesis_test.go | 4 ---- params/bootnodes.go | 6 ------ params/config.go | 30 ++++-------------------------- 10 files changed, 5 insertions(+), 85 deletions(-) diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index 071a3cd65..436e558b5 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -68,7 +68,6 @@ It expects the genesis file as argument.`, utils.RopstenFlag, utils.RinkebyFlag, utils.GoerliFlag, - utils.CalaverasFlag, }, Category: "BLOCKCHAIN COMMANDS", Description: ` diff --git a/cmd/geth/consolecmd.go b/cmd/geth/consolecmd.go index 776412df6..0e156fde9 100644 --- a/cmd/geth/consolecmd.go +++ b/cmd/geth/consolecmd.go @@ -134,8 +134,6 @@ func remoteConsole(ctx *cli.Context) error { path = filepath.Join(path, "rinkeby") } else if ctx.GlobalBool(utils.GoerliFlag.Name) { path = filepath.Join(path, "goerli") - } else if ctx.GlobalBool(utils.CalaverasFlag.Name) { - path = filepath.Join(path, "calaveras") } } endpoint = fmt.Sprintf("%s/geth.ipc", path) diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index bb3cf0390..54ada4089 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -75,7 +75,6 @@ Remove blockchain and state databases`, utils.RopstenFlag, utils.RinkebyFlag, utils.GoerliFlag, - utils.CalaverasFlag, }, Usage: "Inspect the storage size for each type of data in the database", Description: `This commands iterates the entire database. If the optional 'prefix' and 'start' arguments are provided, then the iteration is limited to the given subset of data.`, @@ -91,7 +90,6 @@ Remove blockchain and state databases`, utils.RopstenFlag, utils.RinkebyFlag, utils.GoerliFlag, - utils.CalaverasFlag, }, } dbCompactCmd = cli.Command{ @@ -105,7 +103,6 @@ Remove blockchain and state databases`, utils.RopstenFlag, utils.RinkebyFlag, utils.GoerliFlag, - utils.CalaverasFlag, utils.CacheFlag, utils.CacheDatabaseFlag, }, @@ -125,7 +122,6 @@ corruption if it is aborted during execution'!`, utils.RopstenFlag, utils.RinkebyFlag, utils.GoerliFlag, - utils.CalaverasFlag, }, Description: "This command looks up the specified database key from the database.", } @@ -141,7 +137,6 @@ corruption if it is aborted during execution'!`, utils.RopstenFlag, utils.RinkebyFlag, utils.GoerliFlag, - utils.CalaverasFlag, }, Description: `This command deletes the specified database key from the database. WARNING: This is a low-level operation which may cause database corruption!`, @@ -158,7 +153,6 @@ WARNING: This is a low-level operation which may cause database corruption!`, utils.RopstenFlag, utils.RinkebyFlag, utils.GoerliFlag, - utils.CalaverasFlag, }, Description: `This command sets a given database key to the given value. WARNING: This is a low-level operation which may cause database corruption!`, @@ -175,7 +169,6 @@ WARNING: This is a low-level operation which may cause database corruption!`, utils.RopstenFlag, utils.RinkebyFlag, utils.GoerliFlag, - utils.CalaverasFlag, }, Description: "This command looks up the specified database key from the database.", } @@ -191,7 +184,6 @@ WARNING: This is a low-level operation which may cause database corruption!`, utils.RopstenFlag, utils.RinkebyFlag, utils.GoerliFlag, - utils.CalaverasFlag, }, Description: "This command displays information about the freezer index.", } diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 642405c50..da62b8d4d 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -138,7 +138,6 @@ var ( utils.RopstenFlag, utils.RinkebyFlag, utils.GoerliFlag, - utils.CalaverasFlag, utils.VMEnableDebugFlag, utils.NetworkIdFlag, utils.EthStatsURLFlag, @@ -278,9 +277,6 @@ func prepare(ctx *cli.Context) { case ctx.GlobalIsSet(utils.GoerliFlag.Name): log.Info("Starting Geth on Görli testnet...") - case ctx.GlobalIsSet(utils.CalaverasFlag.Name): - log.Info("Starting Geth on Calaveras testnet...") - case ctx.GlobalIsSet(utils.DeveloperFlag.Name): log.Info("Starting Geth in ephemeral dev mode...") diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 3c401eae8..708edcc79 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -44,7 +44,6 @@ var AppHelpFlagGroups = []flags.FlagGroup{ utils.MainnetFlag, utils.GoerliFlag, utils.RinkebyFlag, - utils.CalaverasFlag, utils.RopstenFlag, utils.SyncModeFlag, utils.ExitWhenSyncedFlag, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 453a553eb..df97c8cf0 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -151,10 +151,6 @@ var ( Name: "goerli", Usage: "Görli network: pre-configured proof-of-authority test network", } - CalaverasFlag = cli.BoolFlag{ - Name: "calaveras", - Usage: "Calaveras network: pre-configured proof-of-authority shortlived test network.", - } RinkebyFlag = cli.BoolFlag{ Name: "rinkeby", Usage: "Rinkeby network: pre-configured proof-of-authority test network", @@ -801,9 +797,6 @@ func MakeDataDir(ctx *cli.Context) string { if ctx.GlobalBool(GoerliFlag.Name) { return filepath.Join(path, "goerli") } - if ctx.GlobalBool(CalaverasFlag.Name) { - return filepath.Join(path, "calaveras") - } return path } Fatalf("Cannot determine default data directory, please set manually (--datadir)") @@ -856,8 +849,6 @@ func setBootstrapNodes(ctx *cli.Context, cfg *p2p.Config) { urls = params.RinkebyBootnodes case ctx.GlobalBool(GoerliFlag.Name): urls = params.GoerliBootnodes - case ctx.GlobalBool(CalaverasFlag.Name): - urls = params.CalaverasBootnodes case cfg.BootstrapNodes != nil: return // already set, don't apply defaults. } @@ -1301,8 +1292,6 @@ func setDataDir(ctx *cli.Context, cfg *node.Config) { cfg.DataDir = filepath.Join(node.DefaultDataDir(), "rinkeby") case ctx.GlobalBool(GoerliFlag.Name) && cfg.DataDir == node.DefaultDataDir(): cfg.DataDir = filepath.Join(node.DefaultDataDir(), "goerli") - case ctx.GlobalBool(CalaverasFlag.Name) && cfg.DataDir == node.DefaultDataDir(): - cfg.DataDir = filepath.Join(node.DefaultDataDir(), "calaveras") } } @@ -1488,7 +1477,7 @@ func CheckExclusive(ctx *cli.Context, args ...interface{}) { // SetEthConfig applies eth-related command line flags to the config. func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { // Avoid conflicting network flags - CheckExclusive(ctx, MainnetFlag, DeveloperFlag, RopstenFlag, RinkebyFlag, GoerliFlag, CalaverasFlag) + CheckExclusive(ctx, MainnetFlag, DeveloperFlag, RopstenFlag, RinkebyFlag, GoerliFlag) CheckExclusive(ctx, LightServeFlag, SyncModeFlag, "light") CheckExclusive(ctx, DeveloperFlag, ExternalSignerFlag) // Can't use both ephemeral unlocked and external signer if ctx.GlobalString(GCModeFlag.Name) == "archive" && ctx.GlobalUint64(TxLookupLimitFlag.Name) != 0 { @@ -1641,11 +1630,6 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *ethconfig.Config) { } cfg.Genesis = core.DefaultGoerliGenesisBlock() SetDNSDiscoveryDefaults(cfg, params.GoerliGenesisHash) - case ctx.GlobalBool(CalaverasFlag.Name): - if !ctx.GlobalIsSet(NetworkIdFlag.Name) { - cfg.NetworkId = 123 // https://gist.github.com/holiman/c5697b041b3dc18c50a5cdd382cbdd16 - } - cfg.Genesis = core.DefaultCalaverasGenesisBlock() case ctx.GlobalBool(DeveloperFlag.Name): if !ctx.GlobalIsSet(NetworkIdFlag.Name) { cfg.NetworkId = 1337 @@ -1866,8 +1850,6 @@ func MakeGenesis(ctx *cli.Context) *core.Genesis { genesis = core.DefaultRinkebyGenesisBlock() case ctx.GlobalBool(GoerliFlag.Name): genesis = core.DefaultGoerliGenesisBlock() - case ctx.GlobalBool(CalaverasFlag.Name): - genesis = core.DefaultCalaverasGenesisBlock() case ctx.GlobalBool(DeveloperFlag.Name): Fatalf("Developer chains are ephemeral") } diff --git a/core/genesis.go b/core/genesis.go index dc50ff475..c1f226c34 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -248,8 +248,6 @@ func (g *Genesis) configOrDefault(ghash common.Hash) *params.ChainConfig { return params.RinkebyChainConfig case ghash == params.GoerliGenesisHash: return params.GoerliChainConfig - case ghash == params.CalaverasGenesisHash: - return params.CalaverasChainConfig default: return params.AllEthashProtocolChanges } @@ -399,18 +397,6 @@ func DefaultGoerliGenesisBlock() *Genesis { } } -func DefaultCalaverasGenesisBlock() *Genesis { - // Full genesis: https://gist.github.com/holiman/c6ed9269dce28304ad176314caa75e97 - return &Genesis{ - Config: params.CalaverasChainConfig, - Timestamp: 0x60b3877f, - ExtraData: hexutil.MustDecode("0x00000000000000000000000000000000000000000000000000000000000000005211cea3870c7ba7c6c44b185e62eecdb864cd8c560228ce57d31efbf64c200b2c200aacec78cf17a7148e784fe95a7a750335f8b9572ee28d72e7650000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"), - GasLimit: 0x47b760, - Difficulty: big.NewInt(1), - Alloc: decodePrealloc(calaverasAllocData), - } -} - // DeveloperGenesisBlock returns the 'geth --dev' genesis block. func DeveloperGenesisBlock(period uint64, faucet common.Address) *Genesis { // Override the default period to the user requested one diff --git a/core/genesis_test.go b/core/genesis_test.go index 487c30ed6..52c4384ab 100644 --- a/core/genesis_test.go +++ b/core/genesis_test.go @@ -185,10 +185,6 @@ func TestGenesisHashes(t *testing.T) { genesis: DefaultRinkebyGenesisBlock(), hash: params.RinkebyGenesisHash, }, - { - genesis: DefaultCalaverasGenesisBlock(), - hash: params.CalaverasGenesisHash, - }, } for i, c := range cases { b := c.genesis.MustCommit(rawdb.NewMemoryDatabase()) diff --git a/params/bootnodes.go b/params/bootnodes.go index 1610ea819..bc291449e 100644 --- a/params/bootnodes.go +++ b/params/bootnodes.go @@ -67,12 +67,6 @@ var GoerliBootnodes = []string{ "enode://a59e33ccd2b3e52d578f1fbd70c6f9babda2650f0760d6ff3b37742fdcdfdb3defba5d56d315b40c46b70198c7621e63ffa3f987389c7118634b0fefbbdfa7fd@51.15.119.157:40303", } -// CalaverasBootnodes are the enode URLs of the P2P bootstrap nodes running on the -// Calaveras ephemeral test network. -var CalaverasBootnodes = []string{ - "enode://9e1096aa59862a6f164994cb5cb16f5124d6c992cdbf4535ff7dea43ea1512afe5448dca9df1b7ab0726129603f1a3336b631e4d7a1a44c94daddd03241587f9@3.9.20.133:30303", -} - var V5Bootnodes = []string{ // Teku team's bootnode "enr:-KG4QOtcP9X1FbIMOe17QNMKqDxCpm14jcX5tiOE4_TyMrFqbmhPZHK_ZPG2Gxb1GE2xdtodOfx9-cgvNtxnRyHEmC0ghGV0aDKQ9aX9QgAAAAD__________4JpZIJ2NIJpcIQDE8KdiXNlY3AyNTZrMaEDhpehBDbZjM_L9ek699Y7vhUJ-eAdMyQW_Fil522Y0fODdGNwgiMog3VkcIIjKA", diff --git a/params/config.go b/params/config.go index 591c36de2..16c801825 100644 --- a/params/config.go +++ b/params/config.go @@ -27,11 +27,10 @@ import ( // Genesis hashes to enforce below configs on. var ( - MainnetGenesisHash = common.HexToHash("0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3") - RopstenGenesisHash = common.HexToHash("0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d") - RinkebyGenesisHash = common.HexToHash("0x6341fd3daf94b748c72ced5a5b26028f2474f5f00d824504e4fa37a75767e177") - GoerliGenesisHash = common.HexToHash("0xbf7e331f7f7c1dd2e05159666b3bf8bc7a8a3a9eb1d518969eab529dd9b88c1a") - CalaverasGenesisHash = common.HexToHash("0xeb9233d066c275efcdfed8037f4fc082770176aefdbcb7691c71da412a5670f2") + MainnetGenesisHash = common.HexToHash("0xd4e56740f876aef8c010b86a40d5f56745a118d0906a34e69aec8c0db1cb8fa3") + RopstenGenesisHash = common.HexToHash("0x41941023680923e0fe4d74a34bdac8141f2540e3ae90623718e47d66d1ca4a2d") + RinkebyGenesisHash = common.HexToHash("0x6341fd3daf94b748c72ced5a5b26028f2474f5f00d824504e4fa37a75767e177") + GoerliGenesisHash = common.HexToHash("0xbf7e331f7f7c1dd2e05159666b3bf8bc7a8a3a9eb1d518969eab529dd9b88c1a") ) // TrustedCheckpoints associates each known checkpoint with the genesis hash of @@ -221,27 +220,6 @@ var ( Threshold: 2, } - CalaverasChainConfig = &ChainConfig{ - ChainID: big.NewInt(123), - HomesteadBlock: big.NewInt(0), - DAOForkBlock: nil, - DAOForkSupport: true, - EIP150Block: big.NewInt(0), - EIP155Block: big.NewInt(0), - EIP158Block: big.NewInt(0), - ByzantiumBlock: big.NewInt(0), - ConstantinopleBlock: big.NewInt(0), - PetersburgBlock: big.NewInt(0), - IstanbulBlock: big.NewInt(0), - MuirGlacierBlock: nil, - BerlinBlock: big.NewInt(0), - LondonBlock: big.NewInt(500), - Clique: &CliqueConfig{ - Period: 30, - Epoch: 30000, - }, - } - // AllEthashProtocolChanges contains every protocol change (EIPs) introduced // and accepted by the Ethereum core developers into the Ethash consensus. // From 16ecdd583984162923256e563320b8b381da7f46 Mon Sep 17 00:00:00 2001 From: Zachinquarantine Date: Tue, 17 Aug 2021 12:49:19 -0400 Subject: [PATCH 08/17] cmd/utils: add --nousb to the list of deprecated flags (#23388) Adds --nousb as a deprecated flag when someone runs the geth show-deprecated-flags command. --- cmd/utils/flags.go | 4 ---- cmd/utils/flags_legacy.go | 5 +++++ 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index df97c8cf0..8a53f525f 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -125,10 +125,6 @@ var ( Name: "keystore", Usage: "Directory for the keystore (default = inside the datadir)", } - NoUSBFlag = cli.BoolFlag{ - Name: "nousb", - Usage: "Disables monitoring for and managing USB hardware wallets (deprecated)", - } USBFlag = cli.BoolFlag{ Name: "usb", Usage: "Enable monitoring and management of USB hardware wallets", diff --git a/cmd/utils/flags_legacy.go b/cmd/utils/flags_legacy.go index df597e365..627f77475 100644 --- a/cmd/utils/flags_legacy.go +++ b/cmd/utils/flags_legacy.go @@ -36,10 +36,15 @@ var ShowDeprecated = cli.Command{ var DeprecatedFlags = []cli.Flag{ LegacyMinerGasTargetFlag, + NoUSBFlag, } var ( // (Deprecated May 2020, shown in aliased flags section) + NoUSBFlag = cli.BoolFlag{ + Name: "nousb", + Usage: "Disables monitoring for and managing USB hardware wallets (deprecated)", + } LegacyRPCEnabledFlag = cli.BoolFlag{ Name: "rpc", Usage: "Enable the HTTP-RPC server (deprecated and will be removed June 2021, use --http)", From 57feabea663496109e59df669238398239438fb1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E9=99=88=E4=BD=B3?= Date: Wed, 18 Aug 2021 00:55:18 +0800 Subject: [PATCH 09/17] eth, internal/ethapi: make RPC block miner field show block sealer correctly (#23312) Makes the RPC block return the POA sealer for clique blocks on the 'miner' field (was previously zeroes) --- eth/api.go | 2 +- internal/ethapi/api.go | 14 ++++++++------ 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/eth/api.go b/eth/api.go index 8b96d1f31..0f57128d7 100644 --- a/eth/api.go +++ b/eth/api.go @@ -342,7 +342,7 @@ func (api *PrivateDebugAPI) GetBadBlocks(ctx context.Context) ([]*BadBlockArgs, } else { blockRlp = fmt.Sprintf("0x%x", rlpBytes) } - if blockJSON, err = ethapi.RPCMarshalBlock(block, true, true); err != nil { + if blockJSON, err = ethapi.RPCMarshalBlock(block, true, true, api.eth.engine); err != nil { blockJSON = map[string]interface{}{"error": err.Error()} } results = append(results, &BadBlockArgs{ diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 9a82824ad..1af98e107 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -32,6 +32,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/common/math" + "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/consensus/clique" "github.com/ethereum/go-ethereum/consensus/ethash" "github.com/ethereum/go-ethereum/consensus/misc" @@ -1175,7 +1176,8 @@ func FormatLogs(logs []vm.StructLog) []StructLogRes { } // RPCMarshalHeader converts the given header to the RPC output . -func RPCMarshalHeader(head *types.Header) map[string]interface{} { +func RPCMarshalHeader(head *types.Header, engine consensus.Engine) map[string]interface{} { + miner, _ := engine.Author(head) result := map[string]interface{}{ "number": (*hexutil.Big)(head.Number), "hash": head.Hash(), @@ -1185,7 +1187,7 @@ func RPCMarshalHeader(head *types.Header) map[string]interface{} { "sha3Uncles": head.UncleHash, "logsBloom": head.Bloom, "stateRoot": head.Root, - "miner": head.Coinbase, + "miner": miner, "difficulty": (*hexutil.Big)(head.Difficulty), "extraData": hexutil.Bytes(head.Extra), "size": hexutil.Uint64(head.Size()), @@ -1206,8 +1208,8 @@ func RPCMarshalHeader(head *types.Header) map[string]interface{} { // RPCMarshalBlock converts the given block to the RPC output which depends on fullTx. If inclTx is true transactions are // returned. When fullTx is true the returned block contains full transaction details, otherwise it will only contain // transaction hashes. -func RPCMarshalBlock(block *types.Block, inclTx bool, fullTx bool) (map[string]interface{}, error) { - fields := RPCMarshalHeader(block.Header()) +func RPCMarshalBlock(block *types.Block, inclTx bool, fullTx bool, engine consensus.Engine) (map[string]interface{}, error) { + fields := RPCMarshalHeader(block.Header(), engine) fields["size"] = hexutil.Uint64(block.Size()) if inclTx { @@ -1242,7 +1244,7 @@ func RPCMarshalBlock(block *types.Block, inclTx bool, fullTx bool) (map[string]i // rpcMarshalHeader uses the generalized output filler, then adds the total difficulty field, which requires // a `PublicBlockchainAPI`. func (s *PublicBlockChainAPI) rpcMarshalHeader(ctx context.Context, header *types.Header) map[string]interface{} { - fields := RPCMarshalHeader(header) + fields := RPCMarshalHeader(header, s.b.Engine()) fields["totalDifficulty"] = (*hexutil.Big)(s.b.GetTd(ctx, header.Hash())) return fields } @@ -1250,7 +1252,7 @@ func (s *PublicBlockChainAPI) rpcMarshalHeader(ctx context.Context, header *type // rpcMarshalBlock uses the generalized output filler, then adds the total difficulty field, which requires // a `PublicBlockchainAPI`. func (s *PublicBlockChainAPI) rpcMarshalBlock(ctx context.Context, b *types.Block, inclTx bool, fullTx bool) (map[string]interface{}, error) { - fields, err := RPCMarshalBlock(b, inclTx, fullTx) + fields, err := RPCMarshalBlock(b, inclTx, fullTx, s.b.Engine()) if err != nil { return nil, err } From 5566e5d152c490e5e533f3bc7735dddd57a428eb Mon Sep 17 00:00:00 2001 From: chuwt Date: Wed, 18 Aug 2021 18:03:41 +0800 Subject: [PATCH 10/17] eth/downloader: fix typo in comment (#23413) --- core/rawdb/freezer.go | 2 +- eth/downloader/downloader.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/core/rawdb/freezer.go b/core/rawdb/freezer.go index 253de9f7c..4262a615a 100644 --- a/core/rawdb/freezer.go +++ b/core/rawdb/freezer.go @@ -414,7 +414,7 @@ func (f *freezer) freeze(db ethdb.KeyValueStore) { } batch.Reset() - // Wipe out side chains also and track dangling side chians + // Wipe out side chains also and track dangling side chains var dangling []common.Hash for number := first; number < f.frozen; number++ { // Always keep the genesis block in active database diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index f0fab1b7a..a6bf87acb 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -465,7 +465,7 @@ func (d *Downloader) syncWithPeer(p *peerConnection, hash common.Hash, td *big.I } if mode == FastSync && pivot == nil { // If no pivot block was returned, the head is below the min full block - // threshold (i.e. new chian). In that case we won't really fast sync + // threshold (i.e. new chain). In that case we won't really fast sync // anyway, but still need a valid pivot block to avoid some code hitting // nil panics on an access. pivot = d.blockchain.CurrentBlock().Header() @@ -681,7 +681,7 @@ func (d *Downloader) fetchHead(p *peerConnection) (head *types.Header, pivot *ty return head, nil, nil } // At this point we have 2 headers in total and the first is the - // validated head of the chian. Check the pivot number and return, + // validated head of the chain. Check the pivot number and return, pivot := headers[1] if pivot.Number.Uint64() != head.Number.Uint64()-uint64(fsMinFullBlocks) { return nil, nil, fmt.Errorf("%w: remote pivot %d != requested %d", errInvalidChain, pivot.Number, head.Number.Uint64()-uint64(fsMinFullBlocks)) From c368f728c19e7fd7a9613513edda68ffcb503af0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Fri, 20 Aug 2021 15:14:21 +0300 Subject: [PATCH 11/17] Revert "eth: drop eth/65, the last non-reqid protocol version" (#23426) --- cmd/devp2p/internal/ethtest/suite_test.go | 2 +- eth/downloader/downloader.go | 4 +- eth/downloader/downloader_test.go | 95 ++++++++++- eth/downloader/peer.go | 8 +- eth/handler.go | 5 +- eth/handler_eth_test.go | 38 ++--- eth/protocols/eth/handler.go | 47 ++++-- eth/protocols/eth/handler_test.go | 134 +++++++++------ eth/protocols/eth/handlers.go | 107 ++++++++++++ eth/protocols/eth/handshake_test.go | 1 + eth/protocols/eth/peer.go | 194 +++++++++++++++------- eth/protocols/eth/protocol.go | 34 ++-- eth/sync.go | 102 +++++++++++- eth/sync_test.go | 1 + les/client_handler.go | 2 +- 15 files changed, 602 insertions(+), 172 deletions(-) diff --git a/cmd/devp2p/internal/ethtest/suite_test.go b/cmd/devp2p/internal/ethtest/suite_test.go index 6d14404e6..6e3217151 100644 --- a/cmd/devp2p/internal/ethtest/suite_test.go +++ b/cmd/devp2p/internal/ethtest/suite_test.go @@ -45,7 +45,7 @@ func TestEthSuite(t *testing.T) { if err != nil { t.Fatalf("could not create new test suite: %v", err) } - for _, test := range suite.Eth66Tests() { + for _, test := range suite.AllEthTests() { t.Run(test.Name, func(t *testing.T) { result := utesting.RunTAP([]utesting.Test{{Name: test.Name, Fn: test.Fn}}, os.Stdout) if result[0].Failed { diff --git a/eth/downloader/downloader.go b/eth/downloader/downloader.go index a6bf87acb..1767506a3 100644 --- a/eth/downloader/downloader.go +++ b/eth/downloader/downloader.go @@ -448,8 +448,8 @@ func (d *Downloader) syncWithPeer(p *peerConnection, hash common.Hash, td *big.I d.mux.Post(DoneEvent{latest}) } }() - if p.version < eth.ETH66 { - return fmt.Errorf("%w: advertized %d < required %d", errTooOld, p.version, eth.ETH66) + if p.version < eth.ETH65 { + return fmt.Errorf("%w: advertized %d < required %d", errTooOld, p.version, eth.ETH65) } mode := d.getMode() diff --git a/eth/downloader/downloader_test.go b/eth/downloader/downloader_test.go index 17cd3630c..794160993 100644 --- a/eth/downloader/downloader_test.go +++ b/eth/downloader/downloader_test.go @@ -522,6 +522,10 @@ func assertOwnForkedChain(t *testing.T, tester *downloadTester, common int, leng } } +func TestCanonicalSynchronisation65Full(t *testing.T) { testCanonSync(t, eth.ETH65, FullSync) } +func TestCanonicalSynchronisation65Fast(t *testing.T) { testCanonSync(t, eth.ETH65, FastSync) } +func TestCanonicalSynchronisation65Light(t *testing.T) { testCanonSync(t, eth.ETH65, LightSync) } + func TestCanonicalSynchronisation66Full(t *testing.T) { testCanonSync(t, eth.ETH66, FullSync) } func TestCanonicalSynchronisation66Fast(t *testing.T) { testCanonSync(t, eth.ETH66, FastSync) } func TestCanonicalSynchronisation66Light(t *testing.T) { testCanonSync(t, eth.ETH66, LightSync) } @@ -545,6 +549,9 @@ func testCanonSync(t *testing.T, protocol uint, mode SyncMode) { // Tests that if a large batch of blocks are being downloaded, it is throttled // until the cached blocks are retrieved. +func TestThrottling65Full(t *testing.T) { testThrottling(t, eth.ETH65, FullSync) } +func TestThrottling65Fast(t *testing.T) { testThrottling(t, eth.ETH65, FastSync) } + func TestThrottling66Full(t *testing.T) { testThrottling(t, eth.ETH66, FullSync) } func TestThrottling66Fast(t *testing.T) { testThrottling(t, eth.ETH66, FastSync) } @@ -627,6 +634,10 @@ func testThrottling(t *testing.T, protocol uint, mode SyncMode) { // Tests that simple synchronization against a forked chain works correctly. In // this test common ancestor lookup should *not* be short circuited, and a full // binary search should be executed. +func TestForkedSync65Full(t *testing.T) { testForkedSync(t, eth.ETH65, FullSync) } +func TestForkedSync65Fast(t *testing.T) { testForkedSync(t, eth.ETH65, FastSync) } +func TestForkedSync65Light(t *testing.T) { testForkedSync(t, eth.ETH65, LightSync) } + func TestForkedSync66Full(t *testing.T) { testForkedSync(t, eth.ETH66, FullSync) } func TestForkedSync66Fast(t *testing.T) { testForkedSync(t, eth.ETH66, FastSync) } func TestForkedSync66Light(t *testing.T) { testForkedSync(t, eth.ETH66, LightSync) } @@ -656,6 +667,10 @@ func testForkedSync(t *testing.T, protocol uint, mode SyncMode) { // Tests that synchronising against a much shorter but much heavyer fork works // corrently and is not dropped. +func TestHeavyForkedSync65Full(t *testing.T) { testHeavyForkedSync(t, eth.ETH65, FullSync) } +func TestHeavyForkedSync65Fast(t *testing.T) { testHeavyForkedSync(t, eth.ETH65, FastSync) } +func TestHeavyForkedSync65Light(t *testing.T) { testHeavyForkedSync(t, eth.ETH65, LightSync) } + func TestHeavyForkedSync66Full(t *testing.T) { testHeavyForkedSync(t, eth.ETH66, FullSync) } func TestHeavyForkedSync66Fast(t *testing.T) { testHeavyForkedSync(t, eth.ETH66, FastSync) } func TestHeavyForkedSync66Light(t *testing.T) { testHeavyForkedSync(t, eth.ETH66, LightSync) } @@ -687,6 +702,10 @@ func testHeavyForkedSync(t *testing.T, protocol uint, mode SyncMode) { // Tests that chain forks are contained within a certain interval of the current // chain head, ensuring that malicious peers cannot waste resources by feeding // long dead chains. +func TestBoundedForkedSync65Full(t *testing.T) { testBoundedForkedSync(t, eth.ETH65, FullSync) } +func TestBoundedForkedSync65Fast(t *testing.T) { testBoundedForkedSync(t, eth.ETH65, FastSync) } +func TestBoundedForkedSync65Light(t *testing.T) { testBoundedForkedSync(t, eth.ETH65, LightSync) } + func TestBoundedForkedSync66Full(t *testing.T) { testBoundedForkedSync(t, eth.ETH66, FullSync) } func TestBoundedForkedSync66Fast(t *testing.T) { testBoundedForkedSync(t, eth.ETH66, FastSync) } func TestBoundedForkedSync66Light(t *testing.T) { testBoundedForkedSync(t, eth.ETH66, LightSync) } @@ -717,6 +736,16 @@ func testBoundedForkedSync(t *testing.T, protocol uint, mode SyncMode) { // Tests that chain forks are contained within a certain interval of the current // chain head for short but heavy forks too. These are a bit special because they // take different ancestor lookup paths. +func TestBoundedHeavyForkedSync65Full(t *testing.T) { + testBoundedHeavyForkedSync(t, eth.ETH65, FullSync) +} +func TestBoundedHeavyForkedSync65Fast(t *testing.T) { + testBoundedHeavyForkedSync(t, eth.ETH65, FastSync) +} +func TestBoundedHeavyForkedSync65Light(t *testing.T) { + testBoundedHeavyForkedSync(t, eth.ETH65, LightSync) +} + func TestBoundedHeavyForkedSync66Full(t *testing.T) { testBoundedHeavyForkedSync(t, eth.ETH66, FullSync) } @@ -771,6 +800,10 @@ func TestInactiveDownloader63(t *testing.T) { } // Tests that a canceled download wipes all previously accumulated state. +func TestCancel65Full(t *testing.T) { testCancel(t, eth.ETH65, FullSync) } +func TestCancel65Fast(t *testing.T) { testCancel(t, eth.ETH65, FastSync) } +func TestCancel65Light(t *testing.T) { testCancel(t, eth.ETH65, LightSync) } + func TestCancel66Full(t *testing.T) { testCancel(t, eth.ETH66, FullSync) } func TestCancel66Fast(t *testing.T) { testCancel(t, eth.ETH66, FastSync) } func TestCancel66Light(t *testing.T) { testCancel(t, eth.ETH66, LightSync) } @@ -800,6 +833,10 @@ func testCancel(t *testing.T, protocol uint, mode SyncMode) { } // Tests that synchronisation from multiple peers works as intended (multi thread sanity test). +func TestMultiSynchronisation65Full(t *testing.T) { testMultiSynchronisation(t, eth.ETH65, FullSync) } +func TestMultiSynchronisation65Fast(t *testing.T) { testMultiSynchronisation(t, eth.ETH65, FastSync) } +func TestMultiSynchronisation65Light(t *testing.T) { testMultiSynchronisation(t, eth.ETH65, LightSync) } + func TestMultiSynchronisation66Full(t *testing.T) { testMultiSynchronisation(t, eth.ETH66, FullSync) } func TestMultiSynchronisation66Fast(t *testing.T) { testMultiSynchronisation(t, eth.ETH66, FastSync) } func TestMultiSynchronisation66Light(t *testing.T) { testMultiSynchronisation(t, eth.ETH66, LightSync) } @@ -826,6 +863,10 @@ func testMultiSynchronisation(t *testing.T, protocol uint, mode SyncMode) { // Tests that synchronisations behave well in multi-version protocol environments // and not wreak havoc on other nodes in the network. +func TestMultiProtoSynchronisation65Full(t *testing.T) { testMultiProtoSync(t, eth.ETH65, FullSync) } +func TestMultiProtoSynchronisation65Fast(t *testing.T) { testMultiProtoSync(t, eth.ETH65, FastSync) } +func TestMultiProtoSynchronisation65Light(t *testing.T) { testMultiProtoSync(t, eth.ETH65, LightSync) } + func TestMultiProtoSynchronisation66Full(t *testing.T) { testMultiProtoSync(t, eth.ETH66, FullSync) } func TestMultiProtoSynchronisation66Fast(t *testing.T) { testMultiProtoSync(t, eth.ETH66, FastSync) } func TestMultiProtoSynchronisation66Light(t *testing.T) { testMultiProtoSync(t, eth.ETH66, LightSync) } @@ -840,8 +881,8 @@ func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { chain := testChainBase.shorten(blockCacheMaxItems - 15) // Create peers of every type + tester.newPeer("peer 65", eth.ETH65, chain) tester.newPeer("peer 66", eth.ETH66, chain) - //tester.newPeer("peer 65", eth.ETH67, chain) // Synchronise with the requested peer and make sure all blocks were retrieved if err := tester.sync(fmt.Sprintf("peer %d", protocol), nil, mode); err != nil { @@ -850,7 +891,7 @@ func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { assertOwnChain(t, tester, chain.len()) // Check that no peers have been dropped off - for _, version := range []int{66} { + for _, version := range []int{65, 66} { peer := fmt.Sprintf("peer %d", version) if _, ok := tester.peers[peer]; !ok { t.Errorf("%s dropped", peer) @@ -860,6 +901,10 @@ func testMultiProtoSync(t *testing.T, protocol uint, mode SyncMode) { // Tests that if a block is empty (e.g. header only), no body request should be // made, and instead the header should be assembled into a whole block in itself. +func TestEmptyShortCircuit65Full(t *testing.T) { testEmptyShortCircuit(t, eth.ETH65, FullSync) } +func TestEmptyShortCircuit65Fast(t *testing.T) { testEmptyShortCircuit(t, eth.ETH65, FastSync) } +func TestEmptyShortCircuit65Light(t *testing.T) { testEmptyShortCircuit(t, eth.ETH65, LightSync) } + func TestEmptyShortCircuit66Full(t *testing.T) { testEmptyShortCircuit(t, eth.ETH66, FullSync) } func TestEmptyShortCircuit66Fast(t *testing.T) { testEmptyShortCircuit(t, eth.ETH66, FastSync) } func TestEmptyShortCircuit66Light(t *testing.T) { testEmptyShortCircuit(t, eth.ETH66, LightSync) } @@ -910,6 +955,10 @@ func testEmptyShortCircuit(t *testing.T, protocol uint, mode SyncMode) { // Tests that headers are enqueued continuously, preventing malicious nodes from // stalling the downloader by feeding gapped header chains. +func TestMissingHeaderAttack65Full(t *testing.T) { testMissingHeaderAttack(t, eth.ETH65, FullSync) } +func TestMissingHeaderAttack65Fast(t *testing.T) { testMissingHeaderAttack(t, eth.ETH65, FastSync) } +func TestMissingHeaderAttack65Light(t *testing.T) { testMissingHeaderAttack(t, eth.ETH65, LightSync) } + func TestMissingHeaderAttack66Full(t *testing.T) { testMissingHeaderAttack(t, eth.ETH66, FullSync) } func TestMissingHeaderAttack66Fast(t *testing.T) { testMissingHeaderAttack(t, eth.ETH66, FastSync) } func TestMissingHeaderAttack66Light(t *testing.T) { testMissingHeaderAttack(t, eth.ETH66, LightSync) } @@ -938,6 +987,10 @@ func testMissingHeaderAttack(t *testing.T, protocol uint, mode SyncMode) { // Tests that if requested headers are shifted (i.e. first is missing), the queue // detects the invalid numbering. +func TestShiftedHeaderAttack65Full(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH65, FullSync) } +func TestShiftedHeaderAttack65Fast(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH65, FastSync) } +func TestShiftedHeaderAttack65Light(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH65, LightSync) } + func TestShiftedHeaderAttack66Full(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH66, FullSync) } func TestShiftedHeaderAttack66Fast(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH66, FastSync) } func TestShiftedHeaderAttack66Light(t *testing.T) { testShiftedHeaderAttack(t, eth.ETH66, LightSync) } @@ -971,6 +1024,7 @@ func testShiftedHeaderAttack(t *testing.T, protocol uint, mode SyncMode) { // Tests that upon detecting an invalid header, the recent ones are rolled back // for various failure scenarios. Afterwards a full sync is attempted to make // sure no state was corrupted. +func TestInvalidHeaderRollback65Fast(t *testing.T) { testInvalidHeaderRollback(t, eth.ETH65, FastSync) } func TestInvalidHeaderRollback66Fast(t *testing.T) { testInvalidHeaderRollback(t, eth.ETH66, FastSync) } func testInvalidHeaderRollback(t *testing.T, protocol uint, mode SyncMode) { @@ -1061,6 +1115,16 @@ func testInvalidHeaderRollback(t *testing.T, protocol uint, mode SyncMode) { // Tests that a peer advertising a high TD doesn't get to stall the downloader // afterwards by not sending any useful hashes. +func TestHighTDStarvationAttack65Full(t *testing.T) { + testHighTDStarvationAttack(t, eth.ETH65, FullSync) +} +func TestHighTDStarvationAttack65Fast(t *testing.T) { + testHighTDStarvationAttack(t, eth.ETH65, FastSync) +} +func TestHighTDStarvationAttack65Light(t *testing.T) { + testHighTDStarvationAttack(t, eth.ETH65, LightSync) +} + func TestHighTDStarvationAttack66Full(t *testing.T) { testHighTDStarvationAttack(t, eth.ETH66, FullSync) } @@ -1085,6 +1149,7 @@ func testHighTDStarvationAttack(t *testing.T, protocol uint, mode SyncMode) { } // Tests that misbehaving peers are disconnected, whilst behaving ones are not. +func TestBlockHeaderAttackerDropping65(t *testing.T) { testBlockHeaderAttackerDropping(t, eth.ETH65) } func TestBlockHeaderAttackerDropping66(t *testing.T) { testBlockHeaderAttackerDropping(t, eth.ETH66) } func testBlockHeaderAttackerDropping(t *testing.T, protocol uint) { @@ -1137,6 +1202,10 @@ func testBlockHeaderAttackerDropping(t *testing.T, protocol uint) { // Tests that synchronisation progress (origin block number, current block number // and highest block number) is tracked and updated correctly. +func TestSyncProgress65Full(t *testing.T) { testSyncProgress(t, eth.ETH65, FullSync) } +func TestSyncProgress65Fast(t *testing.T) { testSyncProgress(t, eth.ETH65, FastSync) } +func TestSyncProgress65Light(t *testing.T) { testSyncProgress(t, eth.ETH65, LightSync) } + func TestSyncProgress66Full(t *testing.T) { testSyncProgress(t, eth.ETH66, FullSync) } func TestSyncProgress66Fast(t *testing.T) { testSyncProgress(t, eth.ETH66, FastSync) } func TestSyncProgress66Light(t *testing.T) { testSyncProgress(t, eth.ETH66, LightSync) } @@ -1217,6 +1286,10 @@ func checkProgress(t *testing.T, d *Downloader, stage string, want ethereum.Sync // Tests that synchronisation progress (origin block number and highest block // number) is tracked and updated correctly in case of a fork (or manual head // revertal). +func TestForkedSyncProgress65Full(t *testing.T) { testForkedSyncProgress(t, eth.ETH65, FullSync) } +func TestForkedSyncProgress65Fast(t *testing.T) { testForkedSyncProgress(t, eth.ETH65, FastSync) } +func TestForkedSyncProgress65Light(t *testing.T) { testForkedSyncProgress(t, eth.ETH65, LightSync) } + func TestForkedSyncProgress66Full(t *testing.T) { testForkedSyncProgress(t, eth.ETH66, FullSync) } func TestForkedSyncProgress66Fast(t *testing.T) { testForkedSyncProgress(t, eth.ETH66, FastSync) } func TestForkedSyncProgress66Light(t *testing.T) { testForkedSyncProgress(t, eth.ETH66, LightSync) } @@ -1289,6 +1362,10 @@ func testForkedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { // Tests that if synchronisation is aborted due to some failure, then the progress // origin is not updated in the next sync cycle, as it should be considered the // continuation of the previous sync and not a new instance. +func TestFailedSyncProgress65Full(t *testing.T) { testFailedSyncProgress(t, eth.ETH65, FullSync) } +func TestFailedSyncProgress65Fast(t *testing.T) { testFailedSyncProgress(t, eth.ETH65, FastSync) } +func TestFailedSyncProgress65Light(t *testing.T) { testFailedSyncProgress(t, eth.ETH65, LightSync) } + func TestFailedSyncProgress66Full(t *testing.T) { testFailedSyncProgress(t, eth.ETH66, FullSync) } func TestFailedSyncProgress66Fast(t *testing.T) { testFailedSyncProgress(t, eth.ETH66, FastSync) } func TestFailedSyncProgress66Light(t *testing.T) { testFailedSyncProgress(t, eth.ETH66, LightSync) } @@ -1358,6 +1435,10 @@ func testFailedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { // Tests that if an attacker fakes a chain height, after the attack is detected, // the progress height is successfully reduced at the next sync invocation. +func TestFakedSyncProgress65Full(t *testing.T) { testFakedSyncProgress(t, eth.ETH65, FullSync) } +func TestFakedSyncProgress65Fast(t *testing.T) { testFakedSyncProgress(t, eth.ETH65, FastSync) } +func TestFakedSyncProgress65Light(t *testing.T) { testFakedSyncProgress(t, eth.ETH65, LightSync) } + func TestFakedSyncProgress66Full(t *testing.T) { testFakedSyncProgress(t, eth.ETH66, FullSync) } func TestFakedSyncProgress66Fast(t *testing.T) { testFakedSyncProgress(t, eth.ETH66, FastSync) } func TestFakedSyncProgress66Light(t *testing.T) { testFakedSyncProgress(t, eth.ETH66, LightSync) } @@ -1431,6 +1512,10 @@ func testFakedSyncProgress(t *testing.T, protocol uint, mode SyncMode) { // This test reproduces an issue where unexpected deliveries would // block indefinitely if they arrived at the right time. +func TestDeliverHeadersHang65Full(t *testing.T) { testDeliverHeadersHang(t, eth.ETH65, FullSync) } +func TestDeliverHeadersHang65Fast(t *testing.T) { testDeliverHeadersHang(t, eth.ETH65, FastSync) } +func TestDeliverHeadersHang65Light(t *testing.T) { testDeliverHeadersHang(t, eth.ETH65, LightSync) } + func TestDeliverHeadersHang66Full(t *testing.T) { testDeliverHeadersHang(t, eth.ETH66, FullSync) } func TestDeliverHeadersHang66Fast(t *testing.T) { testDeliverHeadersHang(t, eth.ETH66, FastSync) } func TestDeliverHeadersHang66Light(t *testing.T) { testDeliverHeadersHang(t, eth.ETH66, LightSync) } @@ -1588,6 +1673,12 @@ func TestRemoteHeaderRequestSpan(t *testing.T) { // Tests that peers below a pre-configured checkpoint block are prevented from // being fast-synced from, avoiding potential cheap eclipse attacks. +func TestCheckpointEnforcement65Full(t *testing.T) { testCheckpointEnforcement(t, eth.ETH65, FullSync) } +func TestCheckpointEnforcement65Fast(t *testing.T) { testCheckpointEnforcement(t, eth.ETH65, FastSync) } +func TestCheckpointEnforcement65Light(t *testing.T) { + testCheckpointEnforcement(t, eth.ETH65, LightSync) +} + func TestCheckpointEnforcement66Full(t *testing.T) { testCheckpointEnforcement(t, eth.ETH66, FullSync) } func TestCheckpointEnforcement66Fast(t *testing.T) { testCheckpointEnforcement(t, eth.ETH66, FastSync) } func TestCheckpointEnforcement66Light(t *testing.T) { diff --git a/eth/downloader/peer.go b/eth/downloader/peer.go index 863294832..066a36631 100644 --- a/eth/downloader/peer.go +++ b/eth/downloader/peer.go @@ -413,7 +413,7 @@ func (ps *peerSet) HeaderIdlePeers() ([]*peerConnection, int) { throughput := func(p *peerConnection) int { return p.rates.Capacity(eth.BlockHeadersMsg, time.Second) } - return ps.idlePeers(eth.ETH66, eth.ETH66, idle, throughput) + return ps.idlePeers(eth.ETH65, eth.ETH66, idle, throughput) } // BodyIdlePeers retrieves a flat list of all the currently body-idle peers within @@ -425,7 +425,7 @@ func (ps *peerSet) BodyIdlePeers() ([]*peerConnection, int) { throughput := func(p *peerConnection) int { return p.rates.Capacity(eth.BlockBodiesMsg, time.Second) } - return ps.idlePeers(eth.ETH66, eth.ETH66, idle, throughput) + return ps.idlePeers(eth.ETH65, eth.ETH66, idle, throughput) } // ReceiptIdlePeers retrieves a flat list of all the currently receipt-idle peers @@ -437,7 +437,7 @@ func (ps *peerSet) ReceiptIdlePeers() ([]*peerConnection, int) { throughput := func(p *peerConnection) int { return p.rates.Capacity(eth.ReceiptsMsg, time.Second) } - return ps.idlePeers(eth.ETH66, eth.ETH66, idle, throughput) + return ps.idlePeers(eth.ETH65, eth.ETH66, idle, throughput) } // NodeDataIdlePeers retrieves a flat list of all the currently node-data-idle @@ -449,7 +449,7 @@ func (ps *peerSet) NodeDataIdlePeers() ([]*peerConnection, int) { throughput := func(p *peerConnection) int { return p.rates.Capacity(eth.NodeDataMsg, time.Second) } - return ps.idlePeers(eth.ETH66, eth.ETH66, idle, throughput) + return ps.idlePeers(eth.ETH65, eth.ETH66, idle, throughput) } // idlePeers retrieves a flat list of all currently idle peers satisfying the diff --git a/eth/handler.go b/eth/handler.go index 06a8088bf..aff4871af 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -117,6 +117,7 @@ type handler struct { whitelist map[uint64]common.Hash // channels for fetcher, syncer, txsyncLoop + txsyncCh chan *txsync quitSync chan struct{} chainSync *chainSyncer @@ -139,6 +140,7 @@ func newHandler(config *handlerConfig) (*handler, error) { chain: config.Chain, peers: newPeerSet(), whitelist: config.Whitelist, + txsyncCh: make(chan *txsync), quitSync: make(chan struct{}), } if config.Sync == downloader.FullSync { @@ -406,8 +408,9 @@ func (h *handler) Start(maxPeers int) { go h.minedBroadcastLoop() // start sync handlers - h.wg.Add(1) + h.wg.Add(2) go h.chainSync.loop() + go h.txsyncLoop64() // TODO(karalabe): Legacy initial tx echange, drop with eth/64. } func (h *handler) Stop() { diff --git a/eth/handler_eth_test.go b/eth/handler_eth_test.go index 039091244..038de4699 100644 --- a/eth/handler_eth_test.go +++ b/eth/handler_eth_test.go @@ -80,6 +80,7 @@ func (h *testEthHandler) Handle(peer *eth.Peer, packet eth.Packet) error { // Tests that peers are correctly accepted (or rejected) based on the advertised // fork IDs in the protocol handshake. +func TestForkIDSplit65(t *testing.T) { testForkIDSplit(t, eth.ETH65) } func TestForkIDSplit66(t *testing.T) { testForkIDSplit(t, eth.ETH66) } func testForkIDSplit(t *testing.T, protocol uint) { @@ -235,6 +236,7 @@ func testForkIDSplit(t *testing.T, protocol uint) { } // Tests that received transactions are added to the local pool. +func TestRecvTransactions65(t *testing.T) { testRecvTransactions(t, eth.ETH65) } func TestRecvTransactions66(t *testing.T) { testRecvTransactions(t, eth.ETH66) } func testRecvTransactions(t *testing.T, protocol uint) { @@ -292,6 +294,7 @@ func testRecvTransactions(t *testing.T, protocol uint) { } // This test checks that pending transactions are sent. +func TestSendTransactions65(t *testing.T) { testSendTransactions(t, eth.ETH65) } func TestSendTransactions66(t *testing.T) { testSendTransactions(t, eth.ETH66) } func testSendTransactions(t *testing.T, protocol uint) { @@ -303,7 +306,7 @@ func testSendTransactions(t *testing.T, protocol uint) { insert := make([]*types.Transaction, 100) for nonce := range insert { - tx := types.NewTransaction(uint64(nonce), common.Address{}, big.NewInt(0), 100000, big.NewInt(0), make([]byte, 10240)) + tx := types.NewTransaction(uint64(nonce), common.Address{}, big.NewInt(0), 100000, big.NewInt(0), make([]byte, txsyncPackSize/10)) tx, _ = types.SignTx(tx, types.HomesteadSigner{}, testKey) insert[nonce] = tx @@ -377,6 +380,7 @@ func testSendTransactions(t *testing.T, protocol uint) { // Tests that transactions get propagated to all attached peers, either via direct // broadcasts or via announcements/retrievals. +func TestTransactionPropagation65(t *testing.T) { testTransactionPropagation(t, eth.ETH65) } func TestTransactionPropagation66(t *testing.T) { testTransactionPropagation(t, eth.ETH66) } func testTransactionPropagation(t *testing.T, protocol uint) { @@ -517,8 +521,8 @@ func testCheckpointChallenge(t *testing.T, syncmode downloader.SyncMode, checkpo defer p2pLocal.Close() defer p2pRemote.Close() - local := eth.NewPeer(eth.ETH66, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pLocal), p2pLocal, handler.txpool) - remote := eth.NewPeer(eth.ETH66, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pRemote), p2pRemote, handler.txpool) + local := eth.NewPeer(eth.ETH65, p2p.NewPeerPipe(enode.ID{1}, "", nil, p2pLocal), p2pLocal, handler.txpool) + remote := eth.NewPeer(eth.ETH65, p2p.NewPeerPipe(enode.ID{2}, "", nil, p2pRemote), p2pRemote, handler.txpool) defer local.Close() defer remote.Close() @@ -539,39 +543,30 @@ func testCheckpointChallenge(t *testing.T, syncmode downloader.SyncMode, checkpo if err := remote.Handshake(1, td, head.Hash(), genesis.Hash(), forkid.NewIDWithChain(handler.chain), forkid.NewFilter(handler.chain)); err != nil { t.Fatalf("failed to run protocol handshake") } + // Connect a new peer and check that we receive the checkpoint challenge. if checkpoint { - msg, err := p2pRemote.ReadMsg() - if err != nil { - t.Fatalf("failed to read checkpoint challenge: %v", err) - } - request := new(eth.GetBlockHeadersPacket66) - if err := msg.Decode(request); err != nil { - t.Fatalf("failed to decode checkpoint challenge: %v", err) - } - query := request.GetBlockHeadersPacket - if query.Origin.Number != response.Number.Uint64() || query.Amount != 1 || query.Skip != 0 || query.Reverse { - t.Fatalf("challenge mismatch: have [%d, %d, %d, %v] want [%d, %d, %d, %v]", - query.Origin.Number, query.Amount, query.Skip, query.Reverse, - response.Number.Uint64(), 1, 0, false) + if err := remote.ExpectRequestHeadersByNumber(response.Number.Uint64(), 1, 0, false); err != nil { + t.Fatalf("challenge mismatch: %v", err) } // Create a block to reply to the challenge if no timeout is simulated. if !timeout { if empty { - if err := remote.ReplyBlockHeaders(request.RequestId, []*types.Header{}); err != nil { + if err := remote.SendBlockHeaders([]*types.Header{}); err != nil { t.Fatalf("failed to answer challenge: %v", err) } } else if match { - if err := remote.ReplyBlockHeaders(request.RequestId, []*types.Header{response}); err != nil { + if err := remote.SendBlockHeaders([]*types.Header{response}); err != nil { t.Fatalf("failed to answer challenge: %v", err) } } else { - if err := remote.ReplyBlockHeaders(request.RequestId, []*types.Header{{Number: response.Number}}); err != nil { + if err := remote.SendBlockHeaders([]*types.Header{{Number: response.Number}}); err != nil { t.Fatalf("failed to answer challenge: %v", err) } } } } + // Wait until the test timeout passes to ensure proper cleanup time.Sleep(syncChallengeTimeout + 300*time.Millisecond) @@ -624,8 +619,8 @@ func testBroadcastBlock(t *testing.T, peers, bcasts int) { defer sourcePipe.Close() defer sinkPipe.Close() - sourcePeer := eth.NewPeer(eth.ETH66, p2p.NewPeerPipe(enode.ID{byte(i)}, "", nil, sourcePipe), sourcePipe, nil) - sinkPeer := eth.NewPeer(eth.ETH66, p2p.NewPeerPipe(enode.ID{0}, "", nil, sinkPipe), sinkPipe, nil) + sourcePeer := eth.NewPeer(eth.ETH65, p2p.NewPeerPipe(enode.ID{byte(i)}, "", nil, sourcePipe), sourcePipe, nil) + sinkPeer := eth.NewPeer(eth.ETH65, p2p.NewPeerPipe(enode.ID{0}, "", nil, sinkPipe), sinkPipe, nil) defer sourcePeer.Close() defer sinkPeer.Close() @@ -676,6 +671,7 @@ func testBroadcastBlock(t *testing.T, peers, bcasts int) { // Tests that a propagated malformed block (uncles or transactions don't match // with the hashes in the header) gets discarded and not broadcast forward. +func TestBroadcastMalformedBlock65(t *testing.T) { testBroadcastMalformedBlock(t, eth.ETH65) } func TestBroadcastMalformedBlock66(t *testing.T) { testBroadcastMalformedBlock(t, eth.ETH66) } func testBroadcastMalformedBlock(t *testing.T, protocol uint) { diff --git a/eth/protocols/eth/handler.go b/eth/protocols/eth/handler.go index 828930014..6bbaa2f55 100644 --- a/eth/protocols/eth/handler.go +++ b/eth/protocols/eth/handler.go @@ -171,21 +171,39 @@ type Decoder interface { Time() time.Time } +var eth65 = map[uint64]msgHandler{ + GetBlockHeadersMsg: handleGetBlockHeaders, + BlockHeadersMsg: handleBlockHeaders, + GetBlockBodiesMsg: handleGetBlockBodies, + BlockBodiesMsg: handleBlockBodies, + GetNodeDataMsg: handleGetNodeData, + NodeDataMsg: handleNodeData, + GetReceiptsMsg: handleGetReceipts, + ReceiptsMsg: handleReceipts, + NewBlockHashesMsg: handleNewBlockhashes, + NewBlockMsg: handleNewBlock, + TransactionsMsg: handleTransactions, + NewPooledTransactionHashesMsg: handleNewPooledTransactionHashes, + GetPooledTransactionsMsg: handleGetPooledTransactions, + PooledTransactionsMsg: handlePooledTransactions, +} + var eth66 = map[uint64]msgHandler{ NewBlockHashesMsg: handleNewBlockhashes, NewBlockMsg: handleNewBlock, TransactionsMsg: handleTransactions, NewPooledTransactionHashesMsg: handleNewPooledTransactionHashes, - GetBlockHeadersMsg: handleGetBlockHeaders66, - BlockHeadersMsg: handleBlockHeaders66, - GetBlockBodiesMsg: handleGetBlockBodies66, - BlockBodiesMsg: handleBlockBodies66, - GetNodeDataMsg: handleGetNodeData66, - NodeDataMsg: handleNodeData66, - GetReceiptsMsg: handleGetReceipts66, - ReceiptsMsg: handleReceipts66, - GetPooledTransactionsMsg: handleGetPooledTransactions66, - PooledTransactionsMsg: handlePooledTransactions66, + // eth66 messages with request-id + GetBlockHeadersMsg: handleGetBlockHeaders66, + BlockHeadersMsg: handleBlockHeaders66, + GetBlockBodiesMsg: handleGetBlockBodies66, + BlockBodiesMsg: handleBlockBodies66, + GetNodeDataMsg: handleGetNodeData66, + NodeDataMsg: handleNodeData66, + GetReceiptsMsg: handleGetReceipts66, + ReceiptsMsg: handleReceipts66, + GetPooledTransactionsMsg: handleGetPooledTransactions66, + PooledTransactionsMsg: handlePooledTransactions66, } // handleMessage is invoked whenever an inbound message is received from a remote @@ -201,11 +219,10 @@ func handleMessage(backend Backend, peer *Peer) error { } defer msg.Discard() - var handlers = eth66 - //if peer.Version() >= ETH67 { // Left in as a sample when new protocol is added - // handlers = eth67 - //} - + var handlers = eth65 + if peer.Version() >= ETH66 { + handlers = eth66 + } // Track the amount of time it takes to serve the request and run the handler if metrics.Enabled { h := fmt.Sprintf("%s/%s/%d/%#02x", p2p.HandleHistName, ProtocolName, peer.Version(), msg.Code) diff --git a/eth/protocols/eth/handler_test.go b/eth/protocols/eth/handler_test.go index 809f17e36..473be3f9b 100644 --- a/eth/protocols/eth/handler_test.go +++ b/eth/protocols/eth/handler_test.go @@ -110,6 +110,7 @@ func (b *testBackend) Handle(*Peer, Packet) error { } // Tests that block headers can be retrieved from a remote chain based on user queries. +func TestGetBlockHeaders65(t *testing.T) { testGetBlockHeaders(t, ETH65) } func TestGetBlockHeaders66(t *testing.T) { testGetBlockHeaders(t, ETH66) } func testGetBlockHeaders(t *testing.T, protocol uint) { @@ -253,30 +254,44 @@ func testGetBlockHeaders(t *testing.T, protocol uint) { headers = append(headers, backend.chain.GetBlockByHash(hash).Header()) } // Send the hash request and verify the response - p2p.Send(peer.app, GetBlockHeadersMsg, GetBlockHeadersPacket66{ - RequestId: 123, - GetBlockHeadersPacket: tt.query, - }) - if err := p2p.ExpectMsg(peer.app, BlockHeadersMsg, BlockHeadersPacket66{ - RequestId: 123, - BlockHeadersPacket: headers, - }); err != nil { - t.Errorf("test %d: headers mismatch: %v", i, err) + if protocol <= ETH65 { + p2p.Send(peer.app, GetBlockHeadersMsg, tt.query) + if err := p2p.ExpectMsg(peer.app, BlockHeadersMsg, headers); err != nil { + t.Errorf("test %d: headers mismatch: %v", i, err) + } + } else { + p2p.Send(peer.app, GetBlockHeadersMsg, GetBlockHeadersPacket66{ + RequestId: 123, + GetBlockHeadersPacket: tt.query, + }) + if err := p2p.ExpectMsg(peer.app, BlockHeadersMsg, BlockHeadersPacket66{ + RequestId: 123, + BlockHeadersPacket: headers, + }); err != nil { + t.Errorf("test %d: headers mismatch: %v", i, err) + } } // If the test used number origins, repeat with hashes as the too if tt.query.Origin.Hash == (common.Hash{}) { if origin := backend.chain.GetBlockByNumber(tt.query.Origin.Number); origin != nil { tt.query.Origin.Hash, tt.query.Origin.Number = origin.Hash(), 0 - p2p.Send(peer.app, GetBlockHeadersMsg, GetBlockHeadersPacket66{ - RequestId: 456, - GetBlockHeadersPacket: tt.query, - }) - if err := p2p.ExpectMsg(peer.app, BlockHeadersMsg, BlockHeadersPacket66{ - RequestId: 456, - BlockHeadersPacket: headers, - }); err != nil { - t.Errorf("test %d: headers mismatch: %v", i, err) + if protocol <= ETH65 { + p2p.Send(peer.app, GetBlockHeadersMsg, tt.query) + if err := p2p.ExpectMsg(peer.app, BlockHeadersMsg, headers); err != nil { + t.Errorf("test %d: headers mismatch: %v", i, err) + } + } else { + p2p.Send(peer.app, GetBlockHeadersMsg, GetBlockHeadersPacket66{ + RequestId: 456, + GetBlockHeadersPacket: tt.query, + }) + if err := p2p.ExpectMsg(peer.app, BlockHeadersMsg, BlockHeadersPacket66{ + RequestId: 456, + BlockHeadersPacket: headers, + }); err != nil { + t.Errorf("test %d: headers mismatch: %v", i, err) + } } } } @@ -284,6 +299,7 @@ func testGetBlockHeaders(t *testing.T, protocol uint) { } // Tests that block contents can be retrieved from a remote chain based on their hashes. +func TestGetBlockBodies65(t *testing.T) { testGetBlockBodies(t, ETH65) } func TestGetBlockBodies66(t *testing.T) { testGetBlockBodies(t, ETH66) } func testGetBlockBodies(t *testing.T, protocol uint) { @@ -353,20 +369,28 @@ func testGetBlockBodies(t *testing.T, protocol uint) { } } // Send the hash request and verify the response - p2p.Send(peer.app, GetBlockBodiesMsg, GetBlockBodiesPacket66{ - RequestId: 123, - GetBlockBodiesPacket: hashes, - }) - if err := p2p.ExpectMsg(peer.app, BlockBodiesMsg, BlockBodiesPacket66{ - RequestId: 123, - BlockBodiesPacket: bodies, - }); err != nil { - t.Errorf("test %d: bodies mismatch: %v", i, err) + if protocol <= ETH65 { + p2p.Send(peer.app, GetBlockBodiesMsg, hashes) + if err := p2p.ExpectMsg(peer.app, BlockBodiesMsg, bodies); err != nil { + t.Errorf("test %d: bodies mismatch: %v", i, err) + } + } else { + p2p.Send(peer.app, GetBlockBodiesMsg, GetBlockBodiesPacket66{ + RequestId: 123, + GetBlockBodiesPacket: hashes, + }) + if err := p2p.ExpectMsg(peer.app, BlockBodiesMsg, BlockBodiesPacket66{ + RequestId: 123, + BlockBodiesPacket: bodies, + }); err != nil { + t.Errorf("test %d: bodies mismatch: %v", i, err) + } } } } // Tests that the state trie nodes can be retrieved based on hashes. +func TestGetNodeData65(t *testing.T) { testGetNodeData(t, ETH65) } func TestGetNodeData66(t *testing.T) { testGetNodeData(t, ETH66) } func testGetNodeData(t *testing.T, protocol uint) { @@ -425,10 +449,14 @@ func testGetNodeData(t *testing.T, protocol uint) { } it.Release() - p2p.Send(peer.app, GetNodeDataMsg, GetNodeDataPacket66{ - RequestId: 123, - GetNodeDataPacket: hashes, - }) + if protocol <= ETH65 { + p2p.Send(peer.app, GetNodeDataMsg, hashes) + } else { + p2p.Send(peer.app, GetNodeDataMsg, GetNodeDataPacket66{ + RequestId: 123, + GetNodeDataPacket: hashes, + }) + } msg, err := peer.app.ReadMsg() if err != nil { t.Fatalf("failed to read node data response: %v", err) @@ -436,14 +464,18 @@ func testGetNodeData(t *testing.T, protocol uint) { if msg.Code != NodeDataMsg { t.Fatalf("response packet code mismatch: have %x, want %x", msg.Code, NodeDataMsg) } - var ( - data [][]byte - res NodeDataPacket66 - ) - if err := msg.Decode(&res); err != nil { - t.Fatalf("failed to decode response node data: %v", err) + var data [][]byte + if protocol <= ETH65 { + if err := msg.Decode(&data); err != nil { + t.Fatalf("failed to decode response node data: %v", err) + } + } else { + var res NodeDataPacket66 + if err := msg.Decode(&res); err != nil { + t.Fatalf("failed to decode response node data: %v", err) + } + data = res.NodeDataPacket } - data = res.NodeDataPacket // Verify that all hashes correspond to the requested data, and reconstruct a state tree for i, want := range hashes { if hash := crypto.Keccak256Hash(data[i]); hash != want { @@ -474,6 +506,7 @@ func testGetNodeData(t *testing.T, protocol uint) { } // Tests that the transaction receipts can be retrieved based on hashes. +func TestGetBlockReceipts65(t *testing.T) { testGetBlockReceipts(t, ETH65) } func TestGetBlockReceipts66(t *testing.T) { testGetBlockReceipts(t, ETH66) } func testGetBlockReceipts(t *testing.T, protocol uint) { @@ -533,14 +566,21 @@ func testGetBlockReceipts(t *testing.T, protocol uint) { receipts = append(receipts, backend.chain.GetReceiptsByHash(block.Hash())) } // Send the hash request and verify the response - p2p.Send(peer.app, GetReceiptsMsg, GetReceiptsPacket66{ - RequestId: 123, - GetReceiptsPacket: hashes, - }) - if err := p2p.ExpectMsg(peer.app, ReceiptsMsg, ReceiptsPacket66{ - RequestId: 123, - ReceiptsPacket: receipts, - }); err != nil { - t.Errorf("receipts mismatch: %v", err) + if protocol <= ETH65 { + p2p.Send(peer.app, GetReceiptsMsg, hashes) + if err := p2p.ExpectMsg(peer.app, ReceiptsMsg, receipts); err != nil { + t.Errorf("receipts mismatch: %v", err) + } + } else { + p2p.Send(peer.app, GetReceiptsMsg, GetReceiptsPacket66{ + RequestId: 123, + GetReceiptsPacket: hashes, + }) + if err := p2p.ExpectMsg(peer.app, ReceiptsMsg, ReceiptsPacket66{ + RequestId: 123, + ReceiptsPacket: receipts, + }); err != nil { + t.Errorf("receipts mismatch: %v", err) + } } } diff --git a/eth/protocols/eth/handlers.go b/eth/protocols/eth/handlers.go index e54838cbc..d7d993a23 100644 --- a/eth/protocols/eth/handlers.go +++ b/eth/protocols/eth/handlers.go @@ -27,6 +27,17 @@ import ( "github.com/ethereum/go-ethereum/trie" ) +// handleGetBlockHeaders handles Block header query, collect the requested headers and reply +func handleGetBlockHeaders(backend Backend, msg Decoder, peer *Peer) error { + // Decode the complex header query + var query GetBlockHeadersPacket + if err := msg.Decode(&query); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + response := answerGetBlockHeadersQuery(backend, &query, peer) + return peer.SendBlockHeaders(response) +} + // handleGetBlockHeaders66 is the eth/66 version of handleGetBlockHeaders func handleGetBlockHeaders66(backend Backend, msg Decoder, peer *Peer) error { // Decode the complex header query @@ -124,6 +135,16 @@ func answerGetBlockHeadersQuery(backend Backend, query *GetBlockHeadersPacket, p return headers } +func handleGetBlockBodies(backend Backend, msg Decoder, peer *Peer) error { + // Decode the block body retrieval message + var query GetBlockBodiesPacket + if err := msg.Decode(&query); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + response := answerGetBlockBodiesQuery(backend, query, peer) + return peer.SendBlockBodiesRLP(response) +} + func handleGetBlockBodies66(backend Backend, msg Decoder, peer *Peer) error { // Decode the block body retrieval message var query GetBlockBodiesPacket66 @@ -153,6 +174,16 @@ func answerGetBlockBodiesQuery(backend Backend, query GetBlockBodiesPacket, peer return bodies } +func handleGetNodeData(backend Backend, msg Decoder, peer *Peer) error { + // Decode the trie node data retrieval message + var query GetNodeDataPacket + if err := msg.Decode(&query); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + response := answerGetNodeDataQuery(backend, query, peer) + return peer.SendNodeData(response) +} + func handleGetNodeData66(backend Backend, msg Decoder, peer *Peer) error { // Decode the trie node data retrieval message var query GetNodeDataPacket66 @@ -192,6 +223,16 @@ func answerGetNodeDataQuery(backend Backend, query GetNodeDataPacket, peer *Peer return nodes } +func handleGetReceipts(backend Backend, msg Decoder, peer *Peer) error { + // Decode the block receipts retrieval message + var query GetReceiptsPacket + if err := msg.Decode(&query); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + response := answerGetReceiptsQuery(backend, query, peer) + return peer.SendReceiptsRLP(response) +} + func handleGetReceipts66(backend Backend, msg Decoder, peer *Peer) error { // Decode the block receipts retrieval message var query GetReceiptsPacket66 @@ -271,6 +312,15 @@ func handleNewBlock(backend Backend, msg Decoder, peer *Peer) error { return backend.Handle(peer, ann) } +func handleBlockHeaders(backend Backend, msg Decoder, peer *Peer) error { + // A batch of headers arrived to one of our previous requests + res := new(BlockHeadersPacket) + if err := msg.Decode(res); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + return backend.Handle(peer, res) +} + func handleBlockHeaders66(backend Backend, msg Decoder, peer *Peer) error { // A batch of headers arrived to one of our previous requests res := new(BlockHeadersPacket66) @@ -282,6 +332,15 @@ func handleBlockHeaders66(backend Backend, msg Decoder, peer *Peer) error { return backend.Handle(peer, &res.BlockHeadersPacket) } +func handleBlockBodies(backend Backend, msg Decoder, peer *Peer) error { + // A batch of block bodies arrived to one of our previous requests + res := new(BlockBodiesPacket) + if err := msg.Decode(res); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + return backend.Handle(peer, res) +} + func handleBlockBodies66(backend Backend, msg Decoder, peer *Peer) error { // A batch of block bodies arrived to one of our previous requests res := new(BlockBodiesPacket66) @@ -293,6 +352,15 @@ func handleBlockBodies66(backend Backend, msg Decoder, peer *Peer) error { return backend.Handle(peer, &res.BlockBodiesPacket) } +func handleNodeData(backend Backend, msg Decoder, peer *Peer) error { + // A batch of node state data arrived to one of our previous requests + res := new(NodeDataPacket) + if err := msg.Decode(res); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + return backend.Handle(peer, res) +} + func handleNodeData66(backend Backend, msg Decoder, peer *Peer) error { // A batch of node state data arrived to one of our previous requests res := new(NodeDataPacket66) @@ -304,6 +372,15 @@ func handleNodeData66(backend Backend, msg Decoder, peer *Peer) error { return backend.Handle(peer, &res.NodeDataPacket) } +func handleReceipts(backend Backend, msg Decoder, peer *Peer) error { + // A batch of receipts arrived to one of our previous requests + res := new(ReceiptsPacket) + if err := msg.Decode(res); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + return backend.Handle(peer, res) +} + func handleReceipts66(backend Backend, msg Decoder, peer *Peer) error { // A batch of receipts arrived to one of our previous requests res := new(ReceiptsPacket66) @@ -332,6 +409,16 @@ func handleNewPooledTransactionHashes(backend Backend, msg Decoder, peer *Peer) return backend.Handle(peer, ann) } +func handleGetPooledTransactions(backend Backend, msg Decoder, peer *Peer) error { + // Decode the pooled transactions retrieval message + var query GetPooledTransactionsPacket + if err := msg.Decode(&query); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + hashes, txs := answerGetPooledTransactions(backend, query, peer) + return peer.SendPooledTransactionsRLP(hashes, txs) +} + func handleGetPooledTransactions66(backend Backend, msg Decoder, peer *Peer) error { // Decode the pooled transactions retrieval message var query GetPooledTransactionsPacket66 @@ -390,6 +477,26 @@ func handleTransactions(backend Backend, msg Decoder, peer *Peer) error { return backend.Handle(peer, &txs) } +func handlePooledTransactions(backend Backend, msg Decoder, peer *Peer) error { + // Transactions arrived, make sure we have a valid and fresh chain to handle them + if !backend.AcceptTxs() { + return nil + } + // Transactions can be processed, parse all of them and deliver to the pool + var txs PooledTransactionsPacket + if err := msg.Decode(&txs); err != nil { + return fmt.Errorf("%w: message %v: %v", errDecode, msg, err) + } + for i, tx := range txs { + // Validate and mark the remote transaction + if tx == nil { + return fmt.Errorf("%w: transaction %d is nil", errDecode, i) + } + peer.markTransaction(tx.Hash()) + } + return backend.Handle(peer, &txs) +} + func handlePooledTransactions66(backend Backend, msg Decoder, peer *Peer) error { // Transactions arrived, make sure we have a valid and fresh chain to handle them if !backend.AcceptTxs() { diff --git a/eth/protocols/eth/handshake_test.go b/eth/protocols/eth/handshake_test.go index 05d473e05..3bebda2dc 100644 --- a/eth/protocols/eth/handshake_test.go +++ b/eth/protocols/eth/handshake_test.go @@ -27,6 +27,7 @@ import ( ) // Tests that handshake failures are detected and reported correctly. +func TestHandshake65(t *testing.T) { testHandshake(t, ETH65) } func TestHandshake66(t *testing.T) { testHandshake(t, ETH66) } func testHandshake(t *testing.T, protocol uint) { diff --git a/eth/protocols/eth/peer.go b/eth/protocols/eth/peer.go index 98273ccfc..e619c183b 100644 --- a/eth/protocols/eth/peer.go +++ b/eth/protocols/eth/peer.go @@ -108,8 +108,9 @@ func NewPeer(version uint, p *p2p.Peer, rw p2p.MsgReadWriter, txpool TxPool) *Pe // Start up all the broadcasters go peer.broadcastBlocks() go peer.broadcastTransactions() - go peer.announceTransactions() - + if version >= ETH65 { + go peer.announceTransactions() + } return peer } @@ -251,6 +252,22 @@ func (p *Peer) AsyncSendPooledTransactionHashes(hashes []common.Hash) { } } +// SendPooledTransactionsRLP sends requested transactions to the peer and adds the +// hashes in its transaction hash set for future reference. +// +// Note, the method assumes the hashes are correct and correspond to the list of +// transactions being sent. +func (p *Peer) SendPooledTransactionsRLP(hashes []common.Hash, txs []rlp.RawValue) error { + // Mark all the transactions as known, but ensure we don't overflow our limits + for p.knownTxs.Cardinality() > max(0, maxKnownTxs-len(hashes)) { + p.knownTxs.Pop() + } + for _, hash := range hashes { + p.knownTxs.Add(hash) + } + return p2p.Send(p.rw, PooledTransactionsMsg, txs) // Not packed into PooledTransactionsPacket to avoid RLP decoding +} + // ReplyPooledTransactionsRLP is the eth/66 version of SendPooledTransactionsRLP. func (p *Peer) ReplyPooledTransactionsRLP(id uint64, hashes []common.Hash, txs []rlp.RawValue) error { // Mark all the transactions as known, but ensure we don't overflow our limits @@ -329,6 +346,11 @@ func (p *Peer) AsyncSendNewBlock(block *types.Block, td *big.Int) { } } +// SendBlockHeaders sends a batch of block headers to the remote peer. +func (p *Peer) SendBlockHeaders(headers []*types.Header) error { + return p2p.Send(p.rw, BlockHeadersMsg, BlockHeadersPacket(headers)) +} + // ReplyBlockHeaders is the eth/66 version of SendBlockHeaders. func (p *Peer) ReplyBlockHeaders(id uint64, headers []*types.Header) error { return p2p.Send(p.rw, BlockHeadersMsg, BlockHeadersPacket66{ @@ -337,6 +359,12 @@ func (p *Peer) ReplyBlockHeaders(id uint64, headers []*types.Header) error { }) } +// SendBlockBodiesRLP sends a batch of block contents to the remote peer from +// an already RLP encoded format. +func (p *Peer) SendBlockBodiesRLP(bodies []rlp.RawValue) error { + return p2p.Send(p.rw, BlockBodiesMsg, bodies) // Not packed into BlockBodiesPacket to avoid RLP decoding +} + // ReplyBlockBodiesRLP is the eth/66 version of SendBlockBodiesRLP. func (p *Peer) ReplyBlockBodiesRLP(id uint64, bodies []rlp.RawValue) error { // Not packed into BlockBodiesPacket to avoid RLP decoding @@ -346,6 +374,12 @@ func (p *Peer) ReplyBlockBodiesRLP(id uint64, bodies []rlp.RawValue) error { }) } +// SendNodeDataRLP sends a batch of arbitrary internal data, corresponding to the +// hashes requested. +func (p *Peer) SendNodeData(data [][]byte) error { + return p2p.Send(p.rw, NodeDataMsg, NodeDataPacket(data)) +} + // ReplyNodeData is the eth/66 response to GetNodeData. func (p *Peer) ReplyNodeData(id uint64, data [][]byte) error { return p2p.Send(p.rw, NodeDataMsg, NodeDataPacket66{ @@ -354,6 +388,12 @@ func (p *Peer) ReplyNodeData(id uint64, data [][]byte) error { }) } +// SendReceiptsRLP sends a batch of transaction receipts, corresponding to the +// ones requested from an already RLP encoded format. +func (p *Peer) SendReceiptsRLP(receipts []rlp.RawValue) error { + return p2p.Send(p.rw, ReceiptsMsg, receipts) // Not packed into ReceiptsPacket to avoid RLP decoding +} + // ReplyReceiptsRLP is the eth/66 response to GetReceipts. func (p *Peer) ReplyReceiptsRLP(id uint64, receipts []rlp.RawValue) error { return p2p.Send(p.rw, ReceiptsMsg, ReceiptsRLPPacket66{ @@ -366,102 +406,138 @@ func (p *Peer) ReplyReceiptsRLP(id uint64, receipts []rlp.RawValue) error { // single header. It is used solely by the fetcher. func (p *Peer) RequestOneHeader(hash common.Hash) error { p.Log().Debug("Fetching single header", "hash", hash) - id := rand.Uint64() + query := GetBlockHeadersPacket{ + Origin: HashOrNumber{Hash: hash}, + Amount: uint64(1), + Skip: uint64(0), + Reverse: false, + } + if p.Version() >= ETH66 { + id := rand.Uint64() - requestTracker.Track(p.id, p.version, GetBlockHeadersMsg, BlockHeadersMsg, id) - return p2p.Send(p.rw, GetBlockHeadersMsg, &GetBlockHeadersPacket66{ - RequestId: id, - GetBlockHeadersPacket: &GetBlockHeadersPacket{ - Origin: HashOrNumber{Hash: hash}, - Amount: uint64(1), - Skip: uint64(0), - Reverse: false, - }, - }) + requestTracker.Track(p.id, p.version, GetBlockHeadersMsg, BlockHeadersMsg, id) + return p2p.Send(p.rw, GetBlockHeadersMsg, &GetBlockHeadersPacket66{ + RequestId: id, + GetBlockHeadersPacket: &query, + }) + } + return p2p.Send(p.rw, GetBlockHeadersMsg, &query) } // RequestHeadersByHash fetches a batch of blocks' headers corresponding to the // specified header query, based on the hash of an origin block. func (p *Peer) RequestHeadersByHash(origin common.Hash, amount int, skip int, reverse bool) error { p.Log().Debug("Fetching batch of headers", "count", amount, "fromhash", origin, "skip", skip, "reverse", reverse) - id := rand.Uint64() + query := GetBlockHeadersPacket{ + Origin: HashOrNumber{Hash: origin}, + Amount: uint64(amount), + Skip: uint64(skip), + Reverse: reverse, + } + if p.Version() >= ETH66 { + id := rand.Uint64() - requestTracker.Track(p.id, p.version, GetBlockHeadersMsg, BlockHeadersMsg, id) - return p2p.Send(p.rw, GetBlockHeadersMsg, &GetBlockHeadersPacket66{ - RequestId: id, - GetBlockHeadersPacket: &GetBlockHeadersPacket{ - Origin: HashOrNumber{Hash: origin}, - Amount: uint64(amount), - Skip: uint64(skip), - Reverse: reverse, - }, - }) + requestTracker.Track(p.id, p.version, GetBlockHeadersMsg, BlockHeadersMsg, id) + return p2p.Send(p.rw, GetBlockHeadersMsg, &GetBlockHeadersPacket66{ + RequestId: id, + GetBlockHeadersPacket: &query, + }) + } + return p2p.Send(p.rw, GetBlockHeadersMsg, &query) } // RequestHeadersByNumber fetches a batch of blocks' headers corresponding to the // specified header query, based on the number of an origin block. func (p *Peer) RequestHeadersByNumber(origin uint64, amount int, skip int, reverse bool) error { p.Log().Debug("Fetching batch of headers", "count", amount, "fromnum", origin, "skip", skip, "reverse", reverse) - id := rand.Uint64() + query := GetBlockHeadersPacket{ + Origin: HashOrNumber{Number: origin}, + Amount: uint64(amount), + Skip: uint64(skip), + Reverse: reverse, + } + if p.Version() >= ETH66 { + id := rand.Uint64() - requestTracker.Track(p.id, p.version, GetBlockHeadersMsg, BlockHeadersMsg, id) - return p2p.Send(p.rw, GetBlockHeadersMsg, &GetBlockHeadersPacket66{ - RequestId: id, - GetBlockHeadersPacket: &GetBlockHeadersPacket{ - Origin: HashOrNumber{Number: origin}, - Amount: uint64(amount), - Skip: uint64(skip), - Reverse: reverse, - }, - }) + requestTracker.Track(p.id, p.version, GetBlockHeadersMsg, BlockHeadersMsg, id) + return p2p.Send(p.rw, GetBlockHeadersMsg, &GetBlockHeadersPacket66{ + RequestId: id, + GetBlockHeadersPacket: &query, + }) + } + return p2p.Send(p.rw, GetBlockHeadersMsg, &query) +} + +// ExpectRequestHeadersByNumber is a testing method to mirror the recipient side +// of the RequestHeadersByNumber operation. +func (p *Peer) ExpectRequestHeadersByNumber(origin uint64, amount int, skip int, reverse bool) error { + req := &GetBlockHeadersPacket{ + Origin: HashOrNumber{Number: origin}, + Amount: uint64(amount), + Skip: uint64(skip), + Reverse: reverse, + } + return p2p.ExpectMsg(p.rw, GetBlockHeadersMsg, req) } // RequestBodies fetches a batch of blocks' bodies corresponding to the hashes // specified. func (p *Peer) RequestBodies(hashes []common.Hash) error { p.Log().Debug("Fetching batch of block bodies", "count", len(hashes)) - id := rand.Uint64() + if p.Version() >= ETH66 { + id := rand.Uint64() - requestTracker.Track(p.id, p.version, GetBlockBodiesMsg, BlockBodiesMsg, id) - return p2p.Send(p.rw, GetBlockBodiesMsg, &GetBlockBodiesPacket66{ - RequestId: id, - GetBlockBodiesPacket: hashes, - }) + requestTracker.Track(p.id, p.version, GetBlockBodiesMsg, BlockBodiesMsg, id) + return p2p.Send(p.rw, GetBlockBodiesMsg, &GetBlockBodiesPacket66{ + RequestId: id, + GetBlockBodiesPacket: hashes, + }) + } + return p2p.Send(p.rw, GetBlockBodiesMsg, GetBlockBodiesPacket(hashes)) } // RequestNodeData fetches a batch of arbitrary data from a node's known state // data, corresponding to the specified hashes. func (p *Peer) RequestNodeData(hashes []common.Hash) error { p.Log().Debug("Fetching batch of state data", "count", len(hashes)) - id := rand.Uint64() + if p.Version() >= ETH66 { + id := rand.Uint64() - requestTracker.Track(p.id, p.version, GetNodeDataMsg, NodeDataMsg, id) - return p2p.Send(p.rw, GetNodeDataMsg, &GetNodeDataPacket66{ - RequestId: id, - GetNodeDataPacket: hashes, - }) + requestTracker.Track(p.id, p.version, GetNodeDataMsg, NodeDataMsg, id) + return p2p.Send(p.rw, GetNodeDataMsg, &GetNodeDataPacket66{ + RequestId: id, + GetNodeDataPacket: hashes, + }) + } + return p2p.Send(p.rw, GetNodeDataMsg, GetNodeDataPacket(hashes)) } // RequestReceipts fetches a batch of transaction receipts from a remote node. func (p *Peer) RequestReceipts(hashes []common.Hash) error { p.Log().Debug("Fetching batch of receipts", "count", len(hashes)) - id := rand.Uint64() + if p.Version() >= ETH66 { + id := rand.Uint64() - requestTracker.Track(p.id, p.version, GetReceiptsMsg, ReceiptsMsg, id) - return p2p.Send(p.rw, GetReceiptsMsg, &GetReceiptsPacket66{ - RequestId: id, - GetReceiptsPacket: hashes, - }) + requestTracker.Track(p.id, p.version, GetReceiptsMsg, ReceiptsMsg, id) + return p2p.Send(p.rw, GetReceiptsMsg, &GetReceiptsPacket66{ + RequestId: id, + GetReceiptsPacket: hashes, + }) + } + return p2p.Send(p.rw, GetReceiptsMsg, GetReceiptsPacket(hashes)) } // RequestTxs fetches a batch of transactions from a remote node. func (p *Peer) RequestTxs(hashes []common.Hash) error { p.Log().Debug("Fetching batch of transactions", "count", len(hashes)) - id := rand.Uint64() + if p.Version() >= ETH66 { + id := rand.Uint64() - requestTracker.Track(p.id, p.version, GetPooledTransactionsMsg, PooledTransactionsMsg, id) - return p2p.Send(p.rw, GetPooledTransactionsMsg, &GetPooledTransactionsPacket66{ - RequestId: id, - GetPooledTransactionsPacket: hashes, - }) + requestTracker.Track(p.id, p.version, GetPooledTransactionsMsg, PooledTransactionsMsg, id) + return p2p.Send(p.rw, GetPooledTransactionsMsg, &GetPooledTransactionsPacket66{ + RequestId: id, + GetPooledTransactionsPacket: hashes, + }) + } + return p2p.Send(p.rw, GetPooledTransactionsMsg, GetPooledTransactionsPacket(hashes)) } diff --git a/eth/protocols/eth/protocol.go b/eth/protocols/eth/protocol.go index 3c3da30fa..de1b0ed1e 100644 --- a/eth/protocols/eth/protocol.go +++ b/eth/protocols/eth/protocol.go @@ -30,6 +30,7 @@ import ( // Constants to match up protocol versions and messages const ( + ETH65 = 65 ETH66 = 66 ) @@ -39,28 +40,31 @@ const ProtocolName = "eth" // ProtocolVersions are the supported versions of the `eth` protocol (first // is primary). -var ProtocolVersions = []uint{ETH66} +var ProtocolVersions = []uint{ETH66, ETH65} // protocolLengths are the number of implemented message corresponding to // different protocol versions. -var protocolLengths = map[uint]uint64{ETH66: 17} +var protocolLengths = map[uint]uint64{ETH66: 17, ETH65: 17} // maxMessageSize is the maximum cap on the size of a protocol message. const maxMessageSize = 10 * 1024 * 1024 const ( - StatusMsg = 0x00 - NewBlockHashesMsg = 0x01 - TransactionsMsg = 0x02 - GetBlockHeadersMsg = 0x03 - BlockHeadersMsg = 0x04 - GetBlockBodiesMsg = 0x05 - BlockBodiesMsg = 0x06 - NewBlockMsg = 0x07 - GetNodeDataMsg = 0x0d - NodeDataMsg = 0x0e - GetReceiptsMsg = 0x0f - ReceiptsMsg = 0x10 + // Protocol messages in eth/64 + StatusMsg = 0x00 + NewBlockHashesMsg = 0x01 + TransactionsMsg = 0x02 + GetBlockHeadersMsg = 0x03 + BlockHeadersMsg = 0x04 + GetBlockBodiesMsg = 0x05 + BlockBodiesMsg = 0x06 + NewBlockMsg = 0x07 + GetNodeDataMsg = 0x0d + NodeDataMsg = 0x0e + GetReceiptsMsg = 0x0f + ReceiptsMsg = 0x10 + + // Protocol messages overloaded in eth/65 NewPooledTransactionHashesMsg = 0x08 GetPooledTransactionsMsg = 0x09 PooledTransactionsMsg = 0x0a @@ -124,7 +128,7 @@ type GetBlockHeadersPacket struct { Reverse bool // Query direction (false = rising towards latest, true = falling towards genesis) } -// GetBlockHeadersPacket66 represents a block header query over eth/66 +// GetBlockHeadersPacket represents a block header query over eth/66 type GetBlockHeadersPacket66 struct { RequestId uint64 *GetBlockHeadersPacket diff --git a/eth/sync.go b/eth/sync.go index 27941158f..ab114b59f 100644 --- a/eth/sync.go +++ b/eth/sync.go @@ -18,6 +18,7 @@ package eth import ( "math/big" + "math/rand" "sync/atomic" "time" @@ -27,13 +28,23 @@ import ( "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/eth/protocols/eth" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p/enode" ) const ( forceSyncCycle = 10 * time.Second // Time interval to force syncs, even if few peers are available defaultMinSyncPeers = 5 // Amount of peers desired to start syncing + + // This is the target size for the packs of transactions sent by txsyncLoop64. + // A pack can get larger than this if a single transactions exceeds this size. + txsyncPackSize = 100 * 1024 ) +type txsync struct { + p *eth.Peer + txs []*types.Transaction +} + // syncTransactions starts sending all currently pending transactions to the given peer. func (h *handler) syncTransactions(p *eth.Peer) { // Assemble the set of transaction to broadcast or announce to the remote @@ -53,11 +64,94 @@ func (h *handler) syncTransactions(p *eth.Peer) { // The eth/65 protocol introduces proper transaction announcements, so instead // of dripping transactions across multiple peers, just send the entire list as // an announcement and let the remote side decide what they need (likely nothing). - hashes := make([]common.Hash, len(txs)) - for i, tx := range txs { - hashes[i] = tx.Hash() + if p.Version() >= eth.ETH65 { + hashes := make([]common.Hash, len(txs)) + for i, tx := range txs { + hashes[i] = tx.Hash() + } + p.AsyncSendPooledTransactionHashes(hashes) + return + } + // Out of luck, peer is running legacy protocols, drop the txs over + select { + case h.txsyncCh <- &txsync{p: p, txs: txs}: + case <-h.quitSync: + } +} + +// txsyncLoop64 takes care of the initial transaction sync for each new +// connection. When a new peer appears, we relay all currently pending +// transactions. In order to minimise egress bandwidth usage, we send +// the transactions in small packs to one peer at a time. +func (h *handler) txsyncLoop64() { + defer h.wg.Done() + + var ( + pending = make(map[enode.ID]*txsync) + sending = false // whether a send is active + pack = new(txsync) // the pack that is being sent + done = make(chan error, 1) // result of the send + ) + + // send starts a sending a pack of transactions from the sync. + send := func(s *txsync) { + if s.p.Version() >= eth.ETH65 { + panic("initial transaction syncer running on eth/65+") + } + // Fill pack with transactions up to the target size. + size := common.StorageSize(0) + pack.p = s.p + pack.txs = pack.txs[:0] + for i := 0; i < len(s.txs) && size < txsyncPackSize; i++ { + pack.txs = append(pack.txs, s.txs[i]) + size += s.txs[i].Size() + } + // Remove the transactions that will be sent. + s.txs = s.txs[:copy(s.txs, s.txs[len(pack.txs):])] + if len(s.txs) == 0 { + delete(pending, s.p.Peer.ID()) + } + // Send the pack in the background. + s.p.Log().Trace("Sending batch of transactions", "count", len(pack.txs), "bytes", size) + sending = true + go func() { done <- pack.p.SendTransactions(pack.txs) }() + } + // pick chooses the next pending sync. + pick := func() *txsync { + if len(pending) == 0 { + return nil + } + n := rand.Intn(len(pending)) + 1 + for _, s := range pending { + if n--; n == 0 { + return s + } + } + return nil + } + + for { + select { + case s := <-h.txsyncCh: + pending[s.p.Peer.ID()] = s + if !sending { + send(s) + } + case err := <-done: + sending = false + // Stop tracking peers that cause send failures. + if err != nil { + pack.p.Log().Debug("Transaction send failed", "err", err) + delete(pending, pack.p.Peer.ID()) + } + // Schedule the next send. + if s := pick(); s != nil { + send(s) + } + case <-h.quitSync: + return + } } - p.AsyncSendPooledTransactionHashes(hashes) } // chainSyncer coordinates blockchain sync components. diff --git a/eth/sync_test.go b/eth/sync_test.go index e96b9ee81..a0c6f8602 100644 --- a/eth/sync_test.go +++ b/eth/sync_test.go @@ -28,6 +28,7 @@ import ( ) // Tests that fast sync is disabled after a successful sync cycle. +func TestFastSyncDisabling65(t *testing.T) { testFastSyncDisabling(t, eth.ETH65) } func TestFastSyncDisabling66(t *testing.T) { testFastSyncDisabling(t, eth.ETH66) } // Tests that fast sync gets disabled as soon as a real block is successfully diff --git a/les/client_handler.go b/les/client_handler.go index b903b1106..e95996c51 100644 --- a/les/client_handler.go +++ b/les/client_handler.go @@ -472,7 +472,7 @@ func (d *downloaderPeerNotify) registerPeer(p *serverPeer) { handler: h, peer: p, } - h.downloader.RegisterLightPeer(p.id, eth.ETH66, pc) + h.downloader.RegisterLightPeer(p.id, eth.ETH65, pc) } func (d *downloaderPeerNotify) unregisterPeer(p *serverPeer) { From d02c60536799698888d21f093f7c379acdad3147 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Mon, 23 Aug 2021 11:49:39 +0200 Subject: [PATCH 12/17] core: only check sendernoeoa in non fake mode (#23424) --- accounts/abi/bind/backends/simulated.go | 2 +- core/state_transition.go | 17 +++++++++-------- core/types/transaction.go | 10 +++++----- internal/ethapi/transaction_args.go | 4 ++-- les/odr_test.go | 4 ++-- light/odr_test.go | 2 +- tests/state_test_util.go | 2 +- 7 files changed, 21 insertions(+), 20 deletions(-) diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index cb4eea3a0..e410522ac 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -795,7 +795,7 @@ type callMsg struct { func (m callMsg) From() common.Address { return m.CallMsg.From } func (m callMsg) Nonce() uint64 { return 0 } -func (m callMsg) CheckNonce() bool { return false } +func (m callMsg) IsFake() bool { return true } func (m callMsg) To() *common.Address { return m.CallMsg.To } func (m callMsg) GasPrice() *big.Int { return m.CallMsg.GasPrice } func (m callMsg) GasFeeCap() *big.Int { return m.CallMsg.GasFeeCap } diff --git a/core/state_transition.go b/core/state_transition.go index 4c4187df1..6a09f6adc 100644 --- a/core/state_transition.go +++ b/core/state_transition.go @@ -74,7 +74,7 @@ type Message interface { Value() *big.Int Nonce() uint64 - CheckNonce() bool + IsFake() bool Data() []byte AccessList() types.AccessList } @@ -212,8 +212,9 @@ func (st *StateTransition) buyGas() error { } func (st *StateTransition) preCheck() error { - // Make sure this transaction's nonce is correct. - if st.msg.CheckNonce() { + // Only check transactions that are not fake + if !st.msg.IsFake() { + // Make sure this transaction's nonce is correct. stNonce := st.state.GetNonce(st.msg.From()) if msgNonce := st.msg.Nonce(); stNonce < msgNonce { return fmt.Errorf("%w: address %v, tx: %d state: %d", ErrNonceTooHigh, @@ -222,11 +223,11 @@ func (st *StateTransition) preCheck() error { return fmt.Errorf("%w: address %v, tx: %d state: %d", ErrNonceTooLow, st.msg.From().Hex(), msgNonce, stNonce) } - } - // Make sure the sender is an EOA - if codeHash := st.state.GetCodeHash(st.msg.From()); codeHash != emptyCodeHash && codeHash != (common.Hash{}) { - return fmt.Errorf("%w: address %v, codehash: %s", ErrSenderNoEOA, - st.msg.From().Hex(), codeHash) + // Make sure the sender is an EOA + if codeHash := st.state.GetCodeHash(st.msg.From()); codeHash != emptyCodeHash && codeHash != (common.Hash{}) { + return fmt.Errorf("%w: address %v, codehash: %s", ErrSenderNoEOA, + st.msg.From().Hex(), codeHash) + } } // Make sure that transaction gasFeeCap is greater than the baseFee (post london) if st.evm.ChainConfig().IsLondon(st.evm.Context.BlockNumber) { diff --git a/core/types/transaction.go b/core/types/transaction.go index a556f4b57..e21cf2bda 100644 --- a/core/types/transaction.go +++ b/core/types/transaction.go @@ -579,10 +579,10 @@ type Message struct { gasTipCap *big.Int data []byte accessList AccessList - checkNonce bool + isFake bool } -func NewMessage(from common.Address, to *common.Address, nonce uint64, amount *big.Int, gasLimit uint64, gasPrice, gasFeeCap, gasTipCap *big.Int, data []byte, accessList AccessList, checkNonce bool) Message { +func NewMessage(from common.Address, to *common.Address, nonce uint64, amount *big.Int, gasLimit uint64, gasPrice, gasFeeCap, gasTipCap *big.Int, data []byte, accessList AccessList, isFake bool) Message { return Message{ from: from, to: to, @@ -594,7 +594,7 @@ func NewMessage(from common.Address, to *common.Address, nonce uint64, amount *b gasTipCap: gasTipCap, data: data, accessList: accessList, - checkNonce: checkNonce, + isFake: isFake, } } @@ -610,7 +610,7 @@ func (tx *Transaction) AsMessage(s Signer, baseFee *big.Int) (Message, error) { amount: tx.Value(), data: tx.Data(), accessList: tx.AccessList(), - checkNonce: true, + isFake: false, } // If baseFee provided, set gasPrice to effectiveGasPrice. if baseFee != nil { @@ -631,4 +631,4 @@ func (m Message) Gas() uint64 { return m.gasLimit } func (m Message) Nonce() uint64 { return m.nonce } func (m Message) Data() []byte { return m.data } func (m Message) AccessList() AccessList { return m.accessList } -func (m Message) CheckNonce() bool { return m.checkNonce } +func (m Message) IsFake() bool { return m.isFake } diff --git a/internal/ethapi/transaction_args.go b/internal/ethapi/transaction_args.go index 8c12743dc..52811b2a9 100644 --- a/internal/ethapi/transaction_args.go +++ b/internal/ethapi/transaction_args.go @@ -171,7 +171,7 @@ func (args *TransactionArgs) setDefaults(ctx context.Context, b Backend) error { return nil } -// ToMessage converts th transaction arguments to the Message type used by the +// ToMessage converts the transaction arguments to the Message type used by the // core evm. This method is used in calls and traces that do not require a real // live transaction. func (args *TransactionArgs) ToMessage(globalGasCap uint64, baseFee *big.Int) (types.Message, error) { @@ -238,7 +238,7 @@ func (args *TransactionArgs) ToMessage(globalGasCap uint64, baseFee *big.Int) (t if args.AccessList != nil { accessList = *args.AccessList } - msg := types.NewMessage(addr, args.To, 0, value, gas, gasPrice, gasFeeCap, gasTipCap, data, accessList, false) + msg := types.NewMessage(addr, args.To, 0, value, gas, gasPrice, gasFeeCap, gasTipCap, data, accessList, true) return msg, nil } diff --git a/les/odr_test.go b/les/odr_test.go index 91419ec80..ea88495d1 100644 --- a/les/odr_test.go +++ b/les/odr_test.go @@ -135,7 +135,7 @@ func odrContractCall(ctx context.Context, db ethdb.Database, config *params.Chai from := statedb.GetOrNewStateObject(bankAddr) from.SetBalance(math.MaxBig256) - msg := callmsg{types.NewMessage(from.Address(), &testContractAddr, 0, new(big.Int), 100000, big.NewInt(params.InitialBaseFee), big.NewInt(params.InitialBaseFee), new(big.Int), data, nil, false)} + msg := callmsg{types.NewMessage(from.Address(), &testContractAddr, 0, new(big.Int), 100000, big.NewInt(params.InitialBaseFee), big.NewInt(params.InitialBaseFee), new(big.Int), data, nil, true)} context := core.NewEVMBlockContext(header, bc, nil) txContext := core.NewEVMTxContext(msg) @@ -150,7 +150,7 @@ func odrContractCall(ctx context.Context, db ethdb.Database, config *params.Chai header := lc.GetHeaderByHash(bhash) state := light.NewState(ctx, header, lc.Odr()) state.SetBalance(bankAddr, math.MaxBig256) - msg := callmsg{types.NewMessage(bankAddr, &testContractAddr, 0, new(big.Int), 100000, big.NewInt(params.InitialBaseFee), big.NewInt(params.InitialBaseFee), new(big.Int), data, nil, false)} + msg := callmsg{types.NewMessage(bankAddr, &testContractAddr, 0, new(big.Int), 100000, big.NewInt(params.InitialBaseFee), big.NewInt(params.InitialBaseFee), new(big.Int), data, nil, true)} context := core.NewEVMBlockContext(header, lc, nil) txContext := core.NewEVMTxContext(msg) vmenv := vm.NewEVM(context, txContext, state, config, vm.Config{NoBaseFee: true}) diff --git a/light/odr_test.go b/light/odr_test.go index b31e54d28..fdf657a82 100644 --- a/light/odr_test.go +++ b/light/odr_test.go @@ -194,7 +194,7 @@ func odrContractCall(ctx context.Context, db ethdb.Database, bc *core.BlockChain // Perform read-only call. st.SetBalance(testBankAddress, math.MaxBig256) - msg := callmsg{types.NewMessage(testBankAddress, &testContractAddr, 0, new(big.Int), 1000000, big.NewInt(params.InitialBaseFee), big.NewInt(params.InitialBaseFee), new(big.Int), data, nil, false)} + msg := callmsg{types.NewMessage(testBankAddress, &testContractAddr, 0, new(big.Int), 1000000, big.NewInt(params.InitialBaseFee), big.NewInt(params.InitialBaseFee), new(big.Int), data, nil, true)} txContext := core.NewEVMTxContext(msg) context := core.NewEVMBlockContext(header, chain, nil) vmenv := vm.NewEVM(context, txContext, st, config, vm.Config{NoBaseFee: true}) diff --git a/tests/state_test_util.go b/tests/state_test_util.go index 8b8a6a4fd..f7fb08bfb 100644 --- a/tests/state_test_util.go +++ b/tests/state_test_util.go @@ -348,7 +348,7 @@ func (tx *stTransaction) toMessage(ps stPostState, baseFee *big.Int) (core.Messa } msg := types.NewMessage(from, to, tx.Nonce, value, gasLimit, gasPrice, - tx.MaxFeePerGas, tx.MaxPriorityFeePerGas, data, accessList, true) + tx.MaxFeePerGas, tx.MaxPriorityFeePerGas, data, accessList, false) return msg, nil } From bb1f7ebf203f40dae714a3b8445918cfcfc9a7db Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Mon, 23 Aug 2021 12:20:56 +0200 Subject: [PATCH 13/17] signer/core/apitypes: remove dependency on internal/ethapi (#23362) --- signer/core/apitypes/types.go | 69 +++++++++++++++++++++++++++-------- 1 file changed, 53 insertions(+), 16 deletions(-) diff --git a/signer/core/apitypes/types.go b/signer/core/apitypes/types.go index 591d253ab..625959219 100644 --- a/signer/core/apitypes/types.go +++ b/signer/core/apitypes/types.go @@ -19,12 +19,12 @@ package apitypes import ( "encoding/json" "fmt" + "math/big" "strings" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/internal/ethapi" ) type ValidationInfo struct { @@ -97,23 +97,60 @@ func (args SendTxArgs) String() string { return err.Error() } +// ToTransaction converts the arguments to a transaction. func (args *SendTxArgs) ToTransaction() *types.Transaction { - txArgs := ethapi.TransactionArgs{ - Gas: &args.Gas, - GasPrice: args.GasPrice, - MaxFeePerGas: args.MaxFeePerGas, - MaxPriorityFeePerGas: args.MaxPriorityFeePerGas, - Value: &args.Value, - Nonce: &args.Nonce, - Data: args.Data, - Input: args.Input, - AccessList: args.AccessList, - ChainID: args.ChainID, - } // Add the To-field, if specified + var to *common.Address if args.To != nil { - to := args.To.Address() - txArgs.To = &to + dstAddr := args.To.Address() + to = &dstAddr } - return txArgs.ToTransaction() + + var input []byte + if args.Input != nil { + input = *args.Input + } else if args.Data != nil { + input = *args.Data + } + + var data types.TxData + switch { + case args.MaxFeePerGas != nil: + al := types.AccessList{} + if args.AccessList != nil { + al = *args.AccessList + } + data = &types.DynamicFeeTx{ + To: to, + ChainID: (*big.Int)(args.ChainID), + Nonce: uint64(args.Nonce), + Gas: uint64(args.Gas), + GasFeeCap: (*big.Int)(args.MaxFeePerGas), + GasTipCap: (*big.Int)(args.MaxPriorityFeePerGas), + Value: (*big.Int)(&args.Value), + Data: input, + AccessList: al, + } + case args.AccessList != nil: + data = &types.AccessListTx{ + To: to, + ChainID: (*big.Int)(args.ChainID), + Nonce: uint64(args.Nonce), + Gas: uint64(args.Gas), + GasPrice: (*big.Int)(args.GasPrice), + Value: (*big.Int)(&args.Value), + Data: input, + AccessList: *args.AccessList, + } + default: + data = &types.LegacyTx{ + To: to, + Nonce: uint64(args.Nonce), + Gas: uint64(args.Gas), + GasPrice: (*big.Int)(args.GasPrice), + Value: (*big.Int)(&args.Value), + Data: input, + } + } + return types.NewTx(data) } From dfeb2f7e8001aef1005a8d5e1605bae1de0b4f12 Mon Sep 17 00:00:00 2001 From: ucwong Date: Mon, 23 Aug 2021 19:45:16 +0800 Subject: [PATCH 14/17] go.mod: upgrade golang.org/x/sys for go1.17 support (#23406) --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 7e8407f53..ee9ffa045 100644 --- a/go.mod +++ b/go.mod @@ -65,7 +65,7 @@ require ( golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d // indirect golang.org/x/sync v0.0.0-20210220032951-036812b2e83c - golang.org/x/sys v0.0.0-20210423082822-04245dca01da + golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912 golang.org/x/text v0.3.6 golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce diff --git a/go.sum b/go.sum index b4d0b725c..d6e9639d2 100644 --- a/go.sum +++ b/go.sum @@ -525,8 +525,9 @@ golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210316164454-77fc1eacc6aa/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210420205809-ac73e9fd8988/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210423082822-04245dca01da h1:b3NXsE2LusjYGGjL5bxEVZZORm/YEFFrWFjR8eFrw/c= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912 h1:uCLL3g5wH2xjxVREVuAbP9JM5PPKjRbXKRa6IBjkzmU= +golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= From f38abc55f135587ce3cf60c9d574ec2a4ebb8197 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Mon, 23 Aug 2021 23:50:24 +0200 Subject: [PATCH 15/17] eth/gasprice: feeHistory improvements (#23422) * eth/gasprice: cache feeHistory results * eth/gasprice: changed feeHistory block count limitation * eth/gasprice: do not use embedded struct in blockFees * eth/gasprice: fee processing logic cleanup * eth/gasprice: purge feeHistory cache at chain reorgs --- eth/ethconfig/config.go | 4 +- eth/gasprice/feehistory.go | 105 +++++++++++++++++--------------- eth/gasprice/feehistory_test.go | 22 +++---- eth/gasprice/gasprice.go | 20 ++++++ eth/gasprice/gasprice_test.go | 5 ++ 5 files changed, 95 insertions(+), 61 deletions(-) diff --git a/eth/ethconfig/config.go b/eth/ethconfig/config.go index 23ccf2484..89cdb7559 100644 --- a/eth/ethconfig/config.go +++ b/eth/ethconfig/config.go @@ -43,8 +43,8 @@ import ( var FullNodeGPO = gasprice.Config{ Blocks: 20, Percentile: 60, - MaxHeaderHistory: 0, - MaxBlockHistory: 0, + MaxHeaderHistory: 1024, + MaxBlockHistory: 1024, MaxPrice: gasprice.DefaultMaxPrice, IgnorePrice: gasprice.DefaultIgnorePrice, } diff --git a/eth/gasprice/feehistory.go b/eth/gasprice/feehistory.go index 2c4486c6f..970dfd446 100644 --- a/eth/gasprice/feehistory.go +++ b/eth/gasprice/feehistory.go @@ -18,8 +18,10 @@ package gasprice import ( "context" + "encoding/binary" "errors" "fmt" + "math" "math/big" "sort" "sync/atomic" @@ -37,10 +39,6 @@ var ( ) const ( - // maxFeeHistory is the maximum number of blocks that can be retrieved for a - // fee history request. - maxFeeHistory = 1024 - // maxBlockFetchers is the max number of goroutines to spin up to pull blocks // for the fee history calculation (mostly relevant for LES). maxBlockFetchers = 4 @@ -54,10 +52,15 @@ type blockFees struct { block *types.Block // only set if reward percentiles are requested receipts types.Receipts // filled by processBlock + results processedFees + err error +} + +// processedFees contains the results of a processed block and is also used for caching +type processedFees struct { reward []*big.Int baseFee, nextBaseFee *big.Int gasUsedRatio float64 - err error } // txGasAndReward is sorted in ascending order based on reward @@ -82,15 +85,15 @@ func (s sortGasAndReward) Less(i, j int) bool { // fills in the rest of the fields. func (oracle *Oracle) processBlock(bf *blockFees, percentiles []float64) { chainconfig := oracle.backend.ChainConfig() - if bf.baseFee = bf.header.BaseFee; bf.baseFee == nil { - bf.baseFee = new(big.Int) + if bf.results.baseFee = bf.header.BaseFee; bf.results.baseFee == nil { + bf.results.baseFee = new(big.Int) } if chainconfig.IsLondon(big.NewInt(int64(bf.blockNumber + 1))) { - bf.nextBaseFee = misc.CalcBaseFee(chainconfig, bf.header) + bf.results.nextBaseFee = misc.CalcBaseFee(chainconfig, bf.header) } else { - bf.nextBaseFee = new(big.Int) + bf.results.nextBaseFee = new(big.Int) } - bf.gasUsedRatio = float64(bf.header.GasUsed) / float64(bf.header.GasLimit) + bf.results.gasUsedRatio = float64(bf.header.GasUsed) / float64(bf.header.GasLimit) if len(percentiles) == 0 { // rewards were not requested, return null return @@ -100,11 +103,11 @@ func (oracle *Oracle) processBlock(bf *blockFees, percentiles []float64) { return } - bf.reward = make([]*big.Int, len(percentiles)) + bf.results.reward = make([]*big.Int, len(percentiles)) if len(bf.block.Transactions()) == 0 { // return an all zero row if there are no transactions to gather data from - for i := range bf.reward { - bf.reward[i] = new(big.Int) + for i := range bf.results.reward { + bf.results.reward[i] = new(big.Int) } return } @@ -125,7 +128,7 @@ func (oracle *Oracle) processBlock(bf *blockFees, percentiles []float64) { txIndex++ sumGasUsed += sorter[txIndex].gasUsed } - bf.reward[i] = sorter[txIndex].reward + bf.results.reward[i] = sorter[txIndex].reward } } @@ -134,7 +137,7 @@ func (oracle *Oracle) processBlock(bf *blockFees, percentiles []float64) { // also returned if requested and available. // Note: an error is only returned if retrieving the head header has failed. If there are no // retrievable blocks in the specified range then zero block count is returned with no error. -func (oracle *Oracle) resolveBlockRange(ctx context.Context, lastBlock rpc.BlockNumber, blocks, maxHistory int) (*types.Block, []*types.Receipt, uint64, int, error) { +func (oracle *Oracle) resolveBlockRange(ctx context.Context, lastBlock rpc.BlockNumber, blocks int) (*types.Block, []*types.Receipt, uint64, int, error) { var ( headBlock rpc.BlockNumber pendingBlock *types.Block @@ -167,17 +170,6 @@ func (oracle *Oracle) resolveBlockRange(ctx context.Context, lastBlock rpc.Block } else if pendingBlock == nil && lastBlock > headBlock { return nil, nil, 0, 0, fmt.Errorf("%w: requested %d, head %d", errRequestBeyondHead, lastBlock, headBlock) } - if maxHistory != 0 { - // limit retrieval to the given number of latest blocks - if tooOldCount := int64(headBlock) - int64(maxHistory) - int64(lastBlock) + int64(blocks); tooOldCount > 0 { - // tooOldCount is the number of requested blocks that are too old to be served - if int64(blocks) > tooOldCount { - blocks -= int(tooOldCount) - } else { - return nil, nil, 0, 0, nil - } - } - } // ensure not trying to retrieve before genesis if rpc.BlockNumber(blocks) > lastBlock+1 { blocks = int(lastBlock + 1) @@ -202,6 +194,10 @@ func (oracle *Oracle) FeeHistory(ctx context.Context, blocks int, unresolvedLast if blocks < 1 { return common.Big0, nil, nil, nil, nil // returning with no data and no error means there are no retrievable blocks } + maxFeeHistory := oracle.maxHeaderHistory + if len(rewardPercentiles) != 0 { + maxFeeHistory = oracle.maxBlockHistory + } if blocks > maxFeeHistory { log.Warn("Sanitizing fee history length", "requested", blocks, "truncated", maxFeeHistory) blocks = maxFeeHistory @@ -214,17 +210,12 @@ func (oracle *Oracle) FeeHistory(ctx context.Context, blocks int, unresolvedLast return common.Big0, nil, nil, nil, fmt.Errorf("%w: #%d:%f > #%d:%f", errInvalidPercentile, i-1, rewardPercentiles[i-1], i, p) } } - // Only process blocks if reward percentiles were requested - maxHistory := oracle.maxHeaderHistory - if len(rewardPercentiles) != 0 { - maxHistory = oracle.maxBlockHistory - } var ( pendingBlock *types.Block pendingReceipts []*types.Receipt err error ) - pendingBlock, pendingReceipts, lastBlock, blocks, err := oracle.resolveBlockRange(ctx, unresolvedLastBlock, blocks, maxHistory) + pendingBlock, pendingReceipts, lastBlock, blocks, err := oracle.resolveBlockRange(ctx, unresolvedLastBlock, blocks) if err != nil || blocks == 0 { return common.Big0, nil, nil, nil, err } @@ -234,6 +225,10 @@ func (oracle *Oracle) FeeHistory(ctx context.Context, blocks int, unresolvedLast next = oldestBlock results = make(chan *blockFees, blocks) ) + percentileKey := make([]byte, 8*len(rewardPercentiles)) + for i, p := range rewardPercentiles { + binary.LittleEndian.PutUint64(percentileKey[i*8:(i+1)*8], math.Float64bits(p)) + } for i := 0; i < maxBlockFetchers && i < blocks; i++ { go func() { for { @@ -246,24 +241,38 @@ func (oracle *Oracle) FeeHistory(ctx context.Context, blocks int, unresolvedLast fees := &blockFees{blockNumber: blockNumber} if pendingBlock != nil && blockNumber >= pendingBlock.NumberU64() { fees.block, fees.receipts = pendingBlock, pendingReceipts + fees.header = fees.block.Header() + oracle.processBlock(fees, rewardPercentiles) + results <- fees } else { - if len(rewardPercentiles) != 0 { - fees.block, fees.err = oracle.backend.BlockByNumber(ctx, rpc.BlockNumber(blockNumber)) - if fees.block != nil && fees.err == nil { - fees.receipts, fees.err = oracle.backend.GetReceipts(ctx, fees.block.Hash()) - } + cacheKey := struct { + number uint64 + percentiles string + }{blockNumber, string(percentileKey)} + + if p, ok := oracle.historyCache.Get(cacheKey); ok { + fees.results = p.(processedFees) + results <- fees } else { - fees.header, fees.err = oracle.backend.HeaderByNumber(ctx, rpc.BlockNumber(blockNumber)) + if len(rewardPercentiles) != 0 { + fees.block, fees.err = oracle.backend.BlockByNumber(ctx, rpc.BlockNumber(blockNumber)) + if fees.block != nil && fees.err == nil { + fees.receipts, fees.err = oracle.backend.GetReceipts(ctx, fees.block.Hash()) + fees.header = fees.block.Header() + } + } else { + fees.header, fees.err = oracle.backend.HeaderByNumber(ctx, rpc.BlockNumber(blockNumber)) + } + if fees.header != nil && fees.err == nil { + oracle.processBlock(fees, rewardPercentiles) + if fees.err == nil { + oracle.historyCache.Add(cacheKey, fees.results) + } + } + // send to results even if empty to guarantee that blocks items are sent in total + results <- fees } } - if fees.block != nil { - fees.header = fees.block.Header() - } - if fees.header != nil { - oracle.processBlock(fees, rewardPercentiles) - } - // send to results even if empty to guarantee that blocks items are sent in total - results <- fees } }() } @@ -279,8 +288,8 @@ func (oracle *Oracle) FeeHistory(ctx context.Context, blocks int, unresolvedLast return common.Big0, nil, nil, nil, fees.err } i := int(fees.blockNumber - oldestBlock) - if fees.header != nil { - reward[i], baseFee[i], baseFee[i+1], gasUsedRatio[i] = fees.reward, fees.baseFee, fees.nextBaseFee, fees.gasUsedRatio + if fees.results.baseFee != nil { + reward[i], baseFee[i], baseFee[i+1], gasUsedRatio[i] = fees.results.reward, fees.results.baseFee, fees.results.nextBaseFee, fees.results.gasUsedRatio } else { // getting no block and no error means we are requesting into the future (might happen because of a reorg) if i < firstMissing { diff --git a/eth/gasprice/feehistory_test.go b/eth/gasprice/feehistory_test.go index 16c74b7db..c259eb0ac 100644 --- a/eth/gasprice/feehistory_test.go +++ b/eth/gasprice/feehistory_test.go @@ -36,20 +36,20 @@ func TestFeeHistory(t *testing.T) { expCount int expErr error }{ - {false, 0, 0, 10, 30, nil, 21, 10, nil}, - {false, 0, 0, 10, 30, []float64{0, 10}, 21, 10, nil}, - {false, 0, 0, 10, 30, []float64{20, 10}, 0, 0, errInvalidPercentile}, - {false, 0, 0, 1000000000, 30, nil, 0, 31, nil}, - {false, 0, 0, 1000000000, rpc.LatestBlockNumber, nil, 0, 33, nil}, - {false, 0, 0, 10, 40, nil, 0, 0, errRequestBeyondHead}, - {true, 0, 0, 10, 40, nil, 0, 0, errRequestBeyondHead}, + {false, 1000, 1000, 10, 30, nil, 21, 10, nil}, + {false, 1000, 1000, 10, 30, []float64{0, 10}, 21, 10, nil}, + {false, 1000, 1000, 10, 30, []float64{20, 10}, 0, 0, errInvalidPercentile}, + {false, 1000, 1000, 1000000000, 30, nil, 0, 31, nil}, + {false, 1000, 1000, 1000000000, rpc.LatestBlockNumber, nil, 0, 33, nil}, + {false, 1000, 1000, 10, 40, nil, 0, 0, errRequestBeyondHead}, + {true, 1000, 1000, 10, 40, nil, 0, 0, errRequestBeyondHead}, {false, 20, 2, 100, rpc.LatestBlockNumber, nil, 13, 20, nil}, {false, 20, 2, 100, rpc.LatestBlockNumber, []float64{0, 10}, 31, 2, nil}, {false, 20, 2, 100, 32, []float64{0, 10}, 31, 2, nil}, - {false, 0, 0, 1, rpc.PendingBlockNumber, nil, 0, 0, nil}, - {false, 0, 0, 2, rpc.PendingBlockNumber, nil, 32, 1, nil}, - {true, 0, 0, 2, rpc.PendingBlockNumber, nil, 32, 2, nil}, - {true, 0, 0, 2, rpc.PendingBlockNumber, []float64{0, 10}, 32, 2, nil}, + {false, 1000, 1000, 1, rpc.PendingBlockNumber, nil, 0, 0, nil}, + {false, 1000, 1000, 2, rpc.PendingBlockNumber, nil, 32, 1, nil}, + {true, 1000, 1000, 2, rpc.PendingBlockNumber, nil, 32, 2, nil}, + {true, 1000, 1000, 2, rpc.PendingBlockNumber, []float64{0, 10}, 32, 2, nil}, } for i, c := range cases { config := Config{ diff --git a/eth/gasprice/gasprice.go b/eth/gasprice/gasprice.go index 407eeaa28..8feb5ef24 100644 --- a/eth/gasprice/gasprice.go +++ b/eth/gasprice/gasprice.go @@ -23,10 +23,13 @@ import ( "sync" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" + lru "github.com/hashicorp/golang-lru" ) const sampleNumber = 3 // Number of transactions sampled in a block @@ -53,6 +56,7 @@ type OracleBackend interface { GetReceipts(ctx context.Context, hash common.Hash) (types.Receipts, error) PendingBlockAndReceipts() (*types.Block, types.Receipts) ChainConfig() *params.ChainConfig + SubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription } // Oracle recommends gas prices based on the content of recent @@ -68,6 +72,7 @@ type Oracle struct { checkBlocks, percentile int maxHeaderHistory, maxBlockHistory int + historyCache *lru.Cache } // NewOracle returns a new gasprice oracle which can recommend suitable @@ -99,6 +104,20 @@ func NewOracle(backend OracleBackend, params Config) *Oracle { } else if ignorePrice.Int64() > 0 { log.Info("Gasprice oracle is ignoring threshold set", "threshold", ignorePrice) } + + cache, _ := lru.New(2048) + headEvent := make(chan core.ChainHeadEvent, 1) + backend.SubscribeChainHeadEvent(headEvent) + go func() { + var lastHead common.Hash + for ev := range headEvent { + if ev.Block.ParentHash() != lastHead { + cache.Purge() + } + lastHead = ev.Block.Hash() + } + }() + return &Oracle{ backend: backend, lastPrice: params.Default, @@ -108,6 +127,7 @@ func NewOracle(backend OracleBackend, params Config) *Oracle { percentile: percent, maxHeaderHistory: params.MaxHeaderHistory, maxBlockHistory: params.MaxBlockHistory, + historyCache: cache, } } diff --git a/eth/gasprice/gasprice_test.go b/eth/gasprice/gasprice_test.go index dea8fea95..feecfddec 100644 --- a/eth/gasprice/gasprice_test.go +++ b/eth/gasprice/gasprice_test.go @@ -29,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rpc" ) @@ -90,6 +91,10 @@ func (b *testBackend) ChainConfig() *params.ChainConfig { return b.chain.Config() } +func (b *testBackend) SubscribeChainHeadEvent(ch chan<- core.ChainHeadEvent) event.Subscription { + return nil +} + func newTestBackend(t *testing.T, londonBlock *big.Int, pending bool) *testBackend { var ( key, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") From 1d995731923ca899964371ddb213d40b7e773818 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 24 Aug 2021 09:00:10 +0200 Subject: [PATCH 16/17] core/vm: faster code analysis (#23381) * core/vm: more detailed benchmark for jumpdest analysis * core/vm: make jd analysis benchmark alloc free * core/vm: improve jumpdest analysis * core/vm: improve worst-case * core/vm: further improvements in analysis * core/vm: improve jumpdest analysis >PUSH15 * core/vm: make jd analysis ref by value * core/vm: fix misspell * core/vm: improve set8 and set16 a bit * core/vm: reduce amount of code * core/vm: optimize byte copying --- core/vm/analysis.go | 94 +++++++++++++++++++++++++++++++++------- core/vm/analysis_test.go | 24 +++++++++- core/vm/instructions.go | 4 ++ core/vm/interpreter.go | 2 +- 4 files changed, 105 insertions(+), 19 deletions(-) diff --git a/core/vm/analysis.go b/core/vm/analysis.go index 0ccf47b97..449cded2a 100644 --- a/core/vm/analysis.go +++ b/core/vm/analysis.go @@ -16,17 +16,49 @@ package vm +const ( + set2BitsMask = uint16(0b1100_0000_0000_0000) + set3BitsMask = uint16(0b1110_0000_0000_0000) + set4BitsMask = uint16(0b1111_0000_0000_0000) + set5BitsMask = uint16(0b1111_1000_0000_0000) + set6BitsMask = uint16(0b1111_1100_0000_0000) + set7BitsMask = uint16(0b1111_1110_0000_0000) +) + // bitvec is a bit vector which maps bytes in a program. // An unset bit means the byte is an opcode, a set bit means // it's data (i.e. argument of PUSHxx). type bitvec []byte -func (bits *bitvec) set(pos uint64) { - (*bits)[pos/8] |= 0x80 >> (pos % 8) +var lookup = [8]byte{ + 0x80, 0x40, 0x20, 0x10, 0x8, 0x4, 0x2, 0x1, } -func (bits *bitvec) set8(pos uint64) { - (*bits)[pos/8] |= 0xFF >> (pos % 8) - (*bits)[pos/8+1] |= ^(0xFF >> (pos % 8)) + +func (bits bitvec) set1(pos uint64) { + bits[pos/8] |= lookup[pos%8] +} + +func (bits bitvec) setN(flag uint16, pos uint64) { + a := flag >> (pos % 8) + bits[pos/8] |= byte(a >> 8) + if b := byte(a); b != 0 { + // If the bit-setting affects the neighbouring byte, we can assign - no need to OR it, + // since it's the first write to that byte + bits[pos/8+1] = b + } +} + +func (bits bitvec) set8(pos uint64) { + a := byte(0xFF >> (pos % 8)) + bits[pos/8] |= a + bits[pos/8+1] = ^a +} + +func (bits bitvec) set16(pos uint64) { + a := byte(0xFF >> (pos % 8)) + bits[pos/8] |= a + bits[pos/8+1] = 0xFF + bits[pos/8+2] = ^a } // codeSegment checks if the position is in a code segment. @@ -40,22 +72,52 @@ func codeBitmap(code []byte) bitvec { // ends with a PUSH32, the algorithm will push zeroes onto the // bitvector outside the bounds of the actual code. bits := make(bitvec, len(code)/8+1+4) + return codeBitmapInternal(code, bits) +} + +// codeBitmapInternal is the internal implementation of codeBitmap. +// It exists for the purpose of being able to run benchmark tests +// without dynamic allocations affecting the results. +func codeBitmapInternal(code, bits bitvec) bitvec { for pc := uint64(0); pc < uint64(len(code)); { op := OpCode(code[pc]) - - if op >= PUSH1 && op <= PUSH32 { - numbits := op - PUSH1 + 1 - pc++ + pc++ + if op < PUSH1 || op > PUSH32 { + continue + } + numbits := op - PUSH1 + 1 + if numbits >= 8 { + for ; numbits >= 16; numbits -= 16 { + bits.set16(pc) + pc += 16 + } for ; numbits >= 8; numbits -= 8 { - bits.set8(pc) // 8 + bits.set8(pc) pc += 8 } - for ; numbits > 0; numbits-- { - bits.set(pc) - pc++ - } - } else { - pc++ + } + switch numbits { + case 1: + bits.set1(pc) + pc += 1 + case 2: + bits.setN(set2BitsMask, pc) + pc += 2 + case 3: + bits.setN(set3BitsMask, pc) + pc += 3 + case 4: + bits.setN(set4BitsMask, pc) + pc += 4 + case 5: + bits.setN(set5BitsMask, pc) + pc += 5 + case 6: + bits.setN(set6BitsMask, pc) + pc += 6 + case 7: + bits.setN(set7BitsMask, pc) + pc += 7 } } return bits diff --git a/core/vm/analysis_test.go b/core/vm/analysis_test.go index fd2d744d8..585bb3097 100644 --- a/core/vm/analysis_test.go +++ b/core/vm/analysis_test.go @@ -47,10 +47,10 @@ func TestJumpDestAnalysis(t *testing.T) { {[]byte{byte(PUSH32)}, 0xFF, 1}, {[]byte{byte(PUSH32)}, 0xFF, 2}, } - for _, test := range tests { + for i, test := range tests { ret := codeBitmap(test.code) if ret[test.which] != test.exp { - t.Fatalf("expected %x, got %02x", test.exp, ret[test.which]) + t.Fatalf("test %d: expected %x, got %02x", i, test.exp, ret[test.which]) } } } @@ -73,3 +73,23 @@ func BenchmarkJumpdestHashing_1200k(bench *testing.B) { } bench.StopTimer() } + +func BenchmarkJumpdestOpAnalysis(bench *testing.B) { + var op OpCode + bencher := func(b *testing.B) { + code := make([]byte, 32*b.N) + for i := range code { + code[i] = byte(op) + } + bits := make(bitvec, len(code)/8+1+4) + b.ResetTimer() + codeBitmapInternal(code, bits) + } + for op = PUSH1; op <= PUSH32; op++ { + bench.Run(op.String(), bencher) + } + op = JUMPDEST + bench.Run(op.String(), bencher) + op = STOP + bench.Run(op.String(), bencher) +} diff --git a/core/vm/instructions.go b/core/vm/instructions.go index f7ef2f900..6c8c6e6e6 100644 --- a/core/vm/instructions.go +++ b/core/vm/instructions.go @@ -669,6 +669,7 @@ func opCall(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([]byt } stack.push(&temp) if err == nil || err == ErrExecutionReverted { + ret = common.CopyBytes(ret) scope.Memory.Set(retOffset.Uint64(), retSize.Uint64(), ret) } scope.Contract.Gas += returnGas @@ -703,6 +704,7 @@ func opCallCode(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) ([ } stack.push(&temp) if err == nil || err == ErrExecutionReverted { + ret = common.CopyBytes(ret) scope.Memory.Set(retOffset.Uint64(), retSize.Uint64(), ret) } scope.Contract.Gas += returnGas @@ -730,6 +732,7 @@ func opDelegateCall(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext } stack.push(&temp) if err == nil || err == ErrExecutionReverted { + ret = common.CopyBytes(ret) scope.Memory.Set(retOffset.Uint64(), retSize.Uint64(), ret) } scope.Contract.Gas += returnGas @@ -757,6 +760,7 @@ func opStaticCall(pc *uint64, interpreter *EVMInterpreter, scope *ScopeContext) } stack.push(&temp) if err == nil || err == ErrExecutionReverted { + ret = common.CopyBytes(ret) scope.Memory.Set(retOffset.Uint64(), retSize.Uint64(), ret) } scope.Contract.Gas += returnGas diff --git a/core/vm/interpreter.go b/core/vm/interpreter.go index 9cf0c4e2c..9fb83799c 100644 --- a/core/vm/interpreter.go +++ b/core/vm/interpreter.go @@ -262,7 +262,7 @@ func (in *EVMInterpreter) Run(contract *Contract, input []byte, readOnly bool) ( // if the operation clears the return data (e.g. it has returning data) // set the last return to the result of the operation. if operation.returns { - in.returnData = common.CopyBytes(res) + in.returnData = res } switch { From 26675454bf93bf904be7a43cce6b3f550115ff90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 24 Aug 2021 10:04:30 +0300 Subject: [PATCH 17/17] params: release Geth v1.10.8 --- params/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/params/version.go b/params/version.go index 3c50a9e8d..8d222ac04 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 10 // Minor version component of the current release - VersionPatch = 8 // Patch version component of the current release - VersionMeta = "unstable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 10 // Minor version component of the current release + VersionPatch = 8 // Patch version component of the current release + VersionMeta = "stable" // Version metadata to append to the version string ) // Version holds the textual version string.