update deps

adapt/state.go

@@ -17,7 +17,7 @@ type StateView interface {
 // StateTrie is an interface exposing only the necessary methods from state.Trie
 type StateTrie interface {
 	GetKey([]byte) []byte
-	NodeIterator([]byte) trie.NodeIterator
+	NodeIterator([]byte) (trie.NodeIterator, error)
 }
 
 // adapts a state.Database to StateView - used in tests
@@ -36,7 +36,7 @@ func (a stateDatabaseView) OpenTrie(root common.Hash) (StateTrie, error) {
 }
 
 func (a stateDatabaseView) ContractCode(hash common.Hash) ([]byte, error) {
-	return a.db.ContractCode(common.Hash{}, hash)
+	return a.db.ContractCode(common.Address{}, hash)
 }
 
 // adapts geth Trie to plugeth
@@ -46,8 +46,8 @@ type adaptTrie struct {
 
 func NewStateTrie(t plugeth.Trie) StateTrie { return adaptTrie{t} }
 
-func (a adaptTrie) NodeIterator(start []byte) trie.NodeIterator {
+func (a adaptTrie) NodeIterator(start []byte) (trie.NodeIterator, error) {
-	return NodeIterator(a.Trie.NodeIterator(start))
+	return NodeIterator(a.Trie.NodeIterator(start)), nil
 }
 
 func NodeIterator(it plugeth.NodeIterator) trie.NodeIterator {

builder.go

@@ -138,8 +138,14 @@ func (sdb *builder) WriteStateDiff(
 	if err != nil {
 		return fmt.Errorf("error opening new state trie: %w", err)
 	}
-	subitersA := iterutils.SubtrieIterators(triea.NodeIterator, uint(sdb.subtrieWorkers))
+	subitersA, err := iterutils.SubtrieIterators(triea.NodeIterator, uint(sdb.subtrieWorkers))
-	subitersB := iterutils.SubtrieIterators(trieb.NodeIterator, uint(sdb.subtrieWorkers))
+	if err != nil {
+		return fmt.Errorf("error creating subtrie iterators for old state trie: %w", err)
+	}
+	subitersB, err := iterutils.SubtrieIterators(trieb.NodeIterator, uint(sdb.subtrieWorkers))
+	if err != nil {
+		return fmt.Errorf("error creating subtrie iterators for new state trie: %w", err)
+	}
 
 	logger := log.New("hash", args.BlockHash, "number", args.BlockNumber)
 	// errgroup will cancel if any group fails
@@ -168,12 +174,11 @@ func (sdb *builder) WriteStateSnapshot(
 	tracker tracker.IteratorTracker,
 ) error {
 	defer metrics.UpdateDuration(time.Now(), metrics.IndexerMetrics.WriteStateDiffTimer)
-	// Load tries for old and new states
+
 	tree, err := sdb.stateCache.OpenTrie(stateRoot)
 	if err != nil {
 		return fmt.Errorf("error opening new state trie: %w", err)
 	}
-
 	subiters, _, err := tracker.Restore(tree.NodeIterator)
 	if err != nil {
 		return fmt.Errorf("error restoring iterators: %w", err)
@@ -186,7 +191,10 @@ func (sdb *builder) WriteStateSnapshot(
 				sdb.subtrieWorkers, len(subiters))
 		}
 	} else {
-		subiters = iterutils.SubtrieIterators(tree.NodeIterator, uint(sdb.subtrieWorkers))
+		subiters, err = iterutils.SubtrieIterators(tree.NodeIterator, uint(sdb.subtrieWorkers))
+		if err != nil {
+			return fmt.Errorf("error creating subtrie iterators for trie: %w", err)
+		}
 		for i := range subiters {
 			subiters[i] = tracker.Tracked(subiters[i])
 		}
@@ -414,12 +422,14 @@ func (sdb *builder) processStorageCreations(
 	log.Debug("Storage root for eventual diff", "root", sr)
 	sTrie, err := sdb.stateCache.OpenTrie(sr)
 	if err != nil {
-		log.Info("error in build storage diff eventual", "error", err)
+		return fmt.Errorf("error opening storage trie for root %s: %w", sr, err)
-		return err
 	}
 
 	var prevBlob []byte
-	it := sTrie.NodeIterator(make([]byte, 0))
+	it, err := sTrie.NodeIterator(nil)
+	if err != nil {
+		return fmt.Errorf("error creating iterator for storage trie with root %s: %w", sr, err)
+	}
 	for it.Next(true) {
 		if it.Leaf() {
 			storageLeafNode := sdb.decodeStorageLeaf(it, prevBlob)
@@ -452,7 +462,7 @@ func (sdb *builder) processStorageUpdates(
 	if newroot == oldroot {
 		return nil
 	}
-	log.Trace("Storage roots for incremental diff", "old", oldroot, "new", newroot)
+	log.Debug("Storage roots for incremental diff", "old", oldroot, "new", newroot)
 	oldTrie, err := sdb.stateCache.OpenTrie(oldroot)
 	if err != nil {
 		return err
@@ -463,7 +473,14 @@ func (sdb *builder) processStorageUpdates(
 	}
 
 	var prevBlob []byte
-	a, b := oldTrie.NodeIterator(nil), newTrie.NodeIterator(nil)
+	a, err := oldTrie.NodeIterator(nil)
+	if err != nil {
+		return err
+	}
+	b, err := newTrie.NodeIterator(nil)
+	if err != nil {
+		return err
+	}
 	it := utils.NewSymmetricDifferenceIterator(a, b)
 	for it.Next(true) {
 		if it.FromA() {
@@ -515,10 +532,12 @@ func (sdb *builder) processRemovedAccountStorage(
 	log.Debug("Storage root for removed diffs", "root", sr)
 	sTrie, err := sdb.stateCache.OpenTrie(sr)
 	if err != nil {
-		log.Info("error in build removed account storage diffs", "error", err)
+		return fmt.Errorf("error opening storage trie for root %s: %w", sr, err)
-		return err
+	}
+	it, err := sTrie.NodeIterator(nil)
+	if err != nil {
+		return fmt.Errorf("error creating iterator for storage trie with root %s: %w", sr, err)
 	}
-	it := sTrie.NodeIterator(nil)
 	for it.Next(true) {
 		if it.Leaf() { // only leaf values are indexed, don't need to demarcate removed intermediate nodes
 			leafKey := make([]byte, len(it.LeafKey()))

builder_test.go

@@ -25,6 +25,7 @@ import (
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/holiman/uint256"
 
 	statediff "github.com/cerc-io/plugeth-statediff"
 	"github.com/cerc-io/plugeth-statediff/indexer/ipld"
@@ -76,7 +77,7 @@ var (
 	})
 	contractAccountAtBlock2 = &types.StateAccount{
 		Nonce:    1,
-		Balance:  big.NewInt(0),
+		Balance:  uint256.NewInt(0),
 		CodeHash: common.HexToHash("0xaaea5efba4fd7b45d7ec03918ac5d8b31aa93b48986af0e6b591f0f087c80127").Bytes(),
 		Root:     crypto.Keccak256Hash(block2StorageBranchRootNode),
 	}
@@ -87,7 +88,7 @@ var (
 	})
 	contractAccountAtBlock3 = &types.StateAccount{
 		Nonce:    1,
-		Balance:  big.NewInt(0),
+		Balance:  uint256.NewInt(0),
 		CodeHash: common.HexToHash("0xaaea5efba4fd7b45d7ec03918ac5d8b31aa93b48986af0e6b591f0f087c80127").Bytes(),
 		Root:     crypto.Keccak256Hash(block3StorageBranchRootNode),
 	}
@@ -98,7 +99,7 @@ var (
 	})
 	contractAccountAtBlock4 = &types.StateAccount{
 		Nonce:    1,
-		Balance:  big.NewInt(0),
+		Balance:  uint256.NewInt(0),
 		CodeHash: common.HexToHash("0xaaea5efba4fd7b45d7ec03918ac5d8b31aa93b48986af0e6b591f0f087c80127").Bytes(),
 		Root:     crypto.Keccak256Hash(block4StorageBranchRootNode),
 	}
@@ -109,7 +110,7 @@ var (
 	})
 	contractAccountAtBlock5 = &types.StateAccount{
 		Nonce:    1,
-		Balance:  big.NewInt(0),
+		Balance:  uint256.NewInt(0),
 		CodeHash: common.HexToHash("0xaaea5efba4fd7b45d7ec03918ac5d8b31aa93b48986af0e6b591f0f087c80127").Bytes(),
 		Root:     crypto.Keccak256Hash(block5StorageBranchRootNode),
 	}
@@ -120,7 +121,7 @@ var (
 	})
 	minerAccountAtBlock1 = &types.StateAccount{
 		Nonce:    0,
-		Balance:  big.NewInt(2000002625000000000),
+		Balance:  uint256.NewInt(2000002625000000000),
 		CodeHash: test_helpers.NullCodeHash.Bytes(),
 		Root:     test_helpers.EmptyContractRoot,
 	}
@@ -131,7 +132,7 @@ var (
 	})
 	minerAccountAtBlock2 = &types.StateAccount{
 		Nonce:    0,
-		Balance:  big.NewInt(4000111203461610525),
+		Balance:  uint256.NewInt(4000111203461610525),
 		CodeHash: test_helpers.NullCodeHash.Bytes(),
 		Root:     test_helpers.EmptyContractRoot,
 	}
@@ -154,7 +155,7 @@ var (
 	})
 	account1AtBlock2 = &types.StateAccount{
 		Nonce:    2,
-		Balance:  big.NewInt(999555797000009000),
+		Balance:  uint256.NewInt(999555797000009000),
 		CodeHash: test_helpers.NullCodeHash.Bytes(),
 		Root:     test_helpers.EmptyContractRoot,
 	}
@@ -165,7 +166,7 @@ var (
 	})
 	account1AtBlock5 = &types.StateAccount{
 		Nonce:    2,
-		Balance:  big.NewInt(2999586469962854280),
+		Balance:  uint256.NewInt(2999586469962854280),
 		CodeHash: test_helpers.NullCodeHash.Bytes(),
 		Root:     test_helpers.EmptyContractRoot,
 	}
@@ -176,7 +177,7 @@ var (
 	})
 	account1AtBlock6 = &types.StateAccount{
 		Nonce:    3,
-		Balance:  big.NewInt(2999557977962854280),
+		Balance:  uint256.NewInt(2999557977962854280),
 		CodeHash: test_helpers.NullCodeHash.Bytes(),
 		Root:     test_helpers.EmptyContractRoot,
 	}
@@ -187,7 +188,7 @@ var (
 	})
 	account2AtBlock2 = &types.StateAccount{
 		Nonce:    0,
-		Balance:  big.NewInt(1000),
+		Balance:  uint256.NewInt(1000),
 		CodeHash: test_helpers.NullCodeHash.Bytes(),
 		Root:     test_helpers.EmptyContractRoot,
 	}
@@ -198,7 +199,7 @@ var (
 	})
 	account2AtBlock3 = &types.StateAccount{
 		Nonce:    0,
-		Balance:  big.NewInt(2000013574009435976),
+		Balance:  uint256.NewInt(2000013574009435976),
 		CodeHash: test_helpers.NullCodeHash.Bytes(),
 		Root:     test_helpers.EmptyContractRoot,
 	}
@@ -209,7 +210,7 @@ var (
 	})
 	account2AtBlock4 = &types.StateAccount{
 		Nonce:    0,
-		Balance:  big.NewInt(4000048088163070348),
+		Balance:  uint256.NewInt(4000048088163070348),
 		CodeHash: test_helpers.NullCodeHash.Bytes(),
 		Root:     test_helpers.EmptyContractRoot,
 	}
@@ -220,7 +221,7 @@ var (
 	})
 	account2AtBlock6 = &types.StateAccount{
 		Nonce:    0,
-		Balance:  big.NewInt(6000063258066544204),
+		Balance:  uint256.NewInt(6000063258066544204),
 		CodeHash: test_helpers.NullCodeHash.Bytes(),
 		Root:     test_helpers.EmptyContractRoot,
 	}
@@ -231,7 +232,7 @@ var (
 	})
 	bankAccountAtBlock0 = &types.StateAccount{
 		Nonce:    0,
-		Balance:  big.NewInt(test_helpers.TestBankFunds.Int64()),
+		Balance:  uint256.NewInt(test_helpers.TestBankFunds.Int64()),
 		CodeHash: test_helpers.NullCodeHash.Bytes(),
 		Root:     test_helpers.EmptyContractRoot,
 	}
@@ -241,7 +242,7 @@ var (
 		bankAccountAtBlock0RLP,
 	})
 
-	block1BankBalance   = big.NewInt(test_helpers.TestBankFunds.Int64() - test_helpers.BalanceChange10000 - test_helpers.GasFees)
+	block1BankBalance   = uint256.NewInt(test_helpers.TestBankFunds.Int64() - test_helpers.BalanceChange10000 - test_helpers.GasFees)
 	bankAccountAtBlock1 = &types.StateAccount{
 		Nonce:    1,
 		Balance:  block1BankBalance,
@@ -257,7 +258,7 @@ var (
 	block2BankBalance   = block1BankBalance.Int64() - test_helpers.BalanceChange1Ether - test_helpers.GasFees
 	bankAccountAtBlock2 = &types.StateAccount{
 		Nonce:    2,
-		Balance:  big.NewInt(block2BankBalance),
+		Balance:  uint256.NewInt(block2BankBalance),
 		CodeHash: test_helpers.NullCodeHash.Bytes(),
 		Root:     test_helpers.EmptyContractRoot,
 	}
@@ -268,7 +269,7 @@ var (
 	})
 	bankAccountAtBlock3 = &types.StateAccount{
 		Nonce:    3,
-		Balance:  big.NewInt(999914255999990000),
+		Balance:  uint256.NewInt(999914255999990000),
 		CodeHash: test_helpers.NullCodeHash.Bytes(),
 		Root:     test_helpers.EmptyContractRoot,
 	}
@@ -279,7 +280,7 @@ var (
 	})
 	bankAccountAtBlock4 = &types.StateAccount{
 		Nonce:    6,
-		Balance:  big.NewInt(999826859999990000),
+		Balance:  uint256.NewInt(999826859999990000),
 		CodeHash: test_helpers.NullCodeHash.Bytes(),
 		Root:     test_helpers.EmptyContractRoot,
 	}
@@ -290,7 +291,7 @@ var (
 	})
 	bankAccountAtBlock5 = &types.StateAccount{
 		Nonce:    8,
-		Balance:  big.NewInt(999761283999990000),
+		Balance:  uint256.NewInt(999761283999990000),
 		CodeHash: test_helpers.NullCodeHash.Bytes(),
 		Root:     test_helpers.EmptyContractRoot,
 	}
@@ -1617,7 +1618,7 @@ var (
 
 	contractAccountAtBlock01 = &types.StateAccount{
 		Nonce:    1,
-		Balance:  big.NewInt(0),
+		Balance:  uint256.NewInt(0),
 		CodeHash: common.HexToHash("0xaaea5efba4fd7b45d7ec03918ac5d8b31aa93b48986af0e6b591f0f087c80127").Bytes(),
 		Root:     crypto.Keccak256Hash(block01StorageBranchRootNode),
 	}
@@ -1629,7 +1630,7 @@ var (
 
 	bankAccountAtBlock01 = &types.StateAccount{
 		Nonce:    1,
-		Balance:  big.NewInt(3999629697375000000),
+		Balance:  uint256.NewInt(3999629697375000000),
 		CodeHash: test_helpers.NullCodeHash.Bytes(),
 		Root:     test_helpers.EmptyContractRoot,
 	}
@@ -1640,7 +1641,7 @@ var (
 	})
 	bankAccountAtBlock02 = &types.StateAccount{
 		Nonce:    2,
-		Balance:  big.NewInt(5999607323457344852),
+		Balance:  uint256.NewInt(5999607323457344852),
 		CodeHash: test_helpers.NullCodeHash.Bytes(),
 		Root:     test_helpers.EmptyContractRoot,
 	}
@@ -1866,8 +1867,8 @@ contract test {
 */
 
 var (
-	b                    = big.NewInt(0).Sub(test_helpers.TestBIGBankFunds, test_helpers.BalanceChangeBIG)
+	b                    = uint256.NewInt(0).Sub(test_helpers.TestBIGBankFunds, test_helpers.BalanceChangeBIG)
-	block1BankBigBalance = big.NewInt(0).Sub(b, big.NewInt(test_helpers.GasFees2))
+	block1BankBigBalance = uint256.NewInt(0).Sub(b, uint256.NewInt(test_helpers.GasFees2))
 	bankAccountAtBlock1b = &types.StateAccount{
 		Nonce:    1,
 		Balance:  block1BankBigBalance,
@@ -1892,7 +1893,7 @@ var (
 		account1AtBlock1bRLP,
 	})
 
-	account1AtBlock2bBalance, _ = big.NewInt(0).SetString("1999999999999999999999999761539571000000000", 10)
+	account1AtBlock2bBalance, _ = uint256.NewInt(0).SetString("1999999999999999999999999761539571000000000", 10)
 	account1AtBlock2b           = &types.StateAccount{
 		Nonce:    1,
 		Balance:  account1AtBlock2bBalance,
@@ -1907,7 +1908,7 @@ var (
 
 	minerAccountAtBlock2b = &types.StateAccount{
 		Nonce:    0,
-		Balance:  big.NewInt(4055891787808414571),
+		Balance:  uint256.NewInt(4055891787808414571),
 		CodeHash: test_helpers.NullCodeHash.Bytes(),
 		Root:     test_helpers.EmptyContractRoot,
 	}
@@ -1919,7 +1920,7 @@ var (
 
 	contractAccountAtBlock2b = &types.StateAccount{
 		Nonce:    1,
-		Balance:  big.NewInt(0),
+		Balance:  uint256.NewInt(0),
 		CodeHash: test_helpers.CodeHashForInternalizedLeafNode.Bytes(),
 		Root:     crypto.Keccak256Hash(block2StorageBranchRootNode),
 	}
@@ -1929,7 +1930,7 @@ var (
 		contractAccountAtBlock2bRLP,
 	})
 
-	bankAccountAtBlock3bBalance, _ = big.NewInt(0).SetString("18000000000000000000000001999920365757724976", 10)
+	bankAccountAtBlock3bBalance, _ = uint256.NewInt(0).SetString("18000000000000000000000001999920365757724976", 10)
 	bankAccountAtBlock3b           = &types.StateAccount{
 		Nonce:    3,
 		Balance:  bankAccountAtBlock3bBalance,
@@ -1944,7 +1945,7 @@ var (
 
 	contractAccountAtBlock3b = &types.StateAccount{
 		Nonce:    1,
-		Balance:  big.NewInt(0),
+		Balance:  uint256.NewInt(0),
 		CodeHash: test_helpers.CodeHashForInternalizedLeafNode.Bytes(),
 		Root:     crypto.Keccak256Hash(block3bStorageBranchRootNode),
 	}
@@ -1954,8 +1955,8 @@ var (
 		contractAccountAtBlock3bRLP,
 	})
 
-	slot40364  = common.BytesToHash(big.NewInt(40364).Bytes())
+	slot40364  = common.BytesToHash(uint256.NewInt(40364).Bytes())
-	slot105566 = common.BytesToHash(big.NewInt(105566).Bytes())
+	slot105566 = common.BytesToHash(uint256.NewInt(105566).Bytes())
 
 	slot40364StorageValue  = utils.Hex2Bytes("01")
 	slot105566StorageValue = utils.Hex2Bytes("02")

go.mod

@@ -4,10 +4,11 @@ go 1.19
 
 require (
 	github.com/cerc-io/eth-iterator-utils v0.1.2
-	github.com/cerc-io/eth-testing v0.3.1
+	github.com/cerc-io/eth-testing v0.4.0
 	github.com/ethereum/go-ethereum v1.13.14
 	github.com/georgysavva/scany v0.2.9
 	github.com/golang/mock v1.6.0
+	github.com/holiman/uint256 v1.2.4
 	github.com/inconshreveable/log15 v2.16.0+incompatible
 	github.com/ipfs/go-cid v0.2.0
 	github.com/jackc/pgconn v1.10.0
@@ -67,7 +68,6 @@ require (
 	github.com/hashicorp/go-bexpr v0.1.10 // indirect
 	github.com/holiman/billy v0.0.0-20240216141850-2abb0c79d3c4 // indirect
 	github.com/holiman/bloomfilter/v2 v2.0.3 // indirect
-	github.com/holiman/uint256 v1.2.4 // indirect
 	github.com/huin/goupnp v1.3.0 // indirect
 	github.com/influxdata/influxdb-client-go/v2 v2.4.0 // indirect
 	github.com/influxdata/influxdb1-client v0.0.0-20220302092344-a9ab5670611c // indirect
@@ -126,7 +126,7 @@ require (
 	golang.org/x/exp v0.0.0-20231110203233-9a3e6036ecaa // indirect
 	golang.org/x/mod v0.14.0 // indirect
 	golang.org/x/net v0.18.0 // indirect
-	golang.org/x/sys v0.15.0 // indirect
+	golang.org/x/sys v0.16.0 // indirect
 	golang.org/x/term v0.15.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
@@ -140,6 +140,8 @@ require (
 )
 
 replace (
+	github.com/cerc-io/eth-iterator-utils => github.com/cerc-io/eth-iterator-utils v0.1.3-0.20240315081922-2062bac80862
+	github.com/cerc-io/eth-testing v0.4.0 => github.com/cerc-io/eth-testing v0.3.2-0.20240315081427-e027e285e4f5
 	github.com/ethereum/go-ethereum => git.vdb.to/cerc-io/plugeth v0.0.0-20240314141656-b705596b358f
 	github.com/openrelayxyz/plugeth-utils => git.vdb.to/cerc-io/plugeth-utils v0.0.0-20240314123353-4b000f96160f
 )

go.sum

@@ -1,10 +1,4 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
-git.vdb.to/cerc-io/eth-iterator-utils v0.1.2 h1:+3+T+J21J/VkhlCFujl8HT4XuwebavIuKj0+qfE+0QM=
-git.vdb.to/cerc-io/eth-iterator-utils v0.1.2/go.mod h1:OvXbdWbZ5viBXC/Ui1EkhsSmGB+AUX+TjGa3UDAfjfg=
-git.vdb.to/cerc-io/eth-testing v0.3.1 h1:sPnlMev6oEgTjsW7GtUkSsjKNG/+X6P9q0izSejLGpM=
-git.vdb.to/cerc-io/eth-testing v0.3.1/go.mod h1:qdvpc/W1xvf2MKx3rMOqvFvYaYIHG77Z1g0lwsmw0Uk=
-git.vdb.to/cerc-io/plugeth v0.0.0-20240314141041-e1ce091e8717 h1:drXdg+140HfRgMco/9RlXe81MzvTSzEstubc9OSpUFQ=
-git.vdb.to/cerc-io/plugeth v0.0.0-20240314141041-e1ce091e8717/go.mod h1:fnpYT8O9v7XtJslNITyO49tbVUveK98xg01Y5r0zNe8=
 git.vdb.to/cerc-io/plugeth v0.0.0-20240314141656-b705596b358f h1:IHnKyYgGiVtzqj3QOoTnvtu13KxpVphDZKXat5mZY4g=
 git.vdb.to/cerc-io/plugeth v0.0.0-20240314141656-b705596b358f/go.mod h1:ms+ND6gyyNTAibYM8uGQpAvhq5vHCHVDd8N7BlgxvWg=
 git.vdb.to/cerc-io/plugeth-utils v0.0.0-20240314123353-4b000f96160f h1:zY00tJD2TGQdKv/BphF97UqExUg4W8m5ezbuTrUyPEQ=
@@ -39,6 +33,10 @@ github.com/btcsuite/btcd/btcec/v2 v2.3.2 h1:5n0X6hX0Zk+6omWcihdYvdAlGf2DfasC0GMf
 github.com/btcsuite/btcd/btcec/v2 v2.3.2/go.mod h1:zYzJ8etWJQIv1Ogk7OzpWjowwOdXY1W/17j2MW85J04=
 github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cerc-io/eth-iterator-utils v0.1.3-0.20240315081922-2062bac80862 h1:XLHOU+16LmRVTDXf6RmiTtHKbh5urQFywh1aJ/CqJwQ=
+github.com/cerc-io/eth-iterator-utils v0.1.3-0.20240315081922-2062bac80862/go.mod h1:O4QE+mgtYjCtzqcTXo/dC7HURb++PZTdrq6MtPdCtoE=
+github.com/cerc-io/eth-testing v0.3.2-0.20240315081427-e027e285e4f5 h1:hS6ASI3zxdoSTSzdLaPVbn0zOTDcHuA69BwYnjzCAKI=
+github.com/cerc-io/eth-testing v0.3.2-0.20240315081427-e027e285e4f5/go.mod h1:CVsmHjFldX9gwaQSQwGmKbmh0g6Dq+bsqB2CxBf9zbk=
 github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk=
 github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
 github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
@@ -453,8 +451,6 @@ github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
 github.com/openrelayxyz/cardinal-types v1.1.1 h1:Lw6Lr/eiHYCnLi851rciCzw/1S3UytUX7kj5zh3QS/Y=
 github.com/openrelayxyz/cardinal-types v1.1.1/go.mod h1:8aaMg6i94V0hhWe3V6Fzc0RSggMx+/Kabsf5o7wMf/E=
-github.com/openrelayxyz/plugeth-utils v1.5.0 h1:4hzAvMKEo5uCnXy5cCHDc7OfXpwAavFNjr3M/ZfTlEA=
-github.com/openrelayxyz/plugeth-utils v1.5.0/go.mod h1:COwKAuTZIsCouCOrIDBhvHZqpbOO1Ojgdy5KTvL8mJg=
 github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU=
 github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
@@ -709,8 +705,8 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
+golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU=
-golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=

indexer/database/dump/indexer.go

@@ -68,12 +68,14 @@ func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receip
 	blockHashStr := blockHash.String()
 	height := block.NumberU64()
 	traceMsg := fmt.Sprintf("indexer stats for statediff at %d with hash %s:\r\n", height, blockHashStr)
-	transactions := block.Transactions()
+
 	var blobGasPrice *big.Int
 	excessBlobGas := block.ExcessBlobGas()
 	if excessBlobGas != nil {
 		blobGasPrice = eip4844.CalcBlobFee(*excessBlobGas)
 	}
+	transactions := block.Transactions()
+
 	// Derive any missing fields
 	if err := receipts.DeriveFields(sdi.chainConfig, blockHash, height, block.Time(), block.BaseFee(), blobGasPrice, transactions); err != nil {
 		return nil, err

indexer/database/file/indexer.go

@@ -27,6 +27,7 @@ import (
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/consensus/misc/eip4844"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/params"
@@ -141,9 +142,16 @@ func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receip
 	blockHashStr := blockHash.String()
 	height := block.NumberU64()
 	traceMsg := fmt.Sprintf("indexer stats for statediff at %d with hash %s:\r\n", height, blockHashStr)
+
+	var blobGasPrice *big.Int
+	excessBlobGas := block.ExcessBlobGas()
+	if excessBlobGas != nil {
+		blobGasPrice = eip4844.CalcBlobFee(*excessBlobGas)
+	}
 	transactions := block.Transactions()
+
 	// Derive any missing fields
-	if err := receipts.DeriveFields(sdi.chainConfig, blockHash, height, block.BaseFee(), transactions); err != nil {
+	if err := receipts.DeriveFields(sdi.chainConfig, blockHash, height, block.Time(), block.BaseFee(), blobGasPrice, transactions); err != nil {
 		return nil, err
 	}
 
@@ -282,6 +290,7 @@ func (sdi *StateDiffIndexer) processUncles(headerID string, blockNumber *big.Int
 type processArgs struct {
 	headerID    string
 	blockNumber *big.Int
+	blockTime   uint64
 	receipts    types.Receipts
 	txs         types.Transactions
 	rctNodes    []*ipld.EthReceipt
@@ -292,7 +301,7 @@ type processArgs struct {
 // processReceiptsAndTxs writes receipt and tx IPLD insert SQL stmts to a file
 func (sdi *StateDiffIndexer) processReceiptsAndTxs(args processArgs) error {
 	// Process receipts and txs
-	signer := types.MakeSigner(sdi.chainConfig, args.blockNumber)
+	signer := types.MakeSigner(sdi.chainConfig, args.blockNumber, args.blockTime)
 	for i, receipt := range args.receipts {
 		txNode := args.txNodes[i]
 		sdi.fileWriter.upsertIPLDNode(args.blockNumber.String(), txNode)

indexer/database/sql/indexer.go

@@ -26,6 +26,7 @@ import (
 	"time"
 
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/consensus/misc/eip4844"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/metrics"
@@ -90,11 +91,16 @@ func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receip
 	t := time.Now()
 	blockHash := block.Hash()
 	height := block.NumberU64()
+
+	var blobGasPrice *big.Int
+	excessBlobGas := block.ExcessBlobGas()
+	if excessBlobGas != nil {
+		blobGasPrice = eip4844.CalcBlobFee(*excessBlobGas)
+	}
 	transactions := block.Transactions()
-	var err error
 
 	// Derive any missing fields
-	if err := receipts.DeriveFields(sdi.chainConfig, blockHash, height, block.BaseFee(), transactions); err != nil {
+	if err := receipts.DeriveFields(sdi.chainConfig, blockHash, height, block.Time(), block.BaseFee(), blobGasPrice, transactions); err != nil {
 		return nil, err
 	}
 
@@ -146,6 +152,7 @@ func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receip
 	err = sdi.processReceiptsAndTxs(batch, processArgs{
 		headerID:    headerID,
 		blockNumber: block.Number(),
+		blockTime:   block.Time(),
 		receipts:    receipts,
 		txs:         transactions,
 		rctNodes:    rctNodes,
@@ -248,6 +255,7 @@ func (sdi *StateDiffIndexer) processUncles(tx *BatchTx, headerID string, blockNu
 type processArgs struct {
 	headerID    string
 	blockNumber *big.Int
+	blockTime   uint64
 	receipts    types.Receipts
 	txs         types.Transactions
 	rctNodes    []*ipld.EthReceipt
@@ -258,7 +266,7 @@ type processArgs struct {
 // processReceiptsAndTxs publishes and indexes receipt and transaction IPLDs in Postgres
 func (sdi *StateDiffIndexer) processReceiptsAndTxs(tx *BatchTx, args processArgs) error {
 	// Process receipts and txs
-	signer := types.MakeSigner(sdi.chainConfig, args.blockNumber)
+	signer := types.MakeSigner(sdi.chainConfig, args.blockNumber, args.blockTime)
 	for i, receipt := range args.receipts {
 		txNode := args.txNodes[i]
 		tx.cacheIPLD(txNode)

metrics_helpers.go

@@ -28,10 +28,10 @@ func countStateDiffBegin(block *types.Block, logger log.Logger) time.Time {
 
 	defaultStatediffMetrics.underway.Inc(1)
 	logger.Debug("writeStateDiff BEGIN",
-		"underway", defaultStatediffMetrics.underway.Count(),
+		"underway", defaultStatediffMetrics.underway.Snapshot().Count(),
-		"succeeded", defaultStatediffMetrics.succeeded.Count(),
+		"succeeded", defaultStatediffMetrics.succeeded.Snapshot().Count(),
-		"failed", defaultStatediffMetrics.failed.Count(),
+		"failed", defaultStatediffMetrics.failed.Snapshot().Count(),
-		"total_time", defaultStatediffMetrics.totalProcessingTime.Value(),
+		"total_time", defaultStatediffMetrics.totalProcessingTime.Snapshot().Value(),
 	)
 
 	return start
@@ -51,10 +51,10 @@ func countStateDiffEnd(start time.Time, logger log.Logger, err *error) time.Dura
 	logger.Debug("writeStateDiff END",
 		"duration", duration,
 		"error", failed,
-		"underway", defaultStatediffMetrics.underway.Count(),
+		"underway", defaultStatediffMetrics.underway.Snapshot().Count(),
-		"succeeded", defaultStatediffMetrics.succeeded.Count(),
+		"succeeded", defaultStatediffMetrics.succeeded.Snapshot().Count(),
-		"failed", defaultStatediffMetrics.failed.Count(),
+		"failed", defaultStatediffMetrics.failed.Snapshot().Count(),
-		"total_time", defaultStatediffMetrics.totalProcessingTime.Value(),
+		"total_time", defaultStatediffMetrics.totalProcessingTime.Snapshot().Value(),
 	)
 
 	return duration
@@ -68,8 +68,8 @@ func countApiRequestBegin(methodName string, blockHashOrNumber interface{}) (tim
 	defaultStatediffMetrics.apiRequestsUnderway.Inc(1)
 
 	logger.Debug("statediff API BEGIN",
-		"underway", defaultStatediffMetrics.apiRequestsUnderway.Count(),
+		"underway", defaultStatediffMetrics.apiRequestsUnderway.Snapshot().Count(),
-		"requests", defaultStatediffMetrics.apiRequests.Count(),
+		"requests", defaultStatediffMetrics.apiRequests.Snapshot().Count(),
 	)
 
 	return start, logger
@@ -82,8 +82,8 @@ func countApiRequestEnd(start time.Time, logger log.Logger, err error) time.Dura
 	logger.Debug("statediff API END",
 		"duration", duration,
 		"error", err != nil,
-		"underway", defaultStatediffMetrics.apiRequestsUnderway.Count(),
+		"underway", defaultStatediffMetrics.apiRequestsUnderway.Snapshot().Count(),
-		"requests", defaultStatediffMetrics.apiRequests.Count(),
+		"requests", defaultStatediffMetrics.apiRequests.Snapshot().Count(),
 	)
 
 	return duration

service.go

@@ -245,7 +245,7 @@ func (sds *Service) WriteLoop(chainEventCh chan core.ChainEvent) {
 			select {
 			case event := <-chainEventCh:
 				// First process metrics for chain events, then forward to workers
-				lastHeight := uint64(defaultStatediffMetrics.lastEventHeight.Value())
+				lastHeight := uint64(defaultStatediffMetrics.lastEventHeight.Snapshot().Value())
 				if lastHeight == 0 {
 					lastHeight = initialPos.indexerBlockNumber
 				}

test_helpers/constant.go

@@ -17,17 +17,16 @@
 package test_helpers
 
 import (
-	"math/big"
-
 	"github.com/ethereum/go-ethereum/params"
+	"github.com/holiman/uint256"
 )
 
 var (
 	BalanceChange1000                   = int64(1000)
 	BalanceChange10000                  = int64(10000)
-	BalanceChangeBIG, _                 = big.NewInt(0).SetString("2000000000000000000000000000000000000000000", 10)
+	BalanceChangeBIG                    = uint256.MustFromDecimal("2000000000000000000000000000000000000000000")
 	BalanceChange1Ether                 = int64(params.Ether)
-	Block1Account1Balance               = big.NewInt(BalanceChange10000)
+	Block1Account1Balance               = uint256.NewInt(uint64(BalanceChange10000))
 	Block1bAccount1Balance              = BalanceChangeBIG
 	GasFees                             = int64(params.GWei) * int64(params.TxGas)
 	GasFees2                            = int64(params.TxGas) * int64(params.InitialBaseFee)

test_helpers/helpers.go

@@ -27,13 +27,14 @@ import (
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/triedb"
 
 	"github.com/cerc-io/plugeth-statediff/utils"
 )
 
 func GenesisBlockForTesting(db ethdb.Database, addr common.Address, balance, baseFee *big.Int, initialGasLimit uint64) *types.Block {
-	alloc := map[common.Address]core.GenesisAccount{
+	alloc := map[common.Address]types.Account{
-		addr: core.GenesisAccount{Balance: balance}}
+		addr: types.Account{Balance: balance}}
 	g := core.Genesis{
 		Alloc:   alloc,
 		BaseFee: baseFee,
@@ -41,7 +42,7 @@ func GenesisBlockForTesting(db ethdb.Database, addr common.Address, balance, bas
 	if initialGasLimit != 0 {
 		g.GasLimit = initialGasLimit
 	}
-	return g.MustCommit(db)
+	return g.MustCommit(db, triedb.NewDatabase(db, nil))
 }
 
 // MakeChain creates a chain of n blocks starting at and including parent.
@@ -148,7 +149,7 @@ func TestChainGenWithInternalLeafNode(i int, block *core.BlockGen) {
 	switch i {
 	case 0:
 		// In block 1, the test bank sends account #1 some ether.
-		tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), Account1Addr, BalanceChangeBIG, params.TxGas, big.NewInt(params.InitialBaseFee), nil), signer, TestBankKey)
+		tx, _ := types.SignTx(types.NewTransaction(block.TxNonce(TestBankAddress), Account1Addr, BalanceChangeBIG.ToBig(), params.TxGas, big.NewInt(params.InitialBaseFee), nil), signer, TestBankKey)
 		block.AddTx(tx)
 	case 1:
 		// In block 2 Account1Addr creates a test contract.

test_helpers/util.go

@@ -6,9 +6,9 @@ import (
 	"github.com/cerc-io/plugeth-statediff/utils/log"
 )
 
-// QuietLogs silences the geth logs and sets the plugin test log level to "warning"
+// QuietLogs discards the geth logs and sets the plugin test log level to "warning"
-// The geth sync logs are noisy, so it can be nice to silence them.
+// The geth sync logs are noisy, so during some tests it helps to silence them.
 func QuietLogs() {
-	geth_log.Root().SetHandler(geth_log.DiscardHandler())
+	geth_log.SetDefault(geth_log.New(geth_log.DiscardHandler()))
 	log.TestLogger.SetLevel(2)
 }

utils/utils.go

@@ -51,8 +51,6 @@ func ChainConfig(chainID uint64) (*params.ChainConfig, error) {
 	switch chainID {
 	case 1:
 		return params.MainnetChainConfig, nil
-	case 4:
-		return params.RinkebyChainConfig, nil
 	case 5:
 		return params.GoerliChainConfig, nil
 	default: