diff --git a/statediff/indexer/indexer.go b/statediff/indexer/indexer.go
index bab5fa938..60d69f932 100644
--- a/statediff/indexer/indexer.go
+++ b/statediff/indexer/indexer.go
@@ -299,7 +299,7 @@ type processArgs struct {
rctTrieNodes []*ipld.EthRctTrie
txNodes []*ipld.EthTx
txTrieNodes []*ipld.EthTxTrie
- logTrieNodes [][]*ipld.EthLogTrie
+ logTrieNodes [][]node.Node
logLeafNodeCIDs [][]cid.Cid
rctLeafNodeCIDs []cid.Cid
}
diff --git a/statediff/indexer/indexer_test.go b/statediff/indexer/indexer_test.go
index 67645a12d..5c73e3de3 100644
--- a/statediff/indexer/indexer_test.go
+++ b/statediff/indexer/indexer_test.go
@@ -49,6 +49,7 @@ var (
mockBlock *types.Block
headerCID, trx1CID, trx2CID, trx3CID, trx4CID, trx5CID cid.Cid
rct1CID, rct2CID, rct3CID, rct4CID, rct5CID cid.Cid
+ rctLeaf1, rctLeaf2, rctLeaf3, rctLeaf4, rctLeaf5 []byte
state1CID, state2CID, storageCID cid.Cid
)
@@ -124,14 +125,49 @@ func init() {
trx3CID, _ = ipld.RawdataToCid(ipld.MEthTx, tx3, multihash.KECCAK_256)
trx4CID, _ = ipld.RawdataToCid(ipld.MEthTx, tx4, multihash.KECCAK_256)
trx5CID, _ = ipld.RawdataToCid(ipld.MEthTx, tx5, multihash.KECCAK_256)
- rct1CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, rct1, multihash.KECCAK_256)
- rct2CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, rct2, multihash.KECCAK_256)
- rct3CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, rct3, multihash.KECCAK_256)
- rct4CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, rct4, multihash.KECCAK_256)
- rct5CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, rct5, multihash.KECCAK_256)
+ /*
+ rct1Node, _ := ipld.NewReceipt(rcts[0])
+ rct2Node, _ := ipld.NewReceipt(rcts[1])
+ rct3Node, _ := ipld.NewReceipt(rcts[2])
+ rct4Node, _ := ipld.NewReceipt(rcts[3])
+ rct5Node, _ := ipld.NewReceipt(rcts[4])
+ */
state1CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, mocks.ContractLeafNode, multihash.KECCAK_256)
state2CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, mocks.AccountLeafNode, multihash.KECCAK_256)
storageCID, _ = ipld.RawdataToCid(ipld.MEthStorageTrie, mocks.StorageLeafNode, multihash.KECCAK_256)
+
+ receiptTrie := ipld.NewRctTrie()
+
+ receiptTrie.Add(0, rct1)
+ receiptTrie.Add(1, rct2)
+ receiptTrie.Add(2, rct3)
+ receiptTrie.Add(3, rct4)
+ receiptTrie.Add(4, rct5)
+
+ rctLeafNodes, keys, _ := receiptTrie.GetLeafNodes()
+
+ rctleafNodeCids := make([]cid.Cid, len(rctLeafNodes))
+ orderedRctLeafNodes := make([][]byte, len(rctLeafNodes))
+ for i, rln := range rctLeafNodes {
+ var idx uint
+
+ r := bytes.NewReader(keys[i].TrieKey)
+ rlp.Decode(r, &idx)
+ rctleafNodeCids[idx] = rln.Cid()
+ orderedRctLeafNodes[idx] = rln.RawData()
+ }
+
+ rct1CID = rctleafNodeCids[0]
+ rct2CID = rctleafNodeCids[1]
+ rct3CID = rctleafNodeCids[2]
+ rct4CID = rctleafNodeCids[3]
+ rct5CID = rctleafNodeCids[4]
+
+ rctLeaf1 = orderedRctLeafNodes[0]
+ rctLeaf2 = orderedRctLeafNodes[1]
+ rctLeaf3 = orderedRctLeafNodes[2]
+ rctLeaf4 = orderedRctLeafNodes[3]
+ rctLeaf5 = orderedRctLeafNodes[4]
}
func setup(t *testing.T) {
@@ -416,7 +452,7 @@ func TestPublishAndIndexer(t *testing.T) {
switch c {
case rct1CID.String():
- shared.ExpectEqual(t, data, rct1)
+ shared.ExpectEqual(t, data, rctLeaf1)
var postStatus uint64
pgStr = `SELECT post_status FROM eth.receipt_cids WHERE leaf_cid = $1`
err = db.Get(&postStatus, pgStr, c)
@@ -425,7 +461,7 @@ func TestPublishAndIndexer(t *testing.T) {
}
shared.ExpectEqual(t, postStatus, mocks.ExpectedPostStatus)
case rct2CID.String():
- shared.ExpectEqual(t, data, rct2)
+ shared.ExpectEqual(t, data, rctLeaf2)
var postState string
pgStr = `SELECT post_state FROM eth.receipt_cids WHERE leaf_cid = $1`
err = db.Get(&postState, pgStr, c)
@@ -434,7 +470,7 @@ func TestPublishAndIndexer(t *testing.T) {
}
shared.ExpectEqual(t, postState, mocks.ExpectedPostState1)
case rct3CID.String():
- shared.ExpectEqual(t, data, rct3)
+ shared.ExpectEqual(t, data, rctLeaf3)
var postState string
pgStr = `SELECT post_state FROM eth.receipt_cids WHERE leaf_cid = $1`
err = db.Get(&postState, pgStr, c)
@@ -443,7 +479,7 @@ func TestPublishAndIndexer(t *testing.T) {
}
shared.ExpectEqual(t, postState, mocks.ExpectedPostState2)
case rct4CID.String():
- shared.ExpectEqual(t, data, rct4)
+ shared.ExpectEqual(t, data, rctLeaf4)
var postState string
pgStr = `SELECT post_state FROM eth.receipt_cids WHERE leaf_cid = $1`
err = db.Get(&postState, pgStr, c)
@@ -452,7 +488,7 @@ func TestPublishAndIndexer(t *testing.T) {
}
shared.ExpectEqual(t, postState, mocks.ExpectedPostState3)
case rct5CID.String():
- shared.ExpectEqual(t, data, rct5)
+ shared.ExpectEqual(t, data, rctLeaf5)
var postState string
pgStr = `SELECT post_state FROM eth.receipt_cids WHERE leaf_cid = $1`
err = db.Get(&postState, pgStr, c)
diff --git a/statediff/indexer/ipfs/ipld/eth_log_trie.go b/statediff/indexer/ipfs/ipld/eth_log_trie.go
index 2e36f0a68..90d694c1b 100644
--- a/statediff/indexer/ipfs/ipld/eth_log_trie.go
+++ b/statediff/indexer/ipfs/ipld/eth_log_trie.go
@@ -3,6 +3,8 @@ package ipld
import (
"fmt"
+ node "github.com/ipfs/go-ipld-format"
+
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ipfs/go-cid"
@@ -91,13 +93,13 @@ func newLogTrie() *logTrie {
// getNodes invokes the localTrie, which computes the root hash of the
// log trie and returns its database keys, to return a slice
// of EthLogTrie nodes.
-func (rt *logTrie) getNodes() ([]*EthLogTrie, error) {
+func (rt *logTrie) getNodes() ([]node.Node, error) {
keys, err := rt.getKeys()
if err != nil {
return nil, err
}
- out := make([]*EthLogTrie, 0, len(keys))
+ out := make([]node.Node, 0, len(keys))
for _, k := range keys {
n, err := rt.getNodeFromDB(k)
if err != nil {
@@ -114,14 +116,8 @@ func (rt *logTrie) getNodeFromDB(key []byte) (*EthLogTrie, error) {
if err != nil {
return nil, err
}
-
- c, err := RawdataToCid(MEthLogTrie, rawdata, multihash.KECCAK_256)
- if err != nil {
- return nil, err
- }
-
tn := &TrieNode{
- cid: c,
+ cid: keccak256ToCid(MEthLogTrie, key),
rawdata: rawdata,
}
return &EthLogTrie{TrieNode: tn}, nil
diff --git a/statediff/indexer/ipfs/ipld/eth_parser.go b/statediff/indexer/ipfs/ipld/eth_parser.go
index 0b4780f8a..d2608bbd5 100644
--- a/statediff/indexer/ipfs/ipld/eth_parser.go
+++ b/statediff/indexer/ipfs/ipld/eth_parser.go
@@ -23,6 +23,8 @@ import (
"io"
"io/ioutil"
+ node "github.com/ipfs/go-ipld-format"
+
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/rlp"
@@ -123,7 +125,7 @@ func FromBlockJSON(r io.Reader) (*EthHeader, []*EthTx, []*EthTxTrie, error) {
// FromBlockAndReceipts takes a block and processes it
// to return it a set of IPLD nodes for further processing.
-func FromBlockAndReceipts(block *types.Block, receipts []*types.Receipt) (*EthHeader, []*EthHeader, []*EthTx, []*EthTxTrie, []*EthReceipt, []*EthRctTrie, [][]*EthLogTrie, [][]cid.Cid, []cid.Cid, error) {
+func FromBlockAndReceipts(block *types.Block, receipts []*types.Receipt) (*EthHeader, []*EthHeader, []*EthTx, []*EthTxTrie, []*EthReceipt, []*EthRctTrie, [][]node.Node, [][]cid.Cid, []cid.Cid, error) {
// Process the header
headerNode, err := NewEthHeader(block.Header())
if err != nil {
@@ -148,10 +150,10 @@ func FromBlockAndReceipts(block *types.Block, receipts []*types.Receipt) (*EthHe
}
// Process the receipts and logs
- rctNodes, tctTrieNodes, logTrieNodes, logLeafNodeCIDs, rctLeafNodeCIDs, err := processReceiptsAndLogs(receipts,
+ rctNodes, tctTrieNodes, logTrieAndLogNodes, logLeafNodeCIDs, rctLeafNodeCIDs, err := processReceiptsAndLogs(receipts,
block.Header().ReceiptHash[:])
- return headerNode, uncleNodes, txNodes, txTrieNodes, rctNodes, tctTrieNodes, logTrieNodes, logLeafNodeCIDs, rctLeafNodeCIDs, err
+ return headerNode, uncleNodes, txNodes, txTrieNodes, rctNodes, tctTrieNodes, logTrieAndLogNodes, logLeafNodeCIDs, rctLeafNodeCIDs, err
}
// processTransactions will take the found transactions in a parsed block body
@@ -180,11 +182,11 @@ func processTransactions(txs []*types.Transaction, expectedTxRoot []byte) ([]*Et
// processReceiptsAndLogs will take in receipts
// to return IPLD node slices for eth-rct, eth-rct-trie, eth-log, eth-log-trie, eth-log-trie-CID, eth-rct-trie-CID
-func processReceiptsAndLogs(rcts []*types.Receipt, expectedRctRoot []byte) ([]*EthReceipt, []*EthRctTrie, [][]*EthLogTrie, [][]cid.Cid, []cid.Cid, error) {
+func processReceiptsAndLogs(rcts []*types.Receipt, expectedRctRoot []byte) ([]*EthReceipt, []*EthRctTrie, [][]node.Node, [][]cid.Cid, []cid.Cid, error) {
// Pre allocating memory.
ethRctNodes := make([]*EthReceipt, 0, len(rcts))
ethLogleafNodeCids := make([][]cid.Cid, 0, len(rcts))
- ethLogTrieNodes := make([][]*EthLogTrie, 0, len(rcts))
+ ethLogTrieAndLogNodes := make([][]node.Node, 0, len(rcts))
receiptTrie := NewRctTrie()
@@ -195,7 +197,7 @@ func processReceiptsAndLogs(rcts []*types.Receipt, expectedRctRoot []byte) ([]*E
return nil, nil, nil, nil, nil, err
}
rct.LogRoot = logTrieHash
- ethLogTrieNodes = append(ethLogTrieNodes, logTrieNodes)
+ ethLogTrieAndLogNodes = append(ethLogTrieAndLogNodes, logTrieNodes)
ethLogleafNodeCids = append(ethLogleafNodeCids, leafNodeCids)
ethRct, err := NewReceipt(rct)
@@ -235,22 +237,38 @@ func processReceiptsAndLogs(rcts []*types.Receipt, expectedRctRoot []byte) ([]*E
ethRctleafNodeCids[idx] = rln.Cid()
}
- return ethRctNodes, rctTrieNodes, ethLogTrieNodes, ethLogleafNodeCids, ethRctleafNodeCids, err
+ return ethRctNodes, rctTrieNodes, ethLogTrieAndLogNodes, ethLogleafNodeCids, ethRctleafNodeCids, err
}
-func processLogs(logs []*types.Log) ([]*EthLogTrie, []cid.Cid, common.Hash, error) {
+const keccak256Length = 32
+
+func processLogs(logs []*types.Log) ([]node.Node, []cid.Cid, common.Hash, error) {
logTr := newLogTrie()
+ shortLog := make(map[uint64]*EthLog, len(logs))
for idx, log := range logs {
- ethLog, err := NewLog(log)
+ logRaw, err := rlp.EncodeToBytes(log)
if err != nil {
return nil, nil, common.Hash{}, err
}
- if err = logTr.Add(idx, ethLog.RawData()); err != nil {
+ // if len(logRaw) <= keccak256Length it is possible this value's "leaf node"
+ // will be stored in its parent branch but only if len(partialPathOfTheNode) + len(logRaw) <= keccak256Length
+ // But we can't tell what the partial path will be until the trie is Commit()-ed
+ // So wait until we collect all the leaf nodes, and if we are missing any at the indexes we note in shortLogCIDs
+ // we know that these "leaf nodes" were internalized into their parent branch node and we move forward with
+ // using the cid.Cid we cached in shortLogCIDs
+ if len(logRaw) <= keccak256Length {
+ logNode, err := NewLog(log)
+ if err != nil {
+ return nil, nil, common.Hash{}, err
+ }
+ shortLog[uint64(idx)] = logNode
+ }
+ if err = logTr.Add(idx, logRaw); err != nil {
return nil, nil, common.Hash{}, err
}
}
- logTrieNodes, err := logTr.getNodes()
+ logTrieAndLogNodes, err := logTr.getNodes()
if err != nil {
return nil, nil, common.Hash{}, err
}
@@ -259,8 +277,7 @@ func processLogs(logs []*types.Log) ([]*EthLogTrie, []cid.Cid, common.Hash, erro
if err != nil {
return nil, nil, common.Hash{}, err
}
-
- leafNodeCids := make([]cid.Cid, len(leafNodes))
+ leafNodeCids := make([]cid.Cid, len(logs))
for i, ln := range leafNodes {
var idx uint
@@ -271,6 +288,15 @@ func processLogs(logs []*types.Log) ([]*EthLogTrie, []cid.Cid, common.Hash, erro
}
leafNodeCids[idx] = ln.Cid()
}
+ // this is where we check which logs <= keccak256Length were actually internalized into parent branch node
+ // and replace those that were with the cid.Cid for the raw log IPLD
+ for i, l := range shortLog {
+ if !leafNodeCids[i].Defined() {
+ leafNodeCids[i] = l.Cid()
+ // if the leaf node was internalized, we append an IPLD for log itself to the list of IPLDs we need to publish
+ logTrieAndLogNodes = append(logTrieAndLogNodes, l)
+ }
+ }
- return logTrieNodes, leafNodeCids, common.BytesToHash(logTr.rootHash()), err
+ return logTrieAndLogNodes, leafNodeCids, common.BytesToHash(logTr.rootHash()), err
}
diff --git a/statediff/indexer/ipfs/ipld/eth_receipt_trie.go b/statediff/indexer/ipfs/ipld/eth_receipt_trie.go
index fc1480703..b31b14d6b 100644
--- a/statediff/indexer/ipfs/ipld/eth_receipt_trie.go
+++ b/statediff/indexer/ipfs/ipld/eth_receipt_trie.go
@@ -166,14 +166,8 @@ func (rt *rctTrie) getNodeFromDB(key []byte) (*EthRctTrie, error) {
if err != nil {
return nil, err
}
-
- cid, err := RawdataToCid(MEthTxReceiptTrie, rawdata, multihash.KECCAK_256)
- if err != nil {
- return nil, err
- }
-
tn := &TrieNode{
- cid: cid,
+ cid: keccak256ToCid(MEthTxReceiptTrie, key),
rawdata: rawdata,
}
diff --git a/statediff/indexer/ipfs/ipld/eth_tx_trie.go b/statediff/indexer/ipfs/ipld/eth_tx_trie.go
index 7e79ff164..4edc7dd7b 100644
--- a/statediff/indexer/ipfs/ipld/eth_tx_trie.go
+++ b/statediff/indexer/ipfs/ipld/eth_tx_trie.go
@@ -135,12 +135,8 @@ func (tt *txTrie) getNodes() ([]*EthTxTrie, error) {
if err != nil {
return nil, err
}
- c, err := RawdataToCid(MEthTxTrie, rawdata, multihash.KECCAK_256)
- if err != nil {
- return nil, err
- }
tn := &TrieNode{
- cid: c,
+ cid: keccak256ToCid(MEthTxTrie, k),
rawdata: rawdata,
}
out = append(out, &EthTxTrie{TrieNode: tn})
diff --git a/statediff/indexer/mainnet_tests/block_12579670.rlp b/statediff/indexer/mainnet_tests/block_12579670.rlp
new file mode 100644
index 000000000..6b4f3f773
Binary files /dev/null and b/statediff/indexer/mainnet_tests/block_12579670.rlp differ
diff --git a/statediff/indexer/mainnet_tests/block_12600011.rlp b/statediff/indexer/mainnet_tests/block_12600011.rlp
new file mode 100644
index 000000000..96032b0c2
Binary files /dev/null and b/statediff/indexer/mainnet_tests/block_12600011.rlp differ
diff --git a/statediff/indexer/mainnet_tests/block_12619985.rlp b/statediff/indexer/mainnet_tests/block_12619985.rlp
new file mode 100644
index 000000000..0e735313f
Binary files /dev/null and b/statediff/indexer/mainnet_tests/block_12619985.rlp differ
diff --git a/statediff/indexer/mainnet_tests/block_12625121.rlp b/statediff/indexer/mainnet_tests/block_12625121.rlp
new file mode 100644
index 000000000..d031e30ea
Binary files /dev/null and b/statediff/indexer/mainnet_tests/block_12625121.rlp differ
diff --git a/statediff/indexer/mainnet_tests/block_12655432.rlp b/statediff/indexer/mainnet_tests/block_12655432.rlp
new file mode 100644
index 000000000..fafc6bd88
Binary files /dev/null and b/statediff/indexer/mainnet_tests/block_12655432.rlp differ
diff --git a/statediff/indexer/mainnet_tests/block_12914664.rlp b/statediff/indexer/mainnet_tests/block_12914664.rlp
new file mode 100644
index 000000000..b8aaeaa61
Binary files /dev/null and b/statediff/indexer/mainnet_tests/block_12914664.rlp differ
diff --git a/statediff/indexer/mainnet_tests/indexer_test.go b/statediff/indexer/mainnet_tests/indexer_test.go
new file mode 100644
index 000000000..16cc57800
--- /dev/null
+++ b/statediff/indexer/mainnet_tests/indexer_test.go
@@ -0,0 +1,107 @@
+// VulcanizeDB
+// Copyright © 2021 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package mainnet_tests
+
+import (
+ "fmt"
+ "math/big"
+ "os"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/ethereum/go-ethereum/core/types"
+ "github.com/ethereum/go-ethereum/params"
+ "github.com/ethereum/go-ethereum/statediff/indexer"
+ "github.com/ethereum/go-ethereum/statediff/indexer/mocks"
+ "github.com/ethereum/go-ethereum/statediff/indexer/postgres"
+ "github.com/ethereum/go-ethereum/statediff/indexer/shared"
+)
+
+var (
+ err error
+ db *postgres.DB
+ chainConf = params.MainnetChainConfig
+)
+
+func init() {
+ if os.Getenv("MODE") != "statediff" {
+ fmt.Println("Skipping statediff test")
+ os.Exit(0)
+ }
+}
+
+func TestPushBlockAndState(t *testing.T) {
+ conf := DefaultTestConfig
+ rawURL := os.Getenv(TEST_RAW_URL)
+ if rawURL == "" {
+ fmt.Printf("Warning: no raw url configured for statediffing mainnet tests, will look for local file and"+
+ "then try default endpoint (%s)\r\n", DefaultTestConfig.RawURL)
+ } else {
+ conf.RawURL = rawURL
+ }
+ for _, blockNumber := range problemBlocks {
+ conf.BlockNumber = big.NewInt(blockNumber)
+ tb, trs, err := TestBlockAndReceipts(conf)
+ require.NoError(t, err)
+ testPushBlockAndState(t, tb, trs)
+ }
+ testBlock, testReceipts, err := TestBlockAndReceiptsFromEnv(conf)
+ require.NoError(t, err)
+ testPushBlockAndState(t, testBlock, testReceipts)
+}
+
+func testPushBlockAndState(t *testing.T, block *types.Block, receipts types.Receipts) {
+ t.Run("Test PushBlock and PushStateNode", func(t *testing.T) {
+ setup(t, block, receipts)
+ tearDown(t)
+ })
+}
+
+func setup(t *testing.T, testBlock *types.Block, testReceipts types.Receipts) {
+ db, err = shared.SetupDB()
+ if err != nil {
+ t.Fatal(err)
+ }
+ ind, err := indexer.NewStateDiffIndexer(chainConf, db)
+ require.NoError(t, err)
+ var tx *indexer.BlockTx
+ tx, err = ind.PushBlock(
+ testBlock,
+ testReceipts,
+ testBlock.Difficulty())
+ require.NoError(t, err)
+
+ defer func() {
+ if err := tx.Close(err); err != nil {
+ t.Fatal(err)
+ }
+ }()
+
+ for _, node := range mocks.StateDiffs {
+ err = ind.PushStateNode(tx, node)
+ require.NoError(t, err)
+ }
+
+ shared.ExpectEqual(t, tx.BlockNumber, testBlock.Number().Uint64())
+}
+
+func tearDown(t *testing.T) {
+ indexer.TearDownDB(t, db)
+ err = db.Close()
+ require.NoError(t, err)
+}
diff --git a/statediff/indexer/mainnet_tests/receipts_12579670.rlp b/statediff/indexer/mainnet_tests/receipts_12579670.rlp
new file mode 100644
index 000000000..e69de8fd9
Binary files /dev/null and b/statediff/indexer/mainnet_tests/receipts_12579670.rlp differ
diff --git a/statediff/indexer/mainnet_tests/receipts_12600011.rlp b/statediff/indexer/mainnet_tests/receipts_12600011.rlp
new file mode 100644
index 000000000..ae6d4f0c2
Binary files /dev/null and b/statediff/indexer/mainnet_tests/receipts_12600011.rlp differ
diff --git a/statediff/indexer/mainnet_tests/receipts_12619985.rlp b/statediff/indexer/mainnet_tests/receipts_12619985.rlp
new file mode 100644
index 000000000..a9ba84bd2
Binary files /dev/null and b/statediff/indexer/mainnet_tests/receipts_12619985.rlp differ
diff --git a/statediff/indexer/mainnet_tests/receipts_12625121.rlp b/statediff/indexer/mainnet_tests/receipts_12625121.rlp
new file mode 100644
index 000000000..4d3a8532c
Binary files /dev/null and b/statediff/indexer/mainnet_tests/receipts_12625121.rlp differ
diff --git a/statediff/indexer/mainnet_tests/receipts_12655432.rlp b/statediff/indexer/mainnet_tests/receipts_12655432.rlp
new file mode 100644
index 000000000..f209f01d8
Binary files /dev/null and b/statediff/indexer/mainnet_tests/receipts_12655432.rlp differ
diff --git a/statediff/indexer/mainnet_tests/receipts_12914664.rlp b/statediff/indexer/mainnet_tests/receipts_12914664.rlp
new file mode 100644
index 000000000..3cf8e8895
Binary files /dev/null and b/statediff/indexer/mainnet_tests/receipts_12914664.rlp differ
diff --git a/statediff/indexer/mainnet_tests/test_helpers.go b/statediff/indexer/mainnet_tests/test_helpers.go
new file mode 100644
index 000000000..7203649d4
--- /dev/null
+++ b/statediff/indexer/mainnet_tests/test_helpers.go
@@ -0,0 +1,235 @@
+// VulcanizeDB
+// Copyright © 2021 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package mainnet_tests
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "math/big"
+ "os"
+
+ "github.com/ethereum/go-ethereum/core/types"
+ "github.com/ethereum/go-ethereum/ethclient"
+ "github.com/ethereum/go-ethereum/rlp"
+)
+
+const (
+ defaultBlockFilePath = "./block"
+ defaultReceiptsFilePath = "./receipts"
+)
+
+const (
+ TEST_RAW_URL = "TEST_RAW_URL"
+ TEST_BLOCK_NUMBER = "TEST_BLOCK_NUMBER"
+)
+
+var problemBlocks = []int64{
+ 12600011,
+ 12619985,
+ 12625121,
+ 12655432,
+ 12579670,
+ 12914664,
+}
+
+// TestConfig holds configuration params for mainnet tests
+type TestConfig struct {
+ RawURL string
+ BlockNumber *big.Int
+ LocalCache bool
+}
+
+// DefaultTestConfig is the default TestConfig
+var DefaultTestConfig = TestConfig{
+ RawURL: "http://127.0.0.1:8545",
+ BlockNumber: big.NewInt(12914664),
+ LocalCache: true,
+}
+
+// TestBlockAndReceiptsFromEnv retrieves the block and receipts using env variables to override default config block number
+func TestBlockAndReceiptsFromEnv(conf TestConfig) (*types.Block, types.Receipts, error) {
+ blockNumberStr := os.Getenv(TEST_BLOCK_NUMBER)
+ blockNumber, ok := new(big.Int).SetString(blockNumberStr, 10)
+ if !ok {
+ fmt.Printf("Warning: no blockNumber configured for statediffing mainnet tests, using default (%d)\r\n",
+ DefaultTestConfig.BlockNumber)
+ } else {
+ conf.BlockNumber = blockNumber
+ }
+ return TestBlockAndReceipts(conf)
+}
+
+// TestBlockAndReceipts retrieves the block and receipts for the provided test config
+// It first tries to load files from the local system before setting up and using an ethclient.Client to pull the data
+func TestBlockAndReceipts(conf TestConfig) (*types.Block, types.Receipts, error) {
+ var cli *ethclient.Client
+ var err error
+ var block *types.Block
+ var receipts types.Receipts
+ blockFilePath := fmt.Sprintf("%s_%s.rlp", defaultBlockFilePath, conf.BlockNumber.String())
+ if _, err = os.Stat(blockFilePath); !errors.Is(err, os.ErrNotExist) {
+ fmt.Printf("local file (%s) found for block %s\n", blockFilePath, conf.BlockNumber.String())
+ block, err = LoadBlockRLP(blockFilePath)
+ if err != nil {
+ fmt.Printf("loading local file (%s) failed (%s), dialing remote client at %s\n", blockFilePath, err.Error(), conf.RawURL)
+ cli, err = ethclient.Dial(conf.RawURL)
+ if err != nil {
+ return nil, nil, err
+ }
+ block, err = FetchBlock(cli, conf.BlockNumber)
+ if err != nil {
+ return nil, nil, err
+ }
+ if conf.LocalCache {
+ if err := WriteBlockRLP(blockFilePath, block); err != nil {
+ return nil, nil, err
+ }
+ }
+ }
+ } else {
+ fmt.Printf("no local file found for block %s, dialing remote client at %s\n", conf.BlockNumber.String(), conf.RawURL)
+ cli, err = ethclient.Dial(conf.RawURL)
+ if err != nil {
+ return nil, nil, err
+ }
+ block, err = FetchBlock(cli, conf.BlockNumber)
+ if err != nil {
+ return nil, nil, err
+ }
+ if conf.LocalCache {
+ if err := WriteBlockRLP(blockFilePath, block); err != nil {
+ return nil, nil, err
+ }
+ }
+ }
+ receiptsFilePath := fmt.Sprintf("%s_%s.rlp", defaultReceiptsFilePath, conf.BlockNumber.String())
+ if _, err = os.Stat(receiptsFilePath); !errors.Is(err, os.ErrNotExist) {
+ fmt.Printf("local file (%s) found for block %s receipts\n", receiptsFilePath, conf.BlockNumber.String())
+ receipts, err = LoadReceiptsEncoding(receiptsFilePath, len(block.Transactions()))
+ if err != nil {
+ fmt.Printf("loading local file (%s) failed (%s), dialing remote client at %s\n", receiptsFilePath, err.Error(), conf.RawURL)
+ if cli == nil {
+ cli, err = ethclient.Dial(conf.RawURL)
+ if err != nil {
+ return nil, nil, err
+ }
+ }
+ receipts, err = FetchReceipts(cli, block)
+ if err != nil {
+ return nil, nil, err
+ }
+ if conf.LocalCache {
+ if err := WriteReceiptsEncoding(receiptsFilePath, block.Number(), receipts); err != nil {
+ return nil, nil, err
+ }
+ }
+ }
+ } else {
+ fmt.Printf("no local file found for block %s receipts, dialing remote client at %s\n", conf.BlockNumber.String(), conf.RawURL)
+ if cli == nil {
+ cli, err = ethclient.Dial(conf.RawURL)
+ if err != nil {
+ return nil, nil, err
+ }
+ }
+ receipts, err = FetchReceipts(cli, block)
+ if err != nil {
+ return nil, nil, err
+ }
+ if conf.LocalCache {
+ if err := WriteReceiptsEncoding(receiptsFilePath, block.Number(), receipts); err != nil {
+ return nil, nil, err
+ }
+ }
+ }
+ return block, receipts, nil
+}
+
+// FetchBlock fetches the block at the provided height using the ethclient.Client
+func FetchBlock(cli *ethclient.Client, blockNumber *big.Int) (*types.Block, error) {
+ return cli.BlockByNumber(context.Background(), blockNumber)
+}
+
+// FetchReceipts fetches the receipts for the provided block using the ethclient.Client
+func FetchReceipts(cli *ethclient.Client, block *types.Block) (types.Receipts, error) {
+ receipts := make(types.Receipts, len(block.Transactions()))
+ for i, tx := range block.Transactions() {
+ rct, err := cli.TransactionReceipt(context.Background(), tx.Hash())
+ if err != nil {
+ return nil, err
+ }
+ receipts[i] = rct
+ }
+ return receipts, nil
+}
+
+// WriteBlockRLP writes out the RLP encoding of the block to the provided filePath
+func WriteBlockRLP(filePath string, block *types.Block) error {
+ if filePath == "" {
+ filePath = fmt.Sprintf("%s_%s.rlp", defaultBlockFilePath, block.Number().String())
+ }
+ if _, err := os.Stat(filePath); !errors.Is(err, os.ErrNotExist) {
+ return fmt.Errorf("cannot create file, file (%s) already exists", filePath)
+ }
+ file, err := os.Create(filePath)
+ if err != nil {
+ return fmt.Errorf("unable to create file (%s), err: %v", filePath, err)
+ }
+ fmt.Printf("writing block rlp to file at %s\r\n", filePath)
+ if err := block.EncodeRLP(file); err != nil {
+ return err
+ }
+ return file.Close()
+}
+
+// LoadBlockRLP loads block from the rlp at filePath
+func LoadBlockRLP(filePath string) (*types.Block, error) {
+ blockBytes, err := os.ReadFile(filePath)
+ if err != nil {
+ return nil, err
+ }
+ block := new(types.Block)
+ return block, rlp.DecodeBytes(blockBytes, block)
+}
+
+// LoadReceiptsEncoding loads receipts from the encoding at filePath
+func LoadReceiptsEncoding(filePath string, cap int) (types.Receipts, error) {
+ rctsBytes, err := os.ReadFile(filePath)
+ if err != nil {
+ return nil, err
+ }
+ receipts := new(types.Receipts)
+ return *receipts, rlp.DecodeBytes(rctsBytes, receipts)
+}
+
+// WriteReceiptsEncoding writes out the consensus encoding of the receipts to the provided io.WriteCloser
+func WriteReceiptsEncoding(filePath string, blockNumber *big.Int, receipts types.Receipts) error {
+ if filePath == "" {
+ filePath = fmt.Sprintf("%s_%s.rlp", defaultReceiptsFilePath, blockNumber.String())
+ }
+ if _, err := os.Stat(filePath); !errors.Is(err, os.ErrNotExist) {
+ return fmt.Errorf("cannot create file, file (%s) already exists", filePath)
+ }
+ file, err := os.Create(filePath)
+ if err != nil {
+ return fmt.Errorf("unable to create file (%s), err: %v", filePath, err)
+ }
+ defer file.Close()
+ fmt.Printf("writing receipts rlp to file at %s\r\n", filePath)
+ return rlp.Encode(file, receipts)
+}