diff --git a/statediff/indexer/database/dump/indexer.go b/statediff/indexer/database/dump/indexer.go
index b2fc70d27..e450f941a 100644
--- a/statediff/indexer/database/dump/indexer.go
+++ b/statediff/indexer/database/dump/indexer.go
@@ -242,7 +242,7 @@ type processArgs struct {
rctTrieNodes []*ipld2.EthRctTrie
txNodes []*ipld2.EthTx
txTrieNodes []*ipld2.EthTxTrie
- logTrieNodes [][]*ipld2.EthLogTrie
+ logTrieNodes [][]node.Node
logLeafNodeCIDs [][]cid.Cid
rctLeafNodeCIDs []cid.Cid
}
diff --git a/statediff/indexer/database/file/indexer.go b/statediff/indexer/database/file/indexer.go
index a9f5dd0fb..870c1f259 100644
--- a/statediff/indexer/database/file/indexer.go
+++ b/statediff/indexer/database/file/indexer.go
@@ -250,7 +250,7 @@ type processArgs struct {
rctTrieNodes []*ipld2.EthRctTrie
txNodes []*ipld2.EthTx
txTrieNodes []*ipld2.EthTxTrie
- logTrieNodes [][]*ipld2.EthLogTrie
+ logTrieNodes [][]node.Node
logLeafNodeCIDs [][]cid.Cid
rctLeafNodeCIDs []cid.Cid
}
diff --git a/statediff/indexer/database/file/mainnet_tests/indexer_test.go b/statediff/indexer/database/file/mainnet_tests/indexer_test.go
index 32850ca29..b721ff05e 100644
--- a/statediff/indexer/database/file/mainnet_tests/indexer_test.go
+++ b/statediff/indexer/database/file/mainnet_tests/indexer_test.go
@@ -49,21 +49,21 @@ func init() {
}
func TestPushBlockAndState(t *testing.T) {
- conf := DefaultTestConfig
- rawURL := os.Getenv(TEST_RAW_URL)
+ conf := test_helpers.DefaultTestConfig
+ rawURL := os.Getenv(test_helpers.TEST_RAW_URL)
if rawURL == "" {
fmt.Printf("Warning: no raw url configured for statediffing mainnet tests, will look for local file and"+
- "then try default endpoint (%s)\r\n", DefaultTestConfig.RawURL)
+ "then try default endpoint (%s)\r\n", test_helpers.DefaultTestConfig.RawURL)
} else {
conf.RawURL = rawURL
}
- for _, blockNumber := range problemBlocks {
+ for _, blockNumber := range test_helpers.ProblemBlocks {
conf.BlockNumber = big.NewInt(blockNumber)
- tb, trs, err := TestBlockAndReceipts(conf)
+ tb, trs, err := test_helpers.TestBlockAndReceipts(conf)
require.NoError(t, err)
testPushBlockAndState(t, tb, trs)
}
- testBlock, testReceipts, err := TestBlockAndReceiptsFromEnv(conf)
+ testBlock, testReceipts, err := test_helpers.TestBlockAndReceiptsFromEnv(conf)
require.NoError(t, err)
testPushBlockAndState(t, testBlock, testReceipts)
}
diff --git a/statediff/indexer/database/sql/indexer.go b/statediff/indexer/database/sql/indexer.go
index 72e978aad..c8d526d6f 100644
--- a/statediff/indexer/database/sql/indexer.go
+++ b/statediff/indexer/database/sql/indexer.go
@@ -297,7 +297,7 @@ type processArgs struct {
rctTrieNodes []*ipld2.EthRctTrie
txNodes []*ipld2.EthTx
txTrieNodes []*ipld2.EthTxTrie
- logTrieNodes [][]*ipld2.EthLogTrie
+ logTrieNodes [][]node.Node
logLeafNodeCIDs [][]cid.Cid
rctLeafNodeCIDs []cid.Cid
}
diff --git a/statediff/indexer/database/sql/mainnet_tests/indexer_test.go b/statediff/indexer/database/sql/mainnet_tests/indexer_test.go
new file mode 100644
index 000000000..68c9bc464
--- /dev/null
+++ b/statediff/indexer/database/sql/mainnet_tests/indexer_test.go
@@ -0,0 +1,109 @@
+// VulcanizeDB
+// Copyright © 2021 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see .
+
+package mainnet_tests
+
+import (
+ "context"
+ "fmt"
+ "math/big"
+ "os"
+ "testing"
+
+ "github.com/stretchr/testify/require"
+
+ "github.com/ethereum/go-ethereum/core/types"
+ "github.com/ethereum/go-ethereum/params"
+ "github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
+ "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
+ "github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
+ "github.com/ethereum/go-ethereum/statediff/indexer/mocks"
+ "github.com/ethereum/go-ethereum/statediff/indexer/test_helpers"
+)
+
+var (
+ err error
+ db sql.Database
+ ind interfaces.StateDiffIndexer
+ chainConf = params.MainnetChainConfig
+)
+
+func init() {
+ if os.Getenv("MODE") != "statediff" {
+ fmt.Println("Skipping statediff test")
+ os.Exit(0)
+ }
+}
+
+func TestPushBlockAndState(t *testing.T) {
+ conf := test_helpers.DefaultTestConfig
+ rawURL := os.Getenv(test_helpers.TEST_RAW_URL)
+ if rawURL == "" {
+ fmt.Printf("Warning: no raw url configured for statediffing mainnet tests, will look for local file and"+
+ "then try default endpoint (%s)\r\n", test_helpers.DefaultTestConfig.RawURL)
+ } else {
+ conf.RawURL = rawURL
+ }
+ for _, blockNumber := range test_helpers.ProblemBlocks {
+ conf.BlockNumber = big.NewInt(blockNumber)
+ tb, trs, err := test_helpers.TestBlockAndReceipts(conf)
+ require.NoError(t, err)
+ testPushBlockAndState(t, tb, trs)
+ }
+ testBlock, testReceipts, err := test_helpers.TestBlockAndReceiptsFromEnv(conf)
+ require.NoError(t, err)
+ testPushBlockAndState(t, testBlock, testReceipts)
+}
+
+func testPushBlockAndState(t *testing.T, block *types.Block, receipts types.Receipts) {
+ t.Run("Test PushBlock and PushStateNode", func(t *testing.T) {
+ setup(t, block, receipts)
+ tearDown(t)
+ })
+}
+
+func setup(t *testing.T, testBlock *types.Block, testReceipts types.Receipts) {
+ db, err = postgres.SetupSQLXDB()
+ if err != nil {
+ t.Fatal(err)
+ }
+ ind, err = sql.NewStateDiffIndexer(context.Background(), chainConf, db)
+ require.NoError(t, err)
+ var tx interfaces.Batch
+ tx, err = ind.PushBlock(
+ testBlock,
+ testReceipts,
+ testBlock.Difficulty())
+ require.NoError(t, err)
+
+ defer func() {
+ if err := tx.Submit(err); err != nil {
+ t.Fatal(err)
+ }
+ }()
+ for _, node := range mocks.StateDiffs {
+ err = ind.PushStateNode(tx, node, testBlock.Hash().String())
+ require.NoError(t, err)
+ }
+
+ test_helpers.ExpectEqual(t, tx.(*sql.BatchTx).BlockNumber, testBlock.Number().Uint64())
+}
+
+func tearDown(t *testing.T) {
+ sql.TearDownDB(t, db)
+ err = ind.Close()
+ require.NoError(t, err)
+}
diff --git a/statediff/indexer/database/sql/postgres/config.go b/statediff/indexer/database/sql/postgres/config.go
index a7c7cc9b4..4fe2972ed 100644
--- a/statediff/indexer/database/sql/postgres/config.go
+++ b/statediff/indexer/database/sql/postgres/config.go
@@ -49,9 +49,9 @@ func ResolveDriverType(str string) (DriverType, error) {
var DefaultConfig = Config{
Hostname: "localhost",
Port: 5432,
- DatabaseName: "vulcanize_test",
- Username: "postgres",
- Password: "",
+ DatabaseName: "vulcanize_public",
+ Username: "vdbm",
+ Password: "password",
}
// Config holds params for a Postgres db
diff --git a/statediff/indexer/ipld/eth_log_trie.go b/statediff/indexer/ipld/eth_log_trie.go
index 1df21642d..8e8af9c79 100644
--- a/statediff/indexer/ipld/eth_log_trie.go
+++ b/statediff/indexer/ipld/eth_log_trie.go
@@ -3,6 +3,8 @@ package ipld
import (
"fmt"
+ node "github.com/ipfs/go-ipld-format"
+
"github.com/ipfs/go-cid"
"github.com/multiformats/go-multihash"
@@ -92,13 +94,13 @@ func newLogTrie() *logTrie {
// getNodes invokes the localTrie, which computes the root hash of the
// log trie and returns its sql keys, to return a slice
// of EthLogTrie nodes.
-func (rt *logTrie) getNodes() ([]*EthLogTrie, error) {
+func (rt *logTrie) getNodes() ([]node.Node, error) {
keys, err := rt.getKeys()
if err != nil {
return nil, err
}
- out := make([]*EthLogTrie, 0, len(keys))
+ out := make([]node.Node, 0, len(keys))
for _, k := range keys {
n, err := rt.getNodeFromDB(k)
if err != nil {
diff --git a/statediff/indexer/ipld/eth_parser.go b/statediff/indexer/ipld/eth_parser.go
index 497facba4..4e08f2d24 100644
--- a/statediff/indexer/ipld/eth_parser.go
+++ b/statediff/indexer/ipld/eth_parser.go
@@ -24,6 +24,7 @@ import (
"io/ioutil"
"github.com/ipfs/go-cid"
+ node "github.com/ipfs/go-ipld-format"
"github.com/multiformats/go-multihash"
"github.com/ethereum/go-ethereum/common"
@@ -124,7 +125,7 @@ func FromBlockJSON(r io.Reader) (*EthHeader, []*EthTx, []*EthTxTrie, error) {
// FromBlockAndReceipts takes a block and processes it
// to return it a set of IPLD nodes for further processing.
-func FromBlockAndReceipts(block *types.Block, receipts []*types.Receipt) (*EthHeader, []*EthHeader, []*EthTx, []*EthTxTrie, []*EthReceipt, []*EthRctTrie, [][]*EthLogTrie, [][]cid.Cid, []cid.Cid, error) {
+func FromBlockAndReceipts(block *types.Block, receipts []*types.Receipt) (*EthHeader, []*EthHeader, []*EthTx, []*EthTxTrie, []*EthReceipt, []*EthRctTrie, [][]node.Node, [][]cid.Cid, []cid.Cid, error) {
// Process the header
headerNode, err := NewEthHeader(block.Header())
if err != nil {
@@ -149,10 +150,10 @@ func FromBlockAndReceipts(block *types.Block, receipts []*types.Receipt) (*EthHe
}
// Process the receipts and logs
- rctNodes, tctTrieNodes, logTrieNodes, logLeafNodeCIDs, rctLeafNodeCIDs, err := processReceiptsAndLogs(receipts,
+ rctNodes, tctTrieNodes, logTrieAndLogNodes, logLeafNodeCIDs, rctLeafNodeCIDs, err := processReceiptsAndLogs(receipts,
block.Header().ReceiptHash[:])
- return headerNode, uncleNodes, txNodes, txTrieNodes, rctNodes, tctTrieNodes, logTrieNodes, logLeafNodeCIDs, rctLeafNodeCIDs, err
+ return headerNode, uncleNodes, txNodes, txTrieNodes, rctNodes, tctTrieNodes, logTrieAndLogNodes, logLeafNodeCIDs, rctLeafNodeCIDs, err
}
// processTransactions will take the found transactions in a parsed block body
@@ -181,11 +182,11 @@ func processTransactions(txs []*types.Transaction, expectedTxRoot []byte) ([]*Et
// processReceiptsAndLogs will take in receipts
// to return IPLD node slices for eth-rct, eth-rct-trie, eth-log, eth-log-trie, eth-log-trie-CID, eth-rct-trie-CID
-func processReceiptsAndLogs(rcts []*types.Receipt, expectedRctRoot []byte) ([]*EthReceipt, []*EthRctTrie, [][]*EthLogTrie, [][]cid.Cid, []cid.Cid, error) {
+func processReceiptsAndLogs(rcts []*types.Receipt, expectedRctRoot []byte) ([]*EthReceipt, []*EthRctTrie, [][]node.Node, [][]cid.Cid, []cid.Cid, error) {
// Pre allocating memory.
ethRctNodes := make([]*EthReceipt, 0, len(rcts))
ethLogleafNodeCids := make([][]cid.Cid, 0, len(rcts))
- ethLogTrieNodes := make([][]*EthLogTrie, 0, len(rcts))
+ ethLogTrieAndLogNodes := make([][]node.Node, 0, len(rcts))
receiptTrie := NewRctTrie()
@@ -196,7 +197,7 @@ func processReceiptsAndLogs(rcts []*types.Receipt, expectedRctRoot []byte) ([]*E
return nil, nil, nil, nil, nil, err
}
rct.LogRoot = logTrieHash
- ethLogTrieNodes = append(ethLogTrieNodes, logTrieNodes)
+ ethLogTrieAndLogNodes = append(ethLogTrieAndLogNodes, logTrieNodes)
ethLogleafNodeCids = append(ethLogleafNodeCids, leafNodeCids)
ethRct, err := NewReceipt(rct)
@@ -236,14 +237,14 @@ func processReceiptsAndLogs(rcts []*types.Receipt, expectedRctRoot []byte) ([]*E
ethRctleafNodeCids[idx] = rln.Cid()
}
- return ethRctNodes, rctTrieNodes, ethLogTrieNodes, ethLogleafNodeCids, ethRctleafNodeCids, err
+ return ethRctNodes, rctTrieNodes, ethLogTrieAndLogNodes, ethLogleafNodeCids, ethRctleafNodeCids, err
}
const keccak256Length = 32
-func processLogs(logs []*types.Log) ([]*EthLogTrie, []cid.Cid, common.Hash, error) {
+func processLogs(logs []*types.Log) ([]node.Node, []cid.Cid, common.Hash, error) {
logTr := newLogTrie()
- shortLogCIDs := make(map[uint64]cid.Cid, len(logs))
+ shortLog := make(map[uint64]*EthLog, len(logs))
for idx, log := range logs {
logRaw, err := rlp.EncodeToBytes(log)
if err != nil {
@@ -260,7 +261,7 @@ func processLogs(logs []*types.Log) ([]*EthLogTrie, []cid.Cid, common.Hash, erro
if err != nil {
return nil, nil, common.Hash{}, err
}
- shortLogCIDs[uint64(idx)] = logNode.Cid()
+ shortLog[uint64(idx)] = logNode
}
if err = logTr.Add(idx, logRaw); err != nil {
return nil, nil, common.Hash{}, err
@@ -289,9 +290,11 @@ func processLogs(logs []*types.Log) ([]*EthLogTrie, []cid.Cid, common.Hash, erro
}
// this is where we check which logs <= keccak256Length were actually internalized into parent branch node
// and replace those that were with the cid.Cid for the raw log IPLD
- for idx, lCID := range shortLogCIDs {
- if !leafNodeCids[idx].Defined() {
- leafNodeCids[idx] = lCID
+ for i, l := range shortLog {
+ if !leafNodeCids[i].Defined() {
+ leafNodeCids[i] = l.Cid()
+ // if the leaf node was internalized, we append an IPLD for log itself to the list of IPLDs we need to publish
+ logTrieNodes = append(logTrieNodes, l)
}
}
diff --git a/statediff/indexer/database/file/mainnet_tests/block_12579670.rlp b/statediff/indexer/mainnet_data/block_12579670.rlp
similarity index 100%
rename from statediff/indexer/database/file/mainnet_tests/block_12579670.rlp
rename to statediff/indexer/mainnet_data/block_12579670.rlp
diff --git a/statediff/indexer/database/file/mainnet_tests/block_12600011.rlp b/statediff/indexer/mainnet_data/block_12600011.rlp
similarity index 100%
rename from statediff/indexer/database/file/mainnet_tests/block_12600011.rlp
rename to statediff/indexer/mainnet_data/block_12600011.rlp
diff --git a/statediff/indexer/database/file/mainnet_tests/block_12619985.rlp b/statediff/indexer/mainnet_data/block_12619985.rlp
similarity index 100%
rename from statediff/indexer/database/file/mainnet_tests/block_12619985.rlp
rename to statediff/indexer/mainnet_data/block_12619985.rlp
diff --git a/statediff/indexer/database/file/mainnet_tests/block_12625121.rlp b/statediff/indexer/mainnet_data/block_12625121.rlp
similarity index 100%
rename from statediff/indexer/database/file/mainnet_tests/block_12625121.rlp
rename to statediff/indexer/mainnet_data/block_12625121.rlp
diff --git a/statediff/indexer/database/file/mainnet_tests/block_12655432.rlp b/statediff/indexer/mainnet_data/block_12655432.rlp
similarity index 100%
rename from statediff/indexer/database/file/mainnet_tests/block_12655432.rlp
rename to statediff/indexer/mainnet_data/block_12655432.rlp
diff --git a/statediff/indexer/database/file/mainnet_tests/block_12914664.rlp b/statediff/indexer/mainnet_data/block_12914664.rlp
similarity index 100%
rename from statediff/indexer/database/file/mainnet_tests/block_12914664.rlp
rename to statediff/indexer/mainnet_data/block_12914664.rlp
diff --git a/statediff/indexer/database/file/mainnet_tests/receipts_12579670.rlp b/statediff/indexer/mainnet_data/receipts_12579670.rlp
similarity index 100%
rename from statediff/indexer/database/file/mainnet_tests/receipts_12579670.rlp
rename to statediff/indexer/mainnet_data/receipts_12579670.rlp
diff --git a/statediff/indexer/database/file/mainnet_tests/receipts_12600011.rlp b/statediff/indexer/mainnet_data/receipts_12600011.rlp
similarity index 100%
rename from statediff/indexer/database/file/mainnet_tests/receipts_12600011.rlp
rename to statediff/indexer/mainnet_data/receipts_12600011.rlp
diff --git a/statediff/indexer/database/file/mainnet_tests/receipts_12619985.rlp b/statediff/indexer/mainnet_data/receipts_12619985.rlp
similarity index 100%
rename from statediff/indexer/database/file/mainnet_tests/receipts_12619985.rlp
rename to statediff/indexer/mainnet_data/receipts_12619985.rlp
diff --git a/statediff/indexer/database/file/mainnet_tests/receipts_12625121.rlp b/statediff/indexer/mainnet_data/receipts_12625121.rlp
similarity index 100%
rename from statediff/indexer/database/file/mainnet_tests/receipts_12625121.rlp
rename to statediff/indexer/mainnet_data/receipts_12625121.rlp
diff --git a/statediff/indexer/database/file/mainnet_tests/receipts_12655432.rlp b/statediff/indexer/mainnet_data/receipts_12655432.rlp
similarity index 100%
rename from statediff/indexer/database/file/mainnet_tests/receipts_12655432.rlp
rename to statediff/indexer/mainnet_data/receipts_12655432.rlp
diff --git a/statediff/indexer/database/file/mainnet_tests/receipts_12914664.rlp b/statediff/indexer/mainnet_data/receipts_12914664.rlp
similarity index 100%
rename from statediff/indexer/database/file/mainnet_tests/receipts_12914664.rlp
rename to statediff/indexer/mainnet_data/receipts_12914664.rlp
diff --git a/statediff/indexer/database/file/mainnet_tests/test_helpers.go b/statediff/indexer/test_helpers/mainnet_test_helpers.go
similarity index 96%
rename from statediff/indexer/database/file/mainnet_tests/test_helpers.go
rename to statediff/indexer/test_helpers/mainnet_test_helpers.go
index 7203649d4..141bb10fd 100644
--- a/statediff/indexer/database/file/mainnet_tests/test_helpers.go
+++ b/statediff/indexer/test_helpers/mainnet_test_helpers.go
@@ -14,7 +14,7 @@
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see .
-package mainnet_tests
+package test_helpers
import (
"context"
@@ -29,8 +29,8 @@ import (
)
const (
- defaultBlockFilePath = "./block"
- defaultReceiptsFilePath = "./receipts"
+ defaultBlockFilePath = "../../../mainnet_data/block"
+ defaultReceiptsFilePath = "../../../mainnet_data/receipts"
)
const (
@@ -38,7 +38,8 @@ const (
TEST_BLOCK_NUMBER = "TEST_BLOCK_NUMBER"
)
-var problemBlocks = []int64{
+// ProblemBlocks list of known problem blocks, with funky edge cases
+var ProblemBlocks = []int64{
12600011,
12619985,
12625121,