statediff/indexer/database/dump/indexer.go

@@ -242,7 +242,7 @@ type processArgs struct {
 	rctTrieNodes    []*ipld2.EthRctTrie
 	txNodes         []*ipld2.EthTx
 	txTrieNodes     []*ipld2.EthTxTrie
-	logTrieNodes    [][]*ipld2.EthLogTrie
+	logTrieNodes    [][]node.Node
 	logLeafNodeCIDs [][]cid.Cid
 	rctLeafNodeCIDs []cid.Cid
 }

statediff/indexer/database/file/indexer.go

@@ -250,7 +250,7 @@ type processArgs struct {
 	rctTrieNodes    []*ipld2.EthRctTrie
 	txNodes         []*ipld2.EthTx
 	txTrieNodes     []*ipld2.EthTxTrie
-	logTrieNodes    [][]*ipld2.EthLogTrie
+	logTrieNodes    [][]node.Node
 	logLeafNodeCIDs [][]cid.Cid
 	rctLeafNodeCIDs []cid.Cid
 }

statediff/indexer/database/file/mainnet_tests/indexer_test.go

@@ -49,21 +49,21 @@ func init() {
 }
 
 func TestPushBlockAndState(t *testing.T) {
-	conf := DefaultTestConfig
-	rawURL := os.Getenv(TEST_RAW_URL)
+	conf := test_helpers.DefaultTestConfig
+	rawURL := os.Getenv(test_helpers.TEST_RAW_URL)
 	if rawURL == "" {
 		fmt.Printf("Warning: no raw url configured for statediffing mainnet tests, will look for local file and"+
-			"then try default endpoint (%s)\r\n", DefaultTestConfig.RawURL)
+			"then try default endpoint (%s)\r\n", test_helpers.DefaultTestConfig.RawURL)
 	} else {
 		conf.RawURL = rawURL
 	}
-	for _, blockNumber := range problemBlocks {
+	for _, blockNumber := range test_helpers.ProblemBlocks {
 		conf.BlockNumber = big.NewInt(blockNumber)
-		tb, trs, err := TestBlockAndReceipts(conf)
+		tb, trs, err := test_helpers.TestBlockAndReceipts(conf)
 		require.NoError(t, err)
 		testPushBlockAndState(t, tb, trs)
 	}
-	testBlock, testReceipts, err := TestBlockAndReceiptsFromEnv(conf)
+	testBlock, testReceipts, err := test_helpers.TestBlockAndReceiptsFromEnv(conf)
 	require.NoError(t, err)
 	testPushBlockAndState(t, testBlock, testReceipts)
 }

statediff/indexer/database/sql/indexer.go

@@ -297,7 +297,7 @@ type processArgs struct {
 	rctTrieNodes    []*ipld2.EthRctTrie
 	txNodes         []*ipld2.EthTx
 	txTrieNodes     []*ipld2.EthTxTrie
-	logTrieNodes    [][]*ipld2.EthLogTrie
+	logTrieNodes    [][]node.Node
 	logLeafNodeCIDs [][]cid.Cid
 	rctLeafNodeCIDs []cid.Cid
 }

statediff/indexer/database/sql/mainnet_tests/indexer_test.go

@@ -0,0 +1,109 @@
+// VulcanizeDB
+// Copyright © 2021 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package mainnet_tests
+
+import (
+	"context"
+	"fmt"
+	"math/big"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
+	"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
+	"github.com/ethereum/go-ethereum/statediff/indexer/mocks"
+	"github.com/ethereum/go-ethereum/statediff/indexer/test_helpers"
+)
+
+var (
+	err       error
+	db        sql.Database
+	ind       interfaces.StateDiffIndexer
+	chainConf = params.MainnetChainConfig
+)
+
+func init() {
+	if os.Getenv("MODE") != "statediff" {
+		fmt.Println("Skipping statediff test")
+		os.Exit(0)
+	}
+}
+
+func TestPushBlockAndState(t *testing.T) {
+	conf := test_helpers.DefaultTestConfig
+	rawURL := os.Getenv(test_helpers.TEST_RAW_URL)
+	if rawURL == "" {
+		fmt.Printf("Warning: no raw url configured for statediffing mainnet tests, will look for local file and"+
+			"then try default endpoint (%s)\r\n", test_helpers.DefaultTestConfig.RawURL)
+	} else {
+		conf.RawURL = rawURL
+	}
+	for _, blockNumber := range test_helpers.ProblemBlocks {
+		conf.BlockNumber = big.NewInt(blockNumber)
+		tb, trs, err := test_helpers.TestBlockAndReceipts(conf)
+		require.NoError(t, err)
+		testPushBlockAndState(t, tb, trs)
+	}
+	testBlock, testReceipts, err := test_helpers.TestBlockAndReceiptsFromEnv(conf)
+	require.NoError(t, err)
+	testPushBlockAndState(t, testBlock, testReceipts)
+}
+
+func testPushBlockAndState(t *testing.T, block *types.Block, receipts types.Receipts) {
+	t.Run("Test PushBlock and PushStateNode", func(t *testing.T) {
+		setup(t, block, receipts)
+		tearDown(t)
+	})
+}
+
+func setup(t *testing.T, testBlock *types.Block, testReceipts types.Receipts) {
+	db, err = postgres.SetupSQLXDB()
+	if err != nil {
+		t.Fatal(err)
+	}
+	ind, err = sql.NewStateDiffIndexer(context.Background(), chainConf, db)
+	require.NoError(t, err)
+	var tx interfaces.Batch
+	tx, err = ind.PushBlock(
+		testBlock,
+		testReceipts,
+		testBlock.Difficulty())
+	require.NoError(t, err)
+
+	defer func() {
+		if err := tx.Submit(err); err != nil {
+			t.Fatal(err)
+		}
+	}()
+	for _, node := range mocks.StateDiffs {
+		err = ind.PushStateNode(tx, node, testBlock.Hash().String())
+		require.NoError(t, err)
+	}
+
+	test_helpers.ExpectEqual(t, tx.(*sql.BatchTx).BlockNumber, testBlock.Number().Uint64())
+}
+
+func tearDown(t *testing.T) {
+	sql.TearDownDB(t, db)
+	err = ind.Close()
+	require.NoError(t, err)
+}

statediff/indexer/database/sql/postgres/config.go

@@ -49,9 +49,9 @@ func ResolveDriverType(str string) (DriverType, error) {
 var DefaultConfig = Config{
 	Hostname:     "localhost",
 	Port:         5432,
-	DatabaseName: "vulcanize_test",
-	Username:     "postgres",
-	Password:     "",
+	DatabaseName: "vulcanize_public",
+	Username:     "vdbm",
+	Password:     "password",
 }
 
 // Config holds params for a Postgres db

statediff/indexer/ipld/eth_log_trie.go

@@ -3,6 +3,8 @@ package ipld
 import (
 	"fmt"
 
+	node "github.com/ipfs/go-ipld-format"
+
 	"github.com/ipfs/go-cid"
 	"github.com/multiformats/go-multihash"
 
@@ -92,13 +94,13 @@ func newLogTrie() *logTrie {
 // getNodes invokes the localTrie, which computes the root hash of the
 // log trie and returns its sql keys, to return a slice
 // of EthLogTrie nodes.
-func (rt *logTrie) getNodes() ([]*EthLogTrie, error) {
+func (rt *logTrie) getNodes() ([]node.Node, error) {
 	keys, err := rt.getKeys()
 	if err != nil {
 		return nil, err
 	}
 
-	out := make([]*EthLogTrie, 0, len(keys))
+	out := make([]node.Node, 0, len(keys))
 	for _, k := range keys {
 		n, err := rt.getNodeFromDB(k)
 		if err != nil {

statediff/indexer/ipld/eth_parser.go

@@ -24,6 +24,7 @@ import (
 	"io/ioutil"
 
 	"github.com/ipfs/go-cid"
+	node "github.com/ipfs/go-ipld-format"
 	"github.com/multiformats/go-multihash"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -124,7 +125,7 @@ func FromBlockJSON(r io.Reader) (*EthHeader, []*EthTx, []*EthTxTrie, error) {
 
 // FromBlockAndReceipts takes a block and processes it
 // to return it a set of IPLD nodes for further processing.
-func FromBlockAndReceipts(block *types.Block, receipts []*types.Receipt) (*EthHeader, []*EthHeader, []*EthTx, []*EthTxTrie, []*EthReceipt, []*EthRctTrie, [][]*EthLogTrie, [][]cid.Cid, []cid.Cid, error) {
+func FromBlockAndReceipts(block *types.Block, receipts []*types.Receipt) (*EthHeader, []*EthHeader, []*EthTx, []*EthTxTrie, []*EthReceipt, []*EthRctTrie, [][]node.Node, [][]cid.Cid, []cid.Cid, error) {
 	// Process the header
 	headerNode, err := NewEthHeader(block.Header())
 	if err != nil {
@@ -149,10 +150,10 @@ func FromBlockAndReceipts(block *types.Block, receipts []*types.Receipt) (*EthHe
 	}
 
 	// Process the receipts and logs
-	rctNodes, tctTrieNodes, logTrieNodes, logLeafNodeCIDs, rctLeafNodeCIDs, err := processReceiptsAndLogs(receipts,
+	rctNodes, tctTrieNodes, logTrieAndLogNodes, logLeafNodeCIDs, rctLeafNodeCIDs, err := processReceiptsAndLogs(receipts,
 		block.Header().ReceiptHash[:])
 
-	return headerNode, uncleNodes, txNodes, txTrieNodes, rctNodes, tctTrieNodes, logTrieNodes, logLeafNodeCIDs, rctLeafNodeCIDs, err
+	return headerNode, uncleNodes, txNodes, txTrieNodes, rctNodes, tctTrieNodes, logTrieAndLogNodes, logLeafNodeCIDs, rctLeafNodeCIDs, err
 }
 
 // processTransactions will take the found transactions in a parsed block body
@@ -181,11 +182,11 @@ func processTransactions(txs []*types.Transaction, expectedTxRoot []byte) ([]*Et
 
 // processReceiptsAndLogs will take in receipts
 // to return IPLD node slices for eth-rct, eth-rct-trie, eth-log, eth-log-trie, eth-log-trie-CID, eth-rct-trie-CID
-func processReceiptsAndLogs(rcts []*types.Receipt, expectedRctRoot []byte) ([]*EthReceipt, []*EthRctTrie, [][]*EthLogTrie, [][]cid.Cid, []cid.Cid, error) {
+func processReceiptsAndLogs(rcts []*types.Receipt, expectedRctRoot []byte) ([]*EthReceipt, []*EthRctTrie, [][]node.Node, [][]cid.Cid, []cid.Cid, error) {
 	// Pre allocating memory.
 	ethRctNodes := make([]*EthReceipt, 0, len(rcts))
 	ethLogleafNodeCids := make([][]cid.Cid, 0, len(rcts))
-	ethLogTrieNodes := make([][]*EthLogTrie, 0, len(rcts))
+	ethLogTrieAndLogNodes := make([][]node.Node, 0, len(rcts))
 
 	receiptTrie := NewRctTrie()
 
@@ -196,7 +197,7 @@ func processReceiptsAndLogs(rcts []*types.Receipt, expectedRctRoot []byte) ([]*E
 			return nil, nil, nil, nil, nil, err
 		}
 		rct.LogRoot = logTrieHash
-		ethLogTrieNodes = append(ethLogTrieNodes, logTrieNodes)
+		ethLogTrieAndLogNodes = append(ethLogTrieAndLogNodes, logTrieNodes)
 		ethLogleafNodeCids = append(ethLogleafNodeCids, leafNodeCids)
 
 		ethRct, err := NewReceipt(rct)
@@ -236,14 +237,14 @@ func processReceiptsAndLogs(rcts []*types.Receipt, expectedRctRoot []byte) ([]*E
 		ethRctleafNodeCids[idx] = rln.Cid()
 	}
 
-	return ethRctNodes, rctTrieNodes, ethLogTrieNodes, ethLogleafNodeCids, ethRctleafNodeCids, err
+	return ethRctNodes, rctTrieNodes, ethLogTrieAndLogNodes, ethLogleafNodeCids, ethRctleafNodeCids, err
 }
 
 const keccak256Length = 32
 
-func processLogs(logs []*types.Log) ([]*EthLogTrie, []cid.Cid, common.Hash, error) {
+func processLogs(logs []*types.Log) ([]node.Node, []cid.Cid, common.Hash, error) {
 	logTr := newLogTrie()
-	shortLogCIDs := make(map[uint64]cid.Cid, len(logs))
+	shortLog := make(map[uint64]*EthLog, len(logs))
 	for idx, log := range logs {
 		logRaw, err := rlp.EncodeToBytes(log)
 		if err != nil {
@@ -260,7 +261,7 @@ func processLogs(logs []*types.Log) ([]*EthLogTrie, []cid.Cid, common.Hash, erro
 			if err != nil {
 				return nil, nil, common.Hash{}, err
 			}
-			shortLogCIDs[uint64(idx)] = logNode.Cid()
+			shortLog[uint64(idx)] = logNode
 		}
 		if err = logTr.Add(idx, logRaw); err != nil {
 			return nil, nil, common.Hash{}, err
@@ -289,9 +290,11 @@ func processLogs(logs []*types.Log) ([]*EthLogTrie, []cid.Cid, common.Hash, erro
 	}
 	// this is where we check which logs <= keccak256Length were actually internalized into parent branch node
 	// and replace those that were with the cid.Cid for the raw log IPLD
-	for idx, lCID := range shortLogCIDs {
-		if !leafNodeCids[idx].Defined() {
-			leafNodeCids[idx] = lCID
+	for i, l := range shortLog {
+		if !leafNodeCids[i].Defined() {
+			leafNodeCids[i] = l.Cid()
+			// if the leaf node was internalized, we append an IPLD for log itself to the list of IPLDs we need to publish
+			logTrieNodes = append(logTrieNodes, l)
 		}
 	}
 

statediff/indexer/test_helpers/mainnet_test_helpers.go

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package mainnet_tests
+package test_helpers
 
 import (
 	"context"
@@ -29,8 +29,8 @@ import (
 )
 
 const (
-	defaultBlockFilePath    = "./block"
-	defaultReceiptsFilePath = "./receipts"
+	defaultBlockFilePath    = "../../../mainnet_data/block"
+	defaultReceiptsFilePath = "../../../mainnet_data/receipts"
 )
 
 const (
@@ -38,7 +38,8 @@ const (
 	TEST_BLOCK_NUMBER = "TEST_BLOCK_NUMBER"
 )
 
-var problemBlocks = []int64{
+// ProblemBlocks list of known problem blocks, with funky edge cases
+var ProblemBlocks = []int64{
 	12600011,
 	12619985,
 	12625121,