cerc-io/go-ethereum
13
0
Fork 0
Code Issues 9 Pull Requests Packages Projects Releases 66 Wiki Activity

update sql indexer to use new v4 schema #211

Merged
telackey merged 3 commits from v1.10.16-statediff-4.0.0 into v1.10.16-statediff-v4 2022-03-24 15:10:35 +00:00
Conversation 0 Commits 3 Files Changed 19 +673 -577
19 changed files with 673 additions and 577 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
statediff/indexer/database/file/batch_tx.go
View File

@ -18,7 +18,7 @@ package file
// BatchTx wraps a void with the state necessary for building the tx concurrently during trie difference iteration // BatchTx wraps a void with the state necessary for building the tx concurrently during trie difference iteration
type BatchTx struct { type BatchTx struct {
BlockNumber uint64 BlockNumber string
submit func(blockTx *BatchTx, err error) error submit func(blockTx *BatchTx, err error) error
} }

162
statediff/indexer/database/file/indexer.go
View File

@ -23,6 +23,7 @@ import (
"math/big" "math/big"
"os" "os"
"sync" "sync"
"sync/atomic"
"time" "time"
"github.com/ipfs/go-cid" "github.com/ipfs/go-cid"
@ -53,10 +54,12 @@ var (
// StateDiffIndexer satisfies the indexer.StateDiffIndexer interface for ethereum statediff objects on top of a void // StateDiffIndexer satisfies the indexer.StateDiffIndexer interface for ethereum statediff objects on top of a void
type StateDiffIndexer struct { type StateDiffIndexer struct {
fileWriter *SQLWriter fileWriter *SQLWriter
chainConfig *params.ChainConfig chainConfig *params.ChainConfig
nodeID string nodeID string
wg *sync.WaitGroup wg *sync.WaitGroup
blockNumber string
removedCacheFlag *uint32
} }
// NewStateDiffIndexer creates a void implementation of interfaces.StateDiffIndexer // NewStateDiffIndexer creates a void implementation of interfaces.StateDiffIndexer
@ -77,7 +80,6 @@ func NewStateDiffIndexer(ctx context.Context, chainConfig *params.ChainConfig, c
wg := new(sync.WaitGroup) wg := new(sync.WaitGroup)
w.Loop() w.Loop()
w.upsertNode(config.NodeInfo) w.upsertNode(config.NodeInfo)
w.upsertIPLDDirect(shared.RemovedNodeMhKey, []byte{})
return &StateDiffIndexer{ return &StateDiffIndexer{
fileWriter: w, fileWriter: w,
chainConfig: chainConfig, chainConfig: chainConfig,
@ -92,6 +94,8 @@ func (sdi *StateDiffIndexer) ReportDBMetrics(time.Duration, <-chan bool) {}
// PushBlock pushes and indexes block data in sql, except state & storage nodes (includes header, uncles, transactions & receipts) // PushBlock pushes and indexes block data in sql, except state & storage nodes (includes header, uncles, transactions & receipts)
// Returns an initiated DB transaction which must be Closed via defer to commit or rollback // Returns an initiated DB transaction which must be Closed via defer to commit or rollback
func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receipts, totalDifficulty *big.Int) (interfaces.Batch, error) { func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receipts, totalDifficulty *big.Int) (interfaces.Batch, error) {
sdi.removedCacheFlag = new(uint32)
sdi.blockNumber = block.Number().String()
start, t := time.Now(), time.Now() start, t := time.Now(), time.Now()
blockHash := block.Hash() blockHash := block.Hash()
blockHashStr := blockHash.String() blockHashStr := blockHash.String()
@ -127,7 +131,7 @@ func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receip
t = time.Now() t = time.Now()
blockTx := &BatchTx{ blockTx := &BatchTx{
BlockNumber: height, BlockNumber: sdi.blockNumber,
submit: func(self *BatchTx, err error) error { submit: func(self *BatchTx, err error) error {
tDiff := time.Since(t) tDiff := time.Since(t)
indexerMetrics.tStateStoreCodeProcessing.Update(tDiff) indexerMetrics.tStateStoreCodeProcessing.Update(tDiff)
@ -189,7 +193,7 @@ func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receip
// processHeader write a header IPLD insert SQL stmt to a file // processHeader write a header IPLD insert SQL stmt to a file
// it returns the headerID // it returns the headerID
func (sdi *StateDiffIndexer) processHeader(header *types.Header, headerNode node.Node, reward, td *big.Int) string { func (sdi *StateDiffIndexer) processHeader(header *types.Header, headerNode node.Node, reward, td *big.Int) string {
sdi.fileWriter.upsertIPLDNode(headerNode) sdi.fileWriter.upsertIPLDNode(sdi.blockNumber, headerNode)
var baseFee *string var baseFee *string
if header.BaseFee != nil { if header.BaseFee != nil {
@ -202,7 +206,7 @@ func (sdi *StateDiffIndexer) processHeader(header *types.Header, headerNode node
CID: headerNode.Cid().String(), CID: headerNode.Cid().String(),
MhKey: shared.MultihashKeyFromCID(headerNode.Cid()), MhKey: shared.MultihashKeyFromCID(headerNode.Cid()),
ParentHash: header.ParentHash.String(), ParentHash: header.ParentHash.String(),
BlockNumber: header.Number.String(), BlockNumber: sdi.blockNumber,
BlockHash: headerID, BlockHash: headerID,
TotalDifficulty: td.String(), TotalDifficulty: td.String(),
Reward: reward.String(), Reward: reward.String(),
@ -221,7 +225,7 @@ func (sdi *StateDiffIndexer) processHeader(header *types.Header, headerNode node
func (sdi *StateDiffIndexer) processUncles(headerID string, blockNumber uint64, uncleNodes []*ipld2.EthHeader) { func (sdi *StateDiffIndexer) processUncles(headerID string, blockNumber uint64, uncleNodes []*ipld2.EthHeader) {
// publish and index uncles // publish and index uncles
for _, uncleNode := range uncleNodes { for _, uncleNode := range uncleNodes {
sdi.fileWriter.upsertIPLDNode(uncleNode) sdi.fileWriter.upsertIPLDNode(sdi.blockNumber, uncleNode)
var uncleReward *big.Int var uncleReward *big.Int
// in PoA networks uncle reward is 0 // in PoA networks uncle reward is 0
if sdi.chainConfig.Clique != nil { if sdi.chainConfig.Clique != nil {
@ -230,12 +234,13 @@ func (sdi *StateDiffIndexer) processUncles(headerID string, blockNumber uint64,
uncleReward = shared.CalcUncleMinerReward(blockNumber, uncleNode.Number.Uint64()) uncleReward = shared.CalcUncleMinerReward(blockNumber, uncleNode.Number.Uint64())
} }
sdi.fileWriter.upsertUncleCID(models.UncleModel{ sdi.fileWriter.upsertUncleCID(models.UncleModel{
HeaderID: headerID, BlockNumber: sdi.blockNumber,
CID: uncleNode.Cid().String(), HeaderID: headerID,
MhKey: shared.MultihashKeyFromCID(uncleNode.Cid()), CID: uncleNode.Cid().String(),
ParentHash: uncleNode.ParentHash.String(), MhKey: shared.MultihashKeyFromCID(uncleNode.Cid()),
BlockHash: uncleNode.Hash().String(), ParentHash: uncleNode.ParentHash.String(),
Reward: uncleReward.String(), BlockHash: uncleNode.Hash().String(),
Reward: uncleReward.String(),
}) })
} }
} }
@ -261,10 +266,10 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(args processArgs) error {
signer := types.MakeSigner(sdi.chainConfig, args.blockNumber) signer := types.MakeSigner(sdi.chainConfig, args.blockNumber)
for i, receipt := range args.receipts { for i, receipt := range args.receipts {
for _, logTrieNode := range args.logTrieNodes[i] { for _, logTrieNode := range args.logTrieNodes[i] {
sdi.fileWriter.upsertIPLDNode(logTrieNode) sdi.fileWriter.upsertIPLDNode(sdi.blockNumber, logTrieNode)
} }
txNode := args.txNodes[i] txNode := args.txNodes[i]
sdi.fileWriter.upsertIPLDNode(txNode) sdi.fileWriter.upsertIPLDNode(sdi.blockNumber, txNode)
// index tx // index tx
trx := args.txs[i] trx := args.txs[i]
@ -281,16 +286,17 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(args processArgs) error {
return fmt.Errorf("error deriving tx sender: %v", err) return fmt.Errorf("error deriving tx sender: %v", err)
} }
txModel := models.TxModel{ txModel := models.TxModel{
HeaderID: args.headerID, BlockNumber: sdi.blockNumber,
Dst: shared.HandleZeroAddrPointer(trx.To()), HeaderID: args.headerID,
Src: shared.HandleZeroAddr(from), Dst: shared.HandleZeroAddrPointer(trx.To()),
TxHash: txID, Src: shared.HandleZeroAddr(from),
Index: int64(i), TxHash: txID,
Data: trx.Data(), Index: int64(i),
CID: txNode.Cid().String(), Data: trx.Data(),
MhKey: shared.MultihashKeyFromCID(txNode.Cid()), CID: txNode.Cid().String(),
Type: trx.Type(), MhKey: shared.MultihashKeyFromCID(txNode.Cid()),
Value: val, Type: trx.Type(),
Value: val,
} }
sdi.fileWriter.upsertTransactionCID(txModel) sdi.fileWriter.upsertTransactionCID(txModel)
@ -301,6 +307,7 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(args processArgs) error {
storageKeys[k] = storageKey.Hex() storageKeys[k] = storageKey.Hex()
} }
accessListElementModel := models.AccessListElementModel{ accessListElementModel := models.AccessListElementModel{
BlockNumber: sdi.blockNumber,
TxID: txID, TxID: txID,
Index: int64(j), Index: int64(j),
Address: accessListElement.Address.Hex(), Address: accessListElement.Address.Hex(),
@ -322,6 +329,7 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(args processArgs) error {
} }
rctModel := &models.ReceiptModel{ rctModel := &models.ReceiptModel{
BlockNumber: sdi.blockNumber,
TxID: txID, TxID: txID,
Contract: contract, Contract: contract,
ContractHash: contractHash, ContractHash: contractHash,
@ -349,16 +357,17 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(args processArgs) error {
} }
logDataSet[idx] = &models.LogsModel{ logDataSet[idx] = &models.LogsModel{
ReceiptID: txID, BlockNumber: sdi.blockNumber,
Address: l.Address.String(), ReceiptID: txID,
Index: int64(l.Index), Address: l.Address.String(),
Data: l.Data, Index: int64(l.Index),
LeafCID: args.logLeafNodeCIDs[i][idx].String(), Data: l.Data,
LeafMhKey: shared.MultihashKeyFromCID(args.logLeafNodeCIDs[i][idx]), LeafCID: args.logLeafNodeCIDs[i][idx].String(),
Topic0: topicSet[0], LeafMhKey: shared.MultihashKeyFromCID(args.logLeafNodeCIDs[i][idx]),
Topic1: topicSet[1], Topic0: topicSet[0],
Topic2: topicSet[2], Topic1: topicSet[1],
Topic3: topicSet[3], Topic2: topicSet[2],
Topic3: topicSet[3],
} }
} }
sdi.fileWriter.upsertLogCID(logDataSet) sdi.fileWriter.upsertLogCID(logDataSet)
@ -366,8 +375,8 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(args processArgs) error {
// publish trie nodes, these aren't indexed directly // publish trie nodes, these aren't indexed directly
for i, n := range args.txTrieNodes { for i, n := range args.txTrieNodes {
sdi.fileWriter.upsertIPLDNode(n) sdi.fileWriter.upsertIPLDNode(sdi.blockNumber, n)
sdi.fileWriter.upsertIPLDNode(args.rctTrieNodes[i]) sdi.fileWriter.upsertIPLDNode(sdi.blockNumber, args.rctTrieNodes[i])
} }
return nil return nil
@ -377,30 +386,34 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(args processArgs) error {
func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdtypes.StateNode, headerID string) error { func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdtypes.StateNode, headerID string) error {
// publish the state node // publish the state node
if stateNode.NodeType == sdtypes.Removed { if stateNode.NodeType == sdtypes.Removed {
// short circuit if it is a Removed node if atomic.LoadUint32(sdi.removedCacheFlag) == 0 {
// this assumes the db has been initialized and a public.blocks entry for the Removed node is present atomic.StoreUint32(sdi.removedCacheFlag, 1)
sdi.fileWriter.upsertIPLDDirect(sdi.blockNumber, shared.RemovedNodeMhKey, []byte{})
}
stateModel := models.StateNodeModel{ stateModel := models.StateNodeModel{
HeaderID: headerID, BlockNumber: sdi.blockNumber,
Path: stateNode.Path, HeaderID: headerID,
StateKey: common.BytesToHash(stateNode.LeafKey).String(), Path: stateNode.Path,
CID: shared.RemovedNodeStateCID, StateKey: common.BytesToHash(stateNode.LeafKey).String(),
MhKey: shared.RemovedNodeMhKey, CID: shared.RemovedNodeStateCID,
NodeType: stateNode.NodeType.Int(), MhKey: shared.RemovedNodeMhKey,
NodeType: stateNode.NodeType.Int(),
} }
sdi.fileWriter.upsertStateCID(stateModel) sdi.fileWriter.upsertStateCID(stateModel)
return nil return nil
} }
stateCIDStr, stateMhKey, err := sdi.fileWriter.upsertIPLDRaw(ipld2.MEthStateTrie, multihash.KECCAK_256, stateNode.NodeValue) stateCIDStr, stateMhKey, err := sdi.fileWriter.upsertIPLDRaw(sdi.blockNumber, ipld2.MEthStateTrie, multihash.KECCAK_256, stateNode.NodeValue)
if err != nil { if err != nil {
return fmt.Errorf("error generating and cacheing state node IPLD: %v", err) return fmt.Errorf("error generating and cacheing state node IPLD: %v", err)
} }
stateModel := models.StateNodeModel{ stateModel := models.StateNodeModel{
HeaderID: headerID, BlockNumber: sdi.blockNumber,
Path: stateNode.Path, HeaderID: headerID,
StateKey: common.BytesToHash(stateNode.LeafKey).String(), Path: stateNode.Path,
CID: stateCIDStr, StateKey: common.BytesToHash(stateNode.LeafKey).String(),
MhKey: stateMhKey, CID: stateCIDStr,
NodeType: stateNode.NodeType.Int(), MhKey: stateMhKey,
NodeType: stateNode.NodeType.Int(),
} }
// index the state node // index the state node
sdi.fileWriter.upsertStateCID(stateModel) sdi.fileWriter.upsertStateCID(stateModel)
@ -418,6 +431,7 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt
return fmt.Errorf("error decoding state account rlp: %s", err.Error()) return fmt.Errorf("error decoding state account rlp: %s", err.Error())
} }
accountModel := models.StateAccountModel{ accountModel := models.StateAccountModel{
BlockNumber: sdi.blockNumber,
HeaderID: headerID, HeaderID: headerID,
StatePath: stateNode.Path, StatePath: stateNode.Path,
Balance: account.Balance.String(), Balance: account.Balance.String(),
@ -430,32 +444,36 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt
// if there are any storage nodes associated with this node, publish and index them // if there are any storage nodes associated with this node, publish and index them
for _, storageNode := range stateNode.StorageNodes { for _, storageNode := range stateNode.StorageNodes {
if storageNode.NodeType == sdtypes.Removed { if storageNode.NodeType == sdtypes.Removed {
// short circuit if it is a Removed node if atomic.LoadUint32(sdi.removedCacheFlag) == 0 {
// this assumes the db has been initialized and a public.blocks entry for the Removed node is present atomic.StoreUint32(sdi.removedCacheFlag, 1)
sdi.fileWriter.upsertIPLDDirect(sdi.blockNumber, shared.RemovedNodeMhKey, []byte{})
}
storageModel := models.StorageNodeModel{ storageModel := models.StorageNodeModel{
HeaderID: headerID, BlockNumber: sdi.blockNumber,
StatePath: stateNode.Path, HeaderID: headerID,
Path: storageNode.Path, StatePath: stateNode.Path,
StorageKey: common.BytesToHash(storageNode.LeafKey).String(), Path: storageNode.Path,
CID: shared.RemovedNodeStorageCID, StorageKey: common.BytesToHash(storageNode.LeafKey).String(),
MhKey: shared.RemovedNodeMhKey, CID: shared.RemovedNodeStorageCID,
NodeType: storageNode.NodeType.Int(), MhKey: shared.RemovedNodeMhKey,
NodeType: storageNode.NodeType.Int(),
} }
sdi.fileWriter.upsertStorageCID(storageModel) sdi.fileWriter.upsertStorageCID(storageModel)
continue continue
} }
storageCIDStr, storageMhKey, err := sdi.fileWriter.upsertIPLDRaw(ipld2.MEthStorageTrie, multihash.KECCAK_256, storageNode.NodeValue) storageCIDStr, storageMhKey, err := sdi.fileWriter.upsertIPLDRaw(sdi.blockNumber, ipld2.MEthStorageTrie, multihash.KECCAK_256, storageNode.NodeValue)
if err != nil { if err != nil {
return fmt.Errorf("error generating and cacheing storage node IPLD: %v", err) return fmt.Errorf("error generating and cacheing storage node IPLD: %v", err)
} }
storageModel := models.StorageNodeModel{ storageModel := models.StorageNodeModel{
HeaderID: headerID, BlockNumber: sdi.blockNumber,
StatePath: stateNode.Path, HeaderID: headerID,
Path: storageNode.Path, StatePath: stateNode.Path,
StorageKey: common.BytesToHash(storageNode.LeafKey).String(), Path: storageNode.Path,
CID: storageCIDStr, StorageKey: common.BytesToHash(storageNode.LeafKey).String(),
MhKey: storageMhKey, CID: storageCIDStr,
NodeType: storageNode.NodeType.Int(), MhKey: storageMhKey,
NodeType: storageNode.NodeType.Int(),
} }
sdi.fileWriter.upsertStorageCID(storageModel) sdi.fileWriter.upsertStorageCID(storageModel)
} }
@ -470,7 +488,7 @@ func (sdi *StateDiffIndexer) PushCodeAndCodeHash(batch interfaces.Batch, codeAnd
if err != nil { if err != nil {
return fmt.Errorf("error deriving multihash key from codehash: %v", err) return fmt.Errorf("error deriving multihash key from codehash: %v", err)
} }
sdi.fileWriter.upsertIPLDDirect(mhKey, codeAndCodeHash.Code) sdi.fileWriter.upsertIPLDDirect(sdi.blockNumber, mhKey, codeAndCodeHash.Code)
return nil return nil
} }

11
statediff/indexer/database/file/indexer_legacy_test.go
View File

@ -33,7 +33,6 @@ import (
"github.com/ethereum/go-ethereum/statediff/indexer/interfaces" "github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
"github.com/ethereum/go-ethereum/statediff/indexer/ipld" "github.com/ethereum/go-ethereum/statediff/indexer/ipld"
"github.com/ethereum/go-ethereum/statediff/indexer/mocks" "github.com/ethereum/go-ethereum/statediff/indexer/mocks"
"github.com/ethereum/go-ethereum/statediff/indexer/test_helpers"
) )
var ( var (
@ -71,7 +70,7 @@ func setupLegacy(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
} }
test_helpers.ExpectEqual(t, tx.(*file.BatchTx).BlockNumber, legacyData.BlockNumber.Uint64()) require.Equal(t, legacyData.BlockNumber.String(), tx.(*file.BatchTx).BlockNumber)
connStr := postgres.DefaultConfig.DbConnectionString() connStr := postgres.DefaultConfig.DbConnectionString()
@ -123,10 +122,10 @@ func TestFileIndexerLegacy(t *testing.T) {
err = sqlxdb.QueryRowx(pgStr, legacyData.BlockNumber.Uint64()).StructScan(header) err = sqlxdb.QueryRowx(pgStr, legacyData.BlockNumber.Uint64()).StructScan(header)
require.NoError(t, err) require.NoError(t, err)
test_helpers.ExpectEqual(t, header.CID, legacyHeaderCID.String()) require.Equal(t, legacyHeaderCID.String(), header.CID)
test_helpers.ExpectEqual(t, header.TD, legacyData.MockBlock.Difficulty().String()) require.Equal(t, legacyData.MockBlock.Difficulty().String(), header.TD)
test_helpers.ExpectEqual(t, header.Reward, "5000000000000011250") require.Equal(t, "5000000000000011250", header.Reward)
test_helpers.ExpectEqual(t, header.Coinbase, legacyData.MockBlock.Coinbase().String()) require.Equal(t, legacyData.MockBlock.Coinbase().String(), header.Coinbase)
require.Nil(t, legacyData.MockHeader.BaseFee) require.Nil(t, legacyData.MockHeader.BaseFee)
}) })
} }

152
statediff/indexer/database/file/indexer_test.go
View File

@ -191,7 +191,7 @@ func setup(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
} }
test_helpers.ExpectEqual(t, tx.(*file.BatchTx).BlockNumber, mocks.BlockNumber.Uint64()) require.Equal(t, mocks.BlockNumber.String(), tx.(*file.BatchTx).BlockNumber)
connStr := postgres.DefaultConfig.DbConnectionString() connStr := postgres.DefaultConfig.DbConnectionString()
@ -223,10 +223,10 @@ func TestFileIndexer(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
test_helpers.ExpectEqual(t, header.CID, headerCID.String()) require.Equal(t, headerCID.String(), header.CID)
test_helpers.ExpectEqual(t, header.TD, mocks.MockBlock.Difficulty().String()) require.Equal(t, mocks.MockBlock.Difficulty().String(), header.TD)
test_helpers.ExpectEqual(t, header.Reward, "2000000000000021250") require.Equal(t, "2000000000000021250", header.Reward)
test_helpers.ExpectEqual(t, header.Coinbase, mocks.MockHeader.Coinbase.String()) require.Equal(t, mocks.MockHeader.Coinbase.String(), header.Coinbase)
dc, err := cid.Decode(header.CID) dc, err := cid.Decode(header.CID)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@ -238,7 +238,7 @@ func TestFileIndexer(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
test_helpers.ExpectEqual(t, data, mocks.MockHeaderRlp) require.Equal(t, mocks.MockHeaderRlp, data)
}) })
t.Run("Publish and index transaction IPLDs in a single tx", func(t *testing.T) { t.Run("Publish and index transaction IPLDs in a single tx", func(t *testing.T) {
setup(t) setup(t)
@ -253,7 +253,7 @@ func TestFileIndexer(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
test_helpers.ExpectEqual(t, len(trxs), 5) require.Equal(t, 5, len(trxs))
expectTrue(t, test_helpers.ListContainsString(trxs, trx1CID.String())) expectTrue(t, test_helpers.ListContainsString(trxs, trx1CID.String()))
expectTrue(t, test_helpers.ListContainsString(trxs, trx2CID.String())) expectTrue(t, test_helpers.ListContainsString(trxs, trx2CID.String()))
expectTrue(t, test_helpers.ListContainsString(trxs, trx3CID.String())) expectTrue(t, test_helpers.ListContainsString(trxs, trx3CID.String()))
@ -280,7 +280,7 @@ func TestFileIndexer(t *testing.T) {
txTypeAndValueStr := `SELECT tx_type, value FROM eth.transaction_cids WHERE cid = $1` txTypeAndValueStr := `SELECT tx_type, value FROM eth.transaction_cids WHERE cid = $1`
switch c { switch c {
case trx1CID.String(): case trx1CID.String():
test_helpers.ExpectEqual(t, data, tx1) require.Equal(t, tx1, data)
txRes := new(txResult) txRes := new(txResult)
err = sqlxdb.QueryRowx(txTypeAndValueStr, c).StructScan(txRes) err = sqlxdb.QueryRowx(txTypeAndValueStr, c).StructScan(txRes)
if err != nil { if err != nil {
@ -293,7 +293,7 @@ func TestFileIndexer(t *testing.T) {
t.Fatalf("expected tx value %s got %s", transactions[0].Value().String(), txRes.Value) t.Fatalf("expected tx value %s got %s", transactions[0].Value().String(), txRes.Value)
} }
case trx2CID.String(): case trx2CID.String():
test_helpers.ExpectEqual(t, data, tx2) require.Equal(t, tx2, data)
txRes := new(txResult) txRes := new(txResult)
err = sqlxdb.QueryRowx(txTypeAndValueStr, c).StructScan(txRes) err = sqlxdb.QueryRowx(txTypeAndValueStr, c).StructScan(txRes)
if err != nil { if err != nil {
@ -306,7 +306,7 @@ func TestFileIndexer(t *testing.T) {
t.Fatalf("expected tx value %s got %s", transactions[1].Value().String(), txRes.Value) t.Fatalf("expected tx value %s got %s", transactions[1].Value().String(), txRes.Value)
} }
case trx3CID.String(): case trx3CID.String():
test_helpers.ExpectEqual(t, data, tx3) require.Equal(t, tx3, data)
txRes := new(txResult) txRes := new(txResult)
err = sqlxdb.QueryRowx(txTypeAndValueStr, c).StructScan(txRes) err = sqlxdb.QueryRowx(txTypeAndValueStr, c).StructScan(txRes)
if err != nil { if err != nil {
@ -319,7 +319,7 @@ func TestFileIndexer(t *testing.T) {
t.Fatalf("expected tx value %s got %s", transactions[2].Value().String(), txRes.Value) t.Fatalf("expected tx value %s got %s", transactions[2].Value().String(), txRes.Value)
} }
case trx4CID.String(): case trx4CID.String():
test_helpers.ExpectEqual(t, data, tx4) require.Equal(t, tx4, data)
txRes := new(txResult) txRes := new(txResult)
err = sqlxdb.QueryRowx(txTypeAndValueStr, c).StructScan(txRes) err = sqlxdb.QueryRowx(txTypeAndValueStr, c).StructScan(txRes)
if err != nil { if err != nil {
@ -332,7 +332,9 @@ func TestFileIndexer(t *testing.T) {
t.Fatalf("expected tx value %s got %s", transactions[3].Value().String(), txRes.Value) t.Fatalf("expected tx value %s got %s", transactions[3].Value().String(), txRes.Value)
} }
accessListElementModels := make([]models.AccessListElementModel, 0) accessListElementModels := make([]models.AccessListElementModel, 0)
pgStr = `SELECT access_list_elements.* FROM eth.access_list_elements INNER JOIN eth.transaction_cids ON (tx_id = transaction_cids.tx_hash) WHERE cid = $1 ORDER BY access_list_elements.index ASC` pgStr = "SELECT cast(access_list_elements.block_number AS TEXT), access_list_elements.index, access_list_elements.tx_id, " +
"access_list_elements.address, access_list_elements.storage_keys FROM eth.access_list_elements " +
"INNER JOIN eth.transaction_cids ON (tx_id = transaction_cids.tx_hash) WHERE cid = $1 ORDER BY access_list_elements.index ASC"
err = sqlxdb.Select(&accessListElementModels, pgStr, c) err = sqlxdb.Select(&accessListElementModels, pgStr, c)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@ -341,18 +343,20 @@ func TestFileIndexer(t *testing.T) {
t.Fatalf("expected two access list entries, got %d", len(accessListElementModels)) t.Fatalf("expected two access list entries, got %d", len(accessListElementModels))
} }
model1 := models.AccessListElementModel{ model1 := models.AccessListElementModel{
Index: accessListElementModels[0].Index, BlockNumber: mocks.BlockNumber.String(),
Address: accessListElementModels[0].Address, Index: accessListElementModels[0].Index,
Address: accessListElementModels[0].Address,
} }
model2 := models.AccessListElementModel{ model2 := models.AccessListElementModel{
BlockNumber: mocks.BlockNumber.String(),
Index: accessListElementModels[1].Index, Index: accessListElementModels[1].Index,
Address: accessListElementModels[1].Address, Address: accessListElementModels[1].Address,
StorageKeys: accessListElementModels[1].StorageKeys, StorageKeys: accessListElementModels[1].StorageKeys,
} }
test_helpers.ExpectEqual(t, model1, mocks.AccessListEntry1Model) require.Equal(t, mocks.AccessListEntry1Model, model1)
test_helpers.ExpectEqual(t, model2, mocks.AccessListEntry2Model) require.Equal(t, mocks.AccessListEntry2Model, model2)
case trx5CID.String(): case trx5CID.String():
test_helpers.ExpectEqual(t, data, tx5) require.Equal(t, tx5, data)
txRes := new(txResult) txRes := new(txResult)
err = sqlxdb.QueryRowx(txTypeAndValueStr, c).StructScan(txRes) err = sqlxdb.QueryRowx(txTypeAndValueStr, c).StructScan(txRes)
if err != nil { if err != nil {
@ -402,7 +406,7 @@ func TestFileIndexer(t *testing.T) {
// expecting MockLog1 and MockLog2 for mockReceipt4 // expecting MockLog1 and MockLog2 for mockReceipt4
expectedLogs := mocks.MockReceipts[i].Logs expectedLogs := mocks.MockReceipts[i].Logs
test_helpers.ExpectEqual(t, len(results), len(expectedLogs)) require.Equal(t, len(expectedLogs), len(results))
var nodeElements []interface{} var nodeElements []interface{}
for idx, r := range results { for idx, r := range results {
@ -413,12 +417,12 @@ func TestFileIndexer(t *testing.T) {
logRaw, err := rlp.EncodeToBytes(expectedLogs[idx]) logRaw, err := rlp.EncodeToBytes(expectedLogs[idx])
require.NoError(t, err) require.NoError(t, err)
// 2nd element of the leaf node contains the encoded log data. // 2nd element of the leaf node contains the encoded log data.
test_helpers.ExpectEqual(t, logRaw, nodeElements[1].([]byte)) require.Equal(t, nodeElements[1].([]byte), logRaw)
} else { } else {
logRaw, err := rlp.EncodeToBytes(expectedLogs[idx]) logRaw, err := rlp.EncodeToBytes(expectedLogs[idx])
require.NoError(t, err) require.NoError(t, err)
// raw log was IPLDized // raw log was IPLDized
test_helpers.ExpectEqual(t, logRaw, r.Data) require.Equal(t, r.Data, logRaw)
} }
} }
} }
@ -439,7 +443,7 @@ func TestFileIndexer(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
test_helpers.ExpectEqual(t, len(rcts), 5) require.Equal(t, 5, len(rcts))
expectTrue(t, test_helpers.ListContainsString(rcts, rct1CID.String())) expectTrue(t, test_helpers.ListContainsString(rcts, rct1CID.String()))
expectTrue(t, test_helpers.ListContainsString(rcts, rct2CID.String())) expectTrue(t, test_helpers.ListContainsString(rcts, rct2CID.String()))
expectTrue(t, test_helpers.ListContainsString(rcts, rct3CID.String())) expectTrue(t, test_helpers.ListContainsString(rcts, rct3CID.String()))
@ -465,7 +469,7 @@ func TestFileIndexer(t *testing.T) {
expectedRct, err := mocks.MockReceipts[idx].MarshalBinary() expectedRct, err := mocks.MockReceipts[idx].MarshalBinary()
require.NoError(t, err) require.NoError(t, err)
test_helpers.ExpectEqual(t, expectedRct, nodeElements[1].([]byte)) require.Equal(t, expectedRct, nodeElements[1].([]byte))
dc, err := cid.Decode(c) dc, err := cid.Decode(c)
if err != nil { if err != nil {
@ -481,46 +485,46 @@ func TestFileIndexer(t *testing.T) {
postStatePgStr := `SELECT post_state FROM eth.receipt_cids WHERE leaf_cid = $1` postStatePgStr := `SELECT post_state FROM eth.receipt_cids WHERE leaf_cid = $1`
switch c { switch c {
case rct1CID.String(): case rct1CID.String():
test_helpers.ExpectEqual(t, data, rctLeaf1) require.Equal(t, rctLeaf1, data)
var postStatus uint64 var postStatus uint64
pgStr = `SELECT post_status FROM eth.receipt_cids WHERE leaf_cid = $1` pgStr = `SELECT post_status FROM eth.receipt_cids WHERE leaf_cid = $1`
err = sqlxdb.Get(&postStatus, pgStr, c) err = sqlxdb.Get(&postStatus, pgStr, c)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
test_helpers.ExpectEqual(t, postStatus, mocks.ExpectedPostStatus) require.Equal(t, mocks.ExpectedPostStatus, postStatus)
case rct2CID.String(): case rct2CID.String():
test_helpers.ExpectEqual(t, data, rctLeaf2) require.Equal(t, rctLeaf2, data)
var postState string var postState string
err = sqlxdb.Get(&postState, postStatePgStr, c) err = sqlxdb.Get(&postState, postStatePgStr, c)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
test_helpers.ExpectEqual(t, postState, mocks.ExpectedPostState1) require.Equal(t, mocks.ExpectedPostState1, postState)
case rct3CID.String(): case rct3CID.String():
test_helpers.ExpectEqual(t, data, rctLeaf3) require.Equal(t, rctLeaf3, data)
var postState string var postState string
err = sqlxdb.Get(&postState, postStatePgStr, c) err = sqlxdb.Get(&postState, postStatePgStr, c)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
test_helpers.ExpectEqual(t, postState, mocks.ExpectedPostState2) require.Equal(t, mocks.ExpectedPostState2, postState)
case rct4CID.String(): case rct4CID.String():
test_helpers.ExpectEqual(t, data, rctLeaf4) require.Equal(t, rctLeaf4, data)
var postState string var postState string
err = sqlxdb.Get(&postState, postStatePgStr, c) err = sqlxdb.Get(&postState, postStatePgStr, c)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
test_helpers.ExpectEqual(t, postState, mocks.ExpectedPostState3) require.Equal(t, mocks.ExpectedPostState3, postState)
case rct5CID.String(): case rct5CID.String():
test_helpers.ExpectEqual(t, data, rctLeaf5) require.Equal(t, rctLeaf5, data)
var postState string var postState string
err = sqlxdb.Get(&postState, postStatePgStr, c) err = sqlxdb.Get(&postState, postStatePgStr, c)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
test_helpers.ExpectEqual(t, postState, mocks.ExpectedPostState3) require.Equal(t, mocks.ExpectedPostState3, postState)
} }
} }
}) })
@ -539,7 +543,7 @@ func TestFileIndexer(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
test_helpers.ExpectEqual(t, len(stateNodes), 2) require.Equal(t, 2, len(stateNodes))
for _, stateNode := range stateNodes { for _, stateNode := range stateNodes {
var data []byte var data []byte
dc, err := cid.Decode(stateNode.CID) dc, err := cid.Decode(stateNode.CID)
@ -552,39 +556,41 @@ func TestFileIndexer(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
pgStr = `SELECT * from eth.state_accounts WHERE header_id = $1 AND state_path = $2` pgStr = `SELECT cast(block_number AS TEXT), header_id, state_path, cast(balance AS TEXT), nonce, code_hash, storage_root from eth.state_accounts WHERE header_id = $1 AND state_path = $2`
var account models.StateAccountModel var account models.StateAccountModel
err = sqlxdb.Get(&account, pgStr, stateNode.HeaderID, stateNode.Path) err = sqlxdb.Get(&account, pgStr, stateNode.HeaderID, stateNode.Path)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
if stateNode.CID == state1CID.String() { if stateNode.CID == state1CID.String() {
test_helpers.ExpectEqual(t, stateNode.NodeType, 2) require.Equal(t, 2, stateNode.NodeType)
test_helpers.ExpectEqual(t, stateNode.StateKey, common.BytesToHash(mocks.ContractLeafKey).Hex()) require.Equal(t, common.BytesToHash(mocks.ContractLeafKey).Hex(), stateNode.StateKey)
test_helpers.ExpectEqual(t, stateNode.Path, []byte{'\x06'}) require.Equal(t, []byte{'\x06'}, stateNode.Path)
test_helpers.ExpectEqual(t, data, mocks.ContractLeafNode) require.Equal(t, mocks.ContractLeafNode, data)
test_helpers.ExpectEqual(t, account, models.StateAccountModel{ require.Equal(t, models.StateAccountModel{
BlockNumber: mocks.BlockNumber.String(),
HeaderID: account.HeaderID, HeaderID: account.HeaderID,
StatePath: stateNode.Path, StatePath: stateNode.Path,
Balance: "0", Balance: "0",
CodeHash: mocks.ContractCodeHash.Bytes(), CodeHash: mocks.ContractCodeHash.Bytes(),
StorageRoot: mocks.ContractRoot, StorageRoot: mocks.ContractRoot,
Nonce: 1, Nonce: 1,
}) }, account)
} }
if stateNode.CID == state2CID.String() { if stateNode.CID == state2CID.String() {
test_helpers.ExpectEqual(t, stateNode.NodeType, 2) require.Equal(t, 2, stateNode.NodeType)
test_helpers.ExpectEqual(t, stateNode.StateKey, common.BytesToHash(mocks.AccountLeafKey).Hex()) require.Equal(t, common.BytesToHash(mocks.AccountLeafKey).Hex(), stateNode.StateKey)
test_helpers.ExpectEqual(t, stateNode.Path, []byte{'\x0c'}) require.Equal(t, []byte{'\x0c'}, stateNode.Path)
test_helpers.ExpectEqual(t, data, mocks.AccountLeafNode) require.Equal(t, mocks.AccountLeafNode, data)
test_helpers.ExpectEqual(t, account, models.StateAccountModel{ require.Equal(t, models.StateAccountModel{
BlockNumber: mocks.BlockNumber.String(),
HeaderID: account.HeaderID, HeaderID: account.HeaderID,
StatePath: stateNode.Path, StatePath: stateNode.Path,
Balance: "1000", Balance: "1000",
CodeHash: mocks.AccountCodeHash.Bytes(), CodeHash: mocks.AccountCodeHash.Bytes(),
StorageRoot: mocks.AccountRoot, StorageRoot: mocks.AccountRoot,
Nonce: 0, Nonce: 0,
}) }, account)
} }
} }
@ -597,7 +603,7 @@ func TestFileIndexer(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
test_helpers.ExpectEqual(t, len(stateNodes), 1) require.Equal(t, 1, len(stateNodes))
stateNode := stateNodes[0] stateNode := stateNodes[0]
var data []byte var data []byte
dc, err := cid.Decode(stateNode.CID) dc, err := cid.Decode(stateNode.CID)
@ -606,14 +612,14 @@ func TestFileIndexer(t *testing.T) {
} }
mhKey := dshelp.MultihashToDsKey(dc.Hash()) mhKey := dshelp.MultihashToDsKey(dc.Hash())
prefixedKey := blockstore.BlockPrefix.String() + mhKey.String() prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
test_helpers.ExpectEqual(t, prefixedKey, shared.RemovedNodeMhKey) require.Equal(t, shared.RemovedNodeMhKey, prefixedKey)
err = sqlxdb.Get(&data, ipfsPgGet, prefixedKey) err = sqlxdb.Get(&data, ipfsPgGet, prefixedKey)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
test_helpers.ExpectEqual(t, stateNode.CID, shared.RemovedNodeStateCID) require.Equal(t, shared.RemovedNodeStateCID, stateNode.CID)
test_helpers.ExpectEqual(t, stateNode.Path, []byte{'\x02'}) require.Equal(t, []byte{'\x02'}, stateNode.Path)
test_helpers.ExpectEqual(t, data, []byte{}) require.Equal(t, []byte{}, data)
}) })
t.Run("Publish and index storage IPLDs in a single tx", func(t *testing.T) { t.Run("Publish and index storage IPLDs in a single tx", func(t *testing.T) {
@ -623,7 +629,7 @@ func TestFileIndexer(t *testing.T) {
// check that storage nodes were properly indexed // check that storage nodes were properly indexed
storageNodes := make([]models.StorageNodeWithStateKeyModel, 0) storageNodes := make([]models.StorageNodeWithStateKeyModel, 0)
pgStr := `SELECT storage_cids.cid, state_cids.state_leaf_key, storage_cids.storage_leaf_key, storage_cids.node_type, storage_cids.storage_path pgStr := `SELECT cast(storage_cids.block_number AS TEXT), storage_cids.cid, state_cids.state_leaf_key, storage_cids.storage_leaf_key, storage_cids.node_type, storage_cids.storage_path
FROM eth.storage_cids, eth.state_cids, eth.header_cids FROM eth.storage_cids, eth.state_cids, eth.header_cids
WHERE (storage_cids.state_path, storage_cids.header_id) = (state_cids.state_path, state_cids.header_id) WHERE (storage_cids.state_path, storage_cids.header_id) = (state_cids.state_path, state_cids.header_id)
AND state_cids.header_id = header_cids.block_hash AND state_cids.header_id = header_cids.block_hash
@ -633,14 +639,15 @@ func TestFileIndexer(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
test_helpers.ExpectEqual(t, len(storageNodes), 1) require.Equal(t, 1, len(storageNodes))
test_helpers.ExpectEqual(t, storageNodes[0], models.StorageNodeWithStateKeyModel{ require.Equal(t, models.StorageNodeWithStateKeyModel{
CID: storageCID.String(), BlockNumber: mocks.BlockNumber.String(),
NodeType: 2, CID: storageCID.String(),
StorageKey: common.BytesToHash(mocks.StorageLeafKey).Hex(), NodeType: 2,
StateKey: common.BytesToHash(mocks.ContractLeafKey).Hex(), StorageKey: common.BytesToHash(mocks.StorageLeafKey).Hex(),
Path: []byte{}, StateKey: common.BytesToHash(mocks.ContractLeafKey).Hex(),
}) Path: []byte{},
}, storageNodes[0])
var data []byte var data []byte
dc, err := cid.Decode(storageNodes[0].CID) dc, err := cid.Decode(storageNodes[0].CID)
if err != nil { if err != nil {
@ -652,11 +659,11 @@ func TestFileIndexer(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
test_helpers.ExpectEqual(t, data, mocks.StorageLeafNode) require.Equal(t, mocks.StorageLeafNode, data)
// check that Removed storage nodes were properly indexed // check that Removed storage nodes were properly indexed
storageNodes = make([]models.StorageNodeWithStateKeyModel, 0) storageNodes = make([]models.StorageNodeWithStateKeyModel, 0)
pgStr = `SELECT storage_cids.cid, state_cids.state_leaf_key, storage_cids.storage_leaf_key, storage_cids.node_type, storage_cids.storage_path pgStr = `SELECT cast(storage_cids.block_number AS TEXT), storage_cids.cid, state_cids.state_leaf_key, storage_cids.storage_leaf_key, storage_cids.node_type, storage_cids.storage_path
FROM eth.storage_cids, eth.state_cids, eth.header_cids FROM eth.storage_cids, eth.state_cids, eth.header_cids
WHERE (storage_cids.state_path, storage_cids.header_id) = (state_cids.state_path, state_cids.header_id) WHERE (storage_cids.state_path, storage_cids.header_id) = (state_cids.state_path, state_cids.header_id)
AND state_cids.header_id = header_cids.block_hash AND state_cids.header_id = header_cids.block_hash
@ -666,25 +673,26 @@ func TestFileIndexer(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
test_helpers.ExpectEqual(t, len(storageNodes), 1) require.Equal(t, 1, len(storageNodes))
test_helpers.ExpectEqual(t, storageNodes[0], models.StorageNodeWithStateKeyModel{ require.Equal(t, models.StorageNodeWithStateKeyModel{
CID: shared.RemovedNodeStorageCID, BlockNumber: mocks.BlockNumber.String(),
NodeType: 3, CID: shared.RemovedNodeStorageCID,
StorageKey: common.BytesToHash(mocks.RemovedLeafKey).Hex(), NodeType: 3,
StateKey: common.BytesToHash(mocks.ContractLeafKey).Hex(), StorageKey: common.BytesToHash(mocks.RemovedLeafKey).Hex(),
Path: []byte{'\x03'}, StateKey: common.BytesToHash(mocks.ContractLeafKey).Hex(),
}) Path: []byte{'\x03'},
}, storageNodes[0])
dc, err = cid.Decode(storageNodes[0].CID) dc, err = cid.Decode(storageNodes[0].CID)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
mhKey = dshelp.MultihashToDsKey(dc.Hash()) mhKey = dshelp.MultihashToDsKey(dc.Hash())
prefixedKey = blockstore.BlockPrefix.String() + mhKey.String() prefixedKey = blockstore.BlockPrefix.String() + mhKey.String()
test_helpers.ExpectEqual(t, prefixedKey, shared.RemovedNodeMhKey) require.Equal(t, shared.RemovedNodeMhKey, prefixedKey)
err = sqlxdb.Get(&data, ipfsPgGet, prefixedKey) err = sqlxdb.Get(&data, ipfsPgGet, prefixedKey)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
test_helpers.ExpectEqual(t, data, []byte{}) require.Equal(t, []byte{}, data)
}) })
} }

72
statediff/indexer/database/file/writer.go
View File

@ -127,34 +127,35 @@ const (
nodeInsert = "INSERT INTO nodes (genesis_block, network_id, node_id, client_name, chain_id) VALUES " + nodeInsert = "INSERT INTO nodes (genesis_block, network_id, node_id, client_name, chain_id) VALUES " +
"('%s', '%s', '%s', '%s', %d);\n" "('%s', '%s', '%s', '%s', %d);\n"
ipldInsert = "INSERT INTO public.blocks (key, data) VALUES ('%s', '\\x%x');\n" ipldInsert = "INSERT INTO public.blocks (block_number, key, data) VALUES ('%s', '%s', '\\x%x');\n"
headerInsert = "INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_id, reward, " + headerInsert = "INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_id, reward, " +
"state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, mh_key, times_validated, coinbase) VALUES " + "state_root, tx_root, receipt_root, uncle_root, bloom, timestamp, mh_key, times_validated, coinbase) VALUES " +
"('%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '\\x%x', %d, '%s', %d, '%s');\n" "('%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '%s', '\\x%x', %d, '%s', %d, '%s');\n"
uncleInsert = "INSERT INTO eth.uncle_cids (block_hash, header_id, parent_hash, cid, reward, mh_key) VALUES " + uncleInsert = "INSERT INTO eth.uncle_cids (block_number, block_hash, header_id, parent_hash, cid, reward, mh_key) VALUES " +
"('%s', '%s', '%s', '%s', '%s', '%s');\n" "('%s', '%s', '%s', '%s', '%s', '%s', '%s');\n"
txInsert = "INSERT INTO eth.transaction_cids (header_id, tx_hash, cid, dst, src, index, mh_key, tx_data, tx_type, " + txInsert = "INSERT INTO eth.transaction_cids (block_number, header_id, tx_hash, cid, dst, src, index, mh_key, tx_data, tx_type, " +
"value) VALUES ('%s', '%s', '%s', '%s', '%s', %d, '%s', '\\x%x', %d, '%s');\n" "value) VALUES ('%s', '%s', '%s', '%s', '%s', '%s', %d, '%s', '\\x%x', %d, '%s');\n"
alInsert = "INSERT INTO eth.access_list_elements (tx_id, index, address, storage_keys) VALUES ('%s', %d, '%s', '%s');\n" alInsert = "INSERT INTO eth.access_list_elements (block_number, tx_id, index, address, storage_keys) VALUES " +
"('%s', '%s', %d, '%s', '%s');\n"
rctInsert = "INSERT INTO eth.receipt_cids (tx_id, leaf_cid, contract, contract_hash, leaf_mh_key, post_state, " + rctInsert = "INSERT INTO eth.receipt_cids (block_number, tx_id, leaf_cid, contract, contract_hash, leaf_mh_key, post_state, " +
"post_status, log_root) VALUES ('%s', '%s', '%s', '%s', '%s', '%s', %d, '%s');\n" "post_status, log_root) VALUES ('%s', '%s', '%s', '%s', '%s', '%s', '%s', %d, '%s');\n"
logInsert = "INSERT INTO eth.log_cids (leaf_cid, leaf_mh_key, rct_id, address, index, topic0, topic1, topic2, " + logInsert = "INSERT INTO eth.log_cids (block_number, leaf_cid, leaf_mh_key, rct_id, address, index, topic0, topic1, topic2, " +
"topic3, log_data) VALUES ('%s', '%s', '%s', '%s', %d, '%s', '%s', '%s', '%s', '\\x%x');\n" "topic3, log_data) VALUES ('%s', '%s', '%s', '%s', '%s', %d, '%s', '%s', '%s', '%s', '\\x%x');\n"
stateInsert = "INSERT INTO eth.state_cids (header_id, state_leaf_key, cid, state_path, node_type, diff, mh_key) " + stateInsert = "INSERT INTO eth.state_cids (block_number, header_id, state_leaf_key, cid, state_path, node_type, diff, mh_key) " +
"VALUES ('%s', '%s', '%s', '\\x%x', %d, %t, '%s');\n" "VALUES ('%s', '%s', '%s', '%s', '\\x%x', %d, %t, '%s');\n"
accountInsert = "INSERT INTO eth.state_accounts (header_id, state_path, balance, nonce, code_hash, storage_root) " + accountInsert = "INSERT INTO eth.state_accounts (block_number, header_id, state_path, balance, nonce, code_hash, storage_root) " +
"VALUES ('%s', '\\x%x', '%s', %d, '\\x%x', '%s');\n" "VALUES ('%s', '%s', '\\x%x', '%s', %d, '\\x%x', '%s');\n"
storageInsert = "INSERT INTO eth.storage_cids (header_id, state_path, storage_leaf_key, cid, storage_path, " + storageInsert = "INSERT INTO eth.storage_cids (block_number, header_id, state_path, storage_leaf_key, cid, storage_path, " +
"node_type, diff, mh_key) VALUES ('%s', '\\x%x', '%s', '%s', '\\x%x', %d, %t, '%s');\n" "node_type, diff, mh_key) VALUES ('%s', '%s', '\\x%x', '%s', '%s', '\\x%x', %d, %t, '%s');\n"
) )
func (sqw *SQLWriter) upsertNode(node nodeinfo.Info) { func (sqw *SQLWriter) upsertNode(node nodeinfo.Info) {
@ -162,32 +163,35 @@ func (sqw *SQLWriter) upsertNode(node nodeinfo.Info) {
} }
func (sqw *SQLWriter) upsertIPLD(ipld models.IPLDModel) { func (sqw *SQLWriter) upsertIPLD(ipld models.IPLDModel) {
sqw.stmts <- []byte(fmt.Sprintf(ipldInsert, ipld.Key, ipld.Data)) sqw.stmts <- []byte(fmt.Sprintf(ipldInsert, ipld.BlockNumber, ipld.Key, ipld.Data))
} }
func (sqw *SQLWriter) upsertIPLDDirect(key string, value []byte) { func (sqw *SQLWriter) upsertIPLDDirect(blockNumber, key string, value []byte) {
sqw.upsertIPLD(models.IPLDModel{ sqw.upsertIPLD(models.IPLDModel{
Key: key, BlockNumber: blockNumber,
Data: value, Key: key,
Data: value,
}) })
} }
func (sqw *SQLWriter) upsertIPLDNode(i node.Node) { func (sqw *SQLWriter) upsertIPLDNode(blockNumber string, i node.Node) {
sqw.upsertIPLD(models.IPLDModel{ sqw.upsertIPLD(models.IPLDModel{
Key: blockstore.BlockPrefix.String() + dshelp.MultihashToDsKey(i.Cid().Hash()).String(), BlockNumber: blockNumber,
Data: i.RawData(), Key: blockstore.BlockPrefix.String() + dshelp.MultihashToDsKey(i.Cid().Hash()).String(),
Data: i.RawData(),
}) })
} }
func (sqw *SQLWriter) upsertIPLDRaw(codec, mh uint64, raw []byte) (string, string, error) { func (sqw *SQLWriter) upsertIPLDRaw(blockNumber string, codec, mh uint64, raw []byte) (string, string, error) {
c, err := ipld.RawdataToCid(codec, raw, mh) c, err := ipld.RawdataToCid(codec, raw, mh)
if err != nil { if err != nil {
return "", "", err return "", "", err
} }
prefixedKey := blockstore.BlockPrefix.String() + dshelp.MultihashToDsKey(c.Hash()).String() prefixedKey := blockstore.BlockPrefix.String() + dshelp.MultihashToDsKey(c.Hash()).String()
sqw.upsertIPLD(models.IPLDModel{ sqw.upsertIPLD(models.IPLDModel{
Key: prefixedKey, BlockNumber: blockNumber,
Data: raw, Key: prefixedKey,
Data: raw,
}) })
return c.String(), prefixedKey, err return c.String(), prefixedKey, err
} }
@ -201,31 +205,31 @@ func (sqw *SQLWriter) upsertHeaderCID(header models.HeaderModel) {
} }
func (sqw *SQLWriter) upsertUncleCID(uncle models.UncleModel) { func (sqw *SQLWriter) upsertUncleCID(uncle models.UncleModel) {
sqw.stmts <- []byte(fmt.Sprintf(uncleInsert, uncle.BlockHash, uncle.HeaderID, uncle.ParentHash, uncle.CID, sqw.stmts <- []byte(fmt.Sprintf(uncleInsert, uncle.BlockNumber, uncle.BlockHash, uncle.HeaderID, uncle.ParentHash, uncle.CID,
uncle.Reward, uncle.MhKey)) uncle.Reward, uncle.MhKey))
} }
func (sqw *SQLWriter) upsertTransactionCID(transaction models.TxModel) { func (sqw *SQLWriter) upsertTransactionCID(transaction models.TxModel) {
sqw.stmts <- []byte(fmt.Sprintf(txInsert, transaction.HeaderID, transaction.TxHash, transaction.CID, transaction.Dst, sqw.stmts <- []byte(fmt.Sprintf(txInsert, transaction.BlockNumber, transaction.HeaderID, transaction.TxHash, transaction.CID, transaction.Dst,
transaction.Src, transaction.Index, transaction.MhKey, transaction.Data, transaction.Type, transaction.Value)) transaction.Src, transaction.Index, transaction.MhKey, transaction.Data, transaction.Type, transaction.Value))
indexerMetrics.transactions.Inc(1) indexerMetrics.transactions.Inc(1)
} }
func (sqw *SQLWriter) upsertAccessListElement(accessListElement models.AccessListElementModel) { func (sqw *SQLWriter) upsertAccessListElement(accessListElement models.AccessListElementModel) {
sqw.stmts <- []byte(fmt.Sprintf(alInsert, accessListElement.TxID, accessListElement.Index, accessListElement.Address, sqw.stmts <- []byte(fmt.Sprintf(alInsert, accessListElement.BlockNumber, accessListElement.TxID, accessListElement.Index, accessListElement.Address,
formatPostgresStringArray(accessListElement.StorageKeys))) formatPostgresStringArray(accessListElement.StorageKeys)))
indexerMetrics.accessListEntries.Inc(1) indexerMetrics.accessListEntries.Inc(1)
} }
func (sqw *SQLWriter) upsertReceiptCID(rct *models.ReceiptModel) { func (sqw *SQLWriter) upsertReceiptCID(rct *models.ReceiptModel) {
sqw.stmts <- []byte(fmt.Sprintf(rctInsert, rct.TxID, rct.LeafCID, rct.Contract, rct.ContractHash, rct.LeafMhKey, sqw.stmts <- []byte(fmt.Sprintf(rctInsert, rct.BlockNumber, rct.TxID, rct.LeafCID, rct.Contract, rct.ContractHash, rct.LeafMhKey,
rct.PostState, rct.PostStatus, rct.LogRoot)) rct.PostState, rct.PostStatus, rct.LogRoot))
indexerMetrics.receipts.Inc(1) indexerMetrics.receipts.Inc(1)
} }
func (sqw *SQLWriter) upsertLogCID(logs []*models.LogsModel) { func (sqw *SQLWriter) upsertLogCID(logs []*models.LogsModel) {
for _, l := range logs { for _, l := range logs {
sqw.stmts <- []byte(fmt.Sprintf(logInsert, l.LeafCID, l.LeafMhKey, l.ReceiptID, l.Address, l.Index, l.Topic0, sqw.stmts <- []byte(fmt.Sprintf(logInsert, l.BlockNumber, l.LeafCID, l.LeafMhKey, l.ReceiptID, l.Address, l.Index, l.Topic0,
l.Topic1, l.Topic2, l.Topic3, l.Data)) l.Topic1, l.Topic2, l.Topic3, l.Data))
indexerMetrics.logs.Inc(1) indexerMetrics.logs.Inc(1)
} }
@ -236,12 +240,12 @@ func (sqw *SQLWriter) upsertStateCID(stateNode models.StateNodeModel) {
if stateNode.StateKey != nullHash.String() { if stateNode.StateKey != nullHash.String() {
stateKey = stateNode.StateKey stateKey = stateNode.StateKey
} }
sqw.stmts <- []byte(fmt.Sprintf(stateInsert, stateNode.HeaderID, stateKey, stateNode.CID, stateNode.Path, sqw.stmts <- []byte(fmt.Sprintf(stateInsert, stateNode.BlockNumber, stateNode.HeaderID, stateKey, stateNode.CID, stateNode.Path,
stateNode.NodeType, true, stateNode.MhKey)) stateNode.NodeType, true, stateNode.MhKey))
} }
func (sqw *SQLWriter) upsertStateAccount(stateAccount models.StateAccountModel) { func (sqw *SQLWriter) upsertStateAccount(stateAccount models.StateAccountModel) {
sqw.stmts <- []byte(fmt.Sprintf(accountInsert, stateAccount.HeaderID, stateAccount.StatePath, stateAccount.Balance, sqw.stmts <- []byte(fmt.Sprintf(accountInsert, stateAccount.BlockNumber, stateAccount.HeaderID, stateAccount.StatePath, stateAccount.Balance,
stateAccount.Nonce, stateAccount.CodeHash, stateAccount.StorageRoot)) stateAccount.Nonce, stateAccount.CodeHash, stateAccount.StorageRoot))
} }
@ -250,6 +254,6 @@ func (sqw *SQLWriter) upsertStorageCID(storageCID models.StorageNodeModel) {
if storageCID.StorageKey != nullHash.String() { if storageCID.StorageKey != nullHash.String() {
storageKey = storageCID.StorageKey storageKey = storageCID.StorageKey
} }
sqw.stmts <- []byte(fmt.Sprintf(storageInsert, storageCID.HeaderID, storageCID.StatePath, storageKey, storageCID.CID, sqw.stmts <- []byte(fmt.Sprintf(storageInsert, storageCID.BlockNumber, storageCID.HeaderID, storageCID.StatePath, storageKey, storageCID.CID,
storageCID.Path, storageCID.NodeType, true, storageCID.MhKey)) storageCID.Path, storageCID.NodeType, true, storageCID.MhKey))
} }

48
statediff/indexer/database/sql/batch_tx.go
View File

@ -18,6 +18,7 @@ package sql
import ( import (
"context" "context"
"sync/atomic"
blockstore "github.com/ipfs/go-ipfs-blockstore" blockstore "github.com/ipfs/go-ipfs-blockstore"
dshelp "github.com/ipfs/go-ipfs-ds-help" dshelp "github.com/ipfs/go-ipfs-ds-help"
@ -29,15 +30,18 @@ import (
"github.com/ethereum/go-ethereum/statediff/indexer/models" "github.com/ethereum/go-ethereum/statediff/indexer/models"
) )
const startingCacheCapacity = 1024 * 24
// BatchTx wraps a sql tx with the state necessary for building the tx concurrently during trie difference iteration // BatchTx wraps a sql tx with the state necessary for building the tx concurrently during trie difference iteration
type BatchTx struct { type BatchTx struct {
BlockNumber uint64 BlockNumber string
ctx context.Context ctx context.Context
dbtx Tx dbtx Tx
stm string stm string
quit chan struct{} quit chan struct{}
iplds chan models.IPLDModel iplds chan models.IPLDModel
ipldCache models.IPLDBatch ipldCache models.IPLDBatch
removedCacheFlag *uint32
submit func(blockTx *BatchTx, err error) error submit func(blockTx *BatchTx, err error) error
} }
@ -48,7 +52,8 @@ func (tx *BatchTx) Submit(err error) error {
} }
func (tx *BatchTx) flush() error { func (tx *BatchTx) flush() error {
_, err := tx.dbtx.Exec(tx.ctx, tx.stm, pq.Array(tx.ipldCache.Keys), pq.Array(tx.ipldCache.Values)) _, err := tx.dbtx.Exec(tx.ctx, tx.stm, pq.Array(tx.ipldCache.BlockNumbers), pq.Array(tx.ipldCache.Keys),
pq.Array(tx.ipldCache.Values))
if err != nil { if err != nil {
return err return err
} }
@ -61,6 +66,7 @@ func (tx *BatchTx) cache() {
for { for {
select { select {
case i := <-tx.iplds: case i := <-tx.iplds:
tx.ipldCache.BlockNumbers = append(tx.ipldCache.BlockNumbers, i.BlockNumber)
tx.ipldCache.Keys = append(tx.ipldCache.Keys, i.Key) tx.ipldCache.Keys = append(tx.ipldCache.Keys, i.Key)
tx.ipldCache.Values = append(tx.ipldCache.Values, i.Data) tx.ipldCache.Values = append(tx.ipldCache.Values, i.Data)
case <-tx.quit: case <-tx.quit:
@ -72,15 +78,17 @@ func (tx *BatchTx) cache() {
func (tx *BatchTx) cacheDirect(key string, value []byte) { func (tx *BatchTx) cacheDirect(key string, value []byte) {
tx.iplds <- models.IPLDModel{ tx.iplds <- models.IPLDModel{
Key: key, BlockNumber: tx.BlockNumber,
Data: value, Key: key,
Data: value,
} }
} }
func (tx *BatchTx) cacheIPLD(i node.Node) { func (tx *BatchTx) cacheIPLD(i node.Node) {
tx.iplds <- models.IPLDModel{ tx.iplds <- models.IPLDModel{
Key: blockstore.BlockPrefix.String() + dshelp.MultihashToDsKey(i.Cid().Hash()).String(), BlockNumber: tx.BlockNumber,
Data: i.RawData(), Key: blockstore.BlockPrefix.String() + dshelp.MultihashToDsKey(i.Cid().Hash()).String(),
Data: i.RawData(),
} }
} }
@ -91,12 +99,24 @@ func (tx *BatchTx) cacheRaw(codec, mh uint64, raw []byte) (string, string, error
} }
prefixedKey := blockstore.BlockPrefix.String() + dshelp.MultihashToDsKey(c.Hash()).String() prefixedKey := blockstore.BlockPrefix.String() + dshelp.MultihashToDsKey(c.Hash()).String()
tx.iplds <- models.IPLDModel{ tx.iplds <- models.IPLDModel{
Key: prefixedKey, BlockNumber: tx.BlockNumber,
Data: raw, Key: prefixedKey,
Data: raw,
} }
return c.String(), prefixedKey, err return c.String(), prefixedKey, err
} }
func (tx *BatchTx) cacheRemoved(key string, value []byte) {
if atomic.LoadUint32(tx.removedCacheFlag) == 0 {
atomic.StoreUint32(tx.removedCacheFlag, 1)
tx.iplds <- models.IPLDModel{
BlockNumber: tx.BlockNumber,
Key: key,
Data: value,
}
}
}
// rollback sql transaction and log any error // rollback sql transaction and log any error
func rollback(ctx context.Context, tx Tx) { func rollback(ctx context.Context, tx Tx) {
if err := tx.Rollback(ctx); err != nil { if err := tx.Rollback(ctx); err != nil {

149
statediff/indexer/database/sql/indexer.go
View File

@ -55,14 +55,11 @@ type StateDiffIndexer struct {
ctx context.Context ctx context.Context
chainConfig *params.ChainConfig chainConfig *params.ChainConfig
dbWriter *Writer dbWriter *Writer
blockNumber string
} }
// NewStateDiffIndexer creates a sql implementation of interfaces.StateDiffIndexer // NewStateDiffIndexer creates a sql implementation of interfaces.StateDiffIndexer
func NewStateDiffIndexer(ctx context.Context, chainConfig *params.ChainConfig, db Database) (*StateDiffIndexer, error) { func NewStateDiffIndexer(ctx context.Context, chainConfig *params.ChainConfig, db Database) (*StateDiffIndexer, error) {
// Write the removed node to the db on init
if _, err := db.Exec(ctx, db.InsertIPLDStm(), shared.RemovedNodeMhKey, []byte{}); err != nil {
return nil, err
}
return &StateDiffIndexer{ return &StateDiffIndexer{
ctx: ctx, ctx: ctx,
chainConfig: chainConfig, chainConfig: chainConfig,
@ -93,6 +90,7 @@ func (sdi *StateDiffIndexer) ReportDBMetrics(delay time.Duration, quit <-chan bo
// Returns an initiated DB transaction which must be Closed via defer to commit or rollback // Returns an initiated DB transaction which must be Closed via defer to commit or rollback
func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receipts, totalDifficulty *big.Int) (interfaces.Batch, error) { func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receipts, totalDifficulty *big.Int) (interfaces.Batch, error) {
start, t := time.Now(), time.Now() start, t := time.Now(), time.Now()
sdi.blockNumber = block.Number().String()
blockHash := block.Hash() blockHash := block.Hash()
blockHashStr := blockHash.String() blockHashStr := blockHash.String()
height := block.NumberU64() height := block.NumberU64()
@ -140,13 +138,18 @@ func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receip
} }
}() }()
blockTx := &BatchTx{ blockTx := &BatchTx{
ctx: sdi.ctx, removedCacheFlag: new(uint32),
BlockNumber: height, ctx: sdi.ctx,
stm: sdi.dbWriter.db.InsertIPLDsStm(), BlockNumber: sdi.blockNumber,
iplds: make(chan models.IPLDModel), stm: sdi.dbWriter.db.InsertIPLDsStm(),
quit: make(chan struct{}), iplds: make(chan models.IPLDModel),
ipldCache: models.IPLDBatch{}, quit: make(chan struct{}),
dbtx: tx, ipldCache: models.IPLDBatch{
BlockNumbers: make([]string, 0, startingCacheCapacity),
Keys: make([]string, 0, startingCacheCapacity),
Values: make([][]byte, 0, startingCacheCapacity),
},
dbtx: tx,
// handle transaction commit or rollback for any return case // handle transaction commit or rollback for any return case
submit: func(self *BatchTx, err error) error { submit: func(self *BatchTx, err error) error {
defer func() { defer func() {
@ -200,7 +203,7 @@ func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receip
traceMsg += fmt.Sprintf("header processing time: %s\r\n", tDiff.String()) traceMsg += fmt.Sprintf("header processing time: %s\r\n", tDiff.String())
t = time.Now() t = time.Now()
// Publish and index uncles // Publish and index uncles
err = sdi.processUncles(blockTx, headerID, height, uncleNodes) err = sdi.processUncles(blockTx, headerID, block.NumberU64(), uncleNodes)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -249,7 +252,7 @@ func (sdi *StateDiffIndexer) processHeader(tx *BatchTx, header *types.Header, he
CID: headerNode.Cid().String(), CID: headerNode.Cid().String(),
MhKey: shared.MultihashKeyFromCID(headerNode.Cid()), MhKey: shared.MultihashKeyFromCID(headerNode.Cid()),
ParentHash: header.ParentHash.String(), ParentHash: header.ParentHash.String(),
BlockNumber: header.Number.String(), BlockNumber: sdi.blockNumber,
BlockHash: headerID, BlockHash: headerID,
TotalDifficulty: td.String(), TotalDifficulty: td.String(),
Reward: reward.String(), Reward: reward.String(),
@ -276,12 +279,13 @@ func (sdi *StateDiffIndexer) processUncles(tx *BatchTx, headerID string, blockNu
uncleReward = shared.CalcUncleMinerReward(blockNumber, uncleNode.Number.Uint64()) uncleReward = shared.CalcUncleMinerReward(blockNumber, uncleNode.Number.Uint64())
} }
uncle := models.UncleModel{ uncle := models.UncleModel{
HeaderID: headerID, BlockNumber: sdi.blockNumber,
CID: uncleNode.Cid().String(), HeaderID: headerID,
MhKey: shared.MultihashKeyFromCID(uncleNode.Cid()), CID: uncleNode.Cid().String(),
ParentHash: uncleNode.ParentHash.String(), MhKey: shared.MultihashKeyFromCID(uncleNode.Cid()),
BlockHash: uncleNode.Hash().String(), ParentHash: uncleNode.ParentHash.String(),
Reward: uncleReward.String(), BlockHash: uncleNode.Hash().String(),
Reward: uncleReward.String(),
} }
if err := sdi.dbWriter.upsertUncleCID(tx.dbtx, uncle); err != nil { if err := sdi.dbWriter.upsertUncleCID(tx.dbtx, uncle); err != nil {
return err return err
@ -331,16 +335,17 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(tx *BatchTx, args processArgs
return fmt.Errorf("error deriving tx sender: %v", err) return fmt.Errorf("error deriving tx sender: %v", err)
} }
txModel := models.TxModel{ txModel := models.TxModel{
HeaderID: args.headerID, BlockNumber: sdi.blockNumber,
Dst: shared.HandleZeroAddrPointer(trx.To()), HeaderID: args.headerID,
Src: shared.HandleZeroAddr(from), Dst: shared.HandleZeroAddrPointer(trx.To()),
TxHash: txID, Src: shared.HandleZeroAddr(from),
Index: int64(i), TxHash: txID,
Data: trx.Data(), Index: int64(i),
CID: txNode.Cid().String(), Data: trx.Data(),
MhKey: shared.MultihashKeyFromCID(txNode.Cid()), CID: txNode.Cid().String(),
Type: trx.Type(), MhKey: shared.MultihashKeyFromCID(txNode.Cid()),
Value: val, Type: trx.Type(),
Value: val,
} }
if err := sdi.dbWriter.upsertTransactionCID(tx.dbtx, txModel); err != nil { if err := sdi.dbWriter.upsertTransactionCID(tx.dbtx, txModel); err != nil {
return err return err
@ -353,6 +358,7 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(tx *BatchTx, args processArgs
storageKeys[k] = storageKey.Hex() storageKeys[k] = storageKey.Hex()
} }
accessListElementModel := models.AccessListElementModel{ accessListElementModel := models.AccessListElementModel{
BlockNumber: sdi.blockNumber,
TxID: txID, TxID: txID,
Index: int64(j), Index: int64(j),
Address: accessListElement.Address.Hex(), Address: accessListElement.Address.Hex(),
@ -376,6 +382,7 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(tx *BatchTx, args processArgs
} }
rctModel := &models.ReceiptModel{ rctModel := &models.ReceiptModel{
BlockNumber: sdi.blockNumber,
TxID: txID, TxID: txID,
Contract: contract, Contract: contract,
ContractHash: contractHash, ContractHash: contractHash,
@ -406,16 +413,17 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(tx *BatchTx, args processArgs
} }
logDataSet[idx] = &models.LogsModel{ logDataSet[idx] = &models.LogsModel{
ReceiptID: txID, BlockNumber: sdi.blockNumber,
Address: l.Address.String(), ReceiptID: txID,
Index: int64(l.Index), Address: l.Address.String(),
Data: l.Data, Index: int64(l.Index),
LeafCID: args.logLeafNodeCIDs[i][idx].String(), Data: l.Data,
LeafMhKey: shared.MultihashKeyFromCID(args.logLeafNodeCIDs[i][idx]), LeafCID: args.logLeafNodeCIDs[i][idx].String(),
Topic0: topicSet[0], LeafMhKey: shared.MultihashKeyFromCID(args.logLeafNodeCIDs[i][idx]),
Topic1: topicSet[1], Topic0: topicSet[0],
Topic2: topicSet[2], Topic1: topicSet[1],
Topic3: topicSet[3], Topic2: topicSet[2],
Topic3: topicSet[3],
} }
} }
@ -441,15 +449,15 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt
} }
// publish the state node // publish the state node
if stateNode.NodeType == sdtypes.Removed { if stateNode.NodeType == sdtypes.Removed {
// short circuit if it is a Removed node tx.cacheRemoved(shared.RemovedNodeMhKey, []byte{})
// this assumes the db has been initialized and a public.blocks entry for the Removed node is present
stateModel := models.StateNodeModel{ stateModel := models.StateNodeModel{
HeaderID: headerID, BlockNumber: sdi.blockNumber,
Path: stateNode.Path, HeaderID: headerID,
StateKey: common.BytesToHash(stateNode.LeafKey).String(), Path: stateNode.Path,
CID: shared.RemovedNodeStateCID, StateKey: common.BytesToHash(stateNode.LeafKey).String(),
MhKey: shared.RemovedNodeMhKey, CID: shared.RemovedNodeStateCID,
NodeType: stateNode.NodeType.Int(), MhKey: shared.RemovedNodeMhKey,
NodeType: stateNode.NodeType.Int(),
} }
return sdi.dbWriter.upsertStateCID(tx.dbtx, stateModel) return sdi.dbWriter.upsertStateCID(tx.dbtx, stateModel)
} }
@ -458,12 +466,13 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt
return fmt.Errorf("error generating and cacheing state node IPLD: %v", err) return fmt.Errorf("error generating and cacheing state node IPLD: %v", err)
} }
stateModel := models.StateNodeModel{ stateModel := models.StateNodeModel{
HeaderID: headerID, BlockNumber: sdi.blockNumber,
Path: stateNode.Path, HeaderID: headerID,
StateKey: common.BytesToHash(stateNode.LeafKey).String(), Path: stateNode.Path,
CID: stateCIDStr, StateKey: common.BytesToHash(stateNode.LeafKey).String(),
MhKey: stateMhKey, CID: stateCIDStr,
NodeType: stateNode.NodeType.Int(), MhKey: stateMhKey,
NodeType: stateNode.NodeType.Int(),
} }
// index the state node // index the state node
if err := sdi.dbWriter.upsertStateCID(tx.dbtx, stateModel); err != nil { if err := sdi.dbWriter.upsertStateCID(tx.dbtx, stateModel); err != nil {
@ -483,6 +492,7 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt
return fmt.Errorf("error decoding state account rlp: %s", err.Error()) return fmt.Errorf("error decoding state account rlp: %s", err.Error())
} }
accountModel := models.StateAccountModel{ accountModel := models.StateAccountModel{
BlockNumber: sdi.blockNumber,
HeaderID: headerID, HeaderID: headerID,
StatePath: stateNode.Path, StatePath: stateNode.Path,
Balance: account.Balance.String(), Balance: account.Balance.String(),
@ -497,16 +507,16 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt
// if there are any storage nodes associated with this node, publish and index them // if there are any storage nodes associated with this node, publish and index them
for _, storageNode := range stateNode.StorageNodes { for _, storageNode := range stateNode.StorageNodes {
if storageNode.NodeType == sdtypes.Removed { if storageNode.NodeType == sdtypes.Removed {
// short circuit if it is a Removed node tx.cacheRemoved(shared.RemovedNodeMhKey, []byte{})
// this assumes the db has been initialized and a public.blocks entry for the Removed node is present
storageModel := models.StorageNodeModel{ storageModel := models.StorageNodeModel{
HeaderID: headerID, BlockNumber: sdi.blockNumber,
StatePath: stateNode.Path, HeaderID: headerID,
Path: storageNode.Path, StatePath: stateNode.Path,
StorageKey: common.BytesToHash(storageNode.LeafKey).String(), Path: storageNode.Path,
CID: shared.RemovedNodeStorageCID, StorageKey: common.BytesToHash(storageNode.LeafKey).String(),
MhKey: shared.RemovedNodeMhKey, CID: shared.RemovedNodeStorageCID,
NodeType: storageNode.NodeType.Int(), MhKey: shared.RemovedNodeMhKey,
NodeType: storageNode.NodeType.Int(),
} }
if err := sdi.dbWriter.upsertStorageCID(tx.dbtx, storageModel); err != nil { if err := sdi.dbWriter.upsertStorageCID(tx.dbtx, storageModel); err != nil {
return err return err
@ -518,13 +528,14 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt
return fmt.Errorf("error generating and cacheing storage node IPLD: %v", err) return fmt.Errorf("error generating and cacheing storage node IPLD: %v", err)
} }
storageModel := models.StorageNodeModel{ storageModel := models.StorageNodeModel{
HeaderID: headerID, BlockNumber: sdi.blockNumber,
StatePath: stateNode.Path, HeaderID: headerID,
Path: storageNode.Path, StatePath: stateNode.Path,
StorageKey: common.BytesToHash(storageNode.LeafKey).String(), Path: storageNode.Path,
CID: storageCIDStr, StorageKey: common.BytesToHash(storageNode.LeafKey).String(),
MhKey: storageMhKey, CID: storageCIDStr,
NodeType: storageNode.NodeType.Int(), MhKey: storageMhKey,
NodeType: storageNode.NodeType.Int(),
} }
if err := sdi.dbWriter.upsertStorageCID(tx.dbtx, storageModel); err != nil { if err := sdi.dbWriter.upsertStorageCID(tx.dbtx, storageModel); err != nil {
return err return err

2
statediff/indexer/database/sql/indexer_shared_test.go
View File

@ -23,7 +23,7 @@ var (
err error err error
ind interfaces.StateDiffIndexer ind interfaces.StateDiffIndexer
ipfsPgGet = `SELECT data FROM public.blocks ipfsPgGet = `SELECT data FROM public.blocks
WHERE key = $1` WHERE key = $1 AND block_number = $2`
tx1, tx2, tx3, tx4, tx5, rct1, rct2, rct3, rct4, rct5 []byte tx1, tx2, tx3, tx4, tx5, rct1, rct2, rct3, rct4, rct5 []byte
mockBlock *types.Block mockBlock *types.Block
headerCID, trx1CID, trx2CID, trx3CID, trx4CID, trx5CID cid.Cid headerCID, trx1CID, trx2CID, trx3CID, trx4CID, trx5CID cid.Cid

11
statediff/indexer/database/sql/pgx_indexer_legacy_test.go
View File

@ -27,7 +27,6 @@ import (
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
"github.com/ethereum/go-ethereum/statediff/indexer/interfaces" "github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
"github.com/ethereum/go-ethereum/statediff/indexer/ipld" "github.com/ethereum/go-ethereum/statediff/indexer/ipld"
"github.com/ethereum/go-ethereum/statediff/indexer/test_helpers"
) )
func setupLegacyPGX(t *testing.T) { func setupLegacyPGX(t *testing.T) {
@ -56,7 +55,7 @@ func setupLegacyPGX(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
} }
test_helpers.ExpectEqual(t, tx.(*sql.BatchTx).BlockNumber, legacyData.BlockNumber.Uint64()) require.Equal(t, legacyData.BlockNumber.String(), tx.(*sql.BatchTx).BlockNumber)
} }
func TestLegacyPGXIndexer(t *testing.T) { func TestLegacyPGXIndexer(t *testing.T) {
@ -81,10 +80,10 @@ func TestLegacyPGXIndexer(t *testing.T) {
&header.CID, &header.TD, &header.Reward, &header.BlockHash, &header.Coinbase) &header.CID, &header.TD, &header.Reward, &header.BlockHash, &header.Coinbase)
require.NoError(t, err) require.NoError(t, err)
test_helpers.ExpectEqual(t, header.CID, legacyHeaderCID.String()) require.Equal(t, legacyHeaderCID.String(), header.CID)
test_helpers.ExpectEqual(t, header.TD, legacyData.MockBlock.Difficulty().String()) require.Equal(t, legacyData.MockBlock.Difficulty().String(), header.TD)
test_helpers.ExpectEqual(t, header.Reward, "5000000000000011250") require.Equal(t, "5000000000000011250", header.Reward)
test_helpers.ExpectEqual(t, header.Coinbase, legacyData.MockHeader.Coinbase.String()) require.Equal(t, legacyData.MockHeader.Coinbase.String(), header.Coinbase)
require.Nil(t, legacyData.MockHeader.BaseFee) require.Nil(t, legacyData.MockHeader.BaseFee)
}) })
} }

176
statediff/indexer/database/sql/pgx_indexer_test.go
View File

@ -20,6 +20,8 @@ import (
"context" "context"
"testing" "testing"
"github.com/lib/pq"
"github.com/ipfs/go-cid" "github.com/ipfs/go-cid"
blockstore "github.com/ipfs/go-ipfs-blockstore" blockstore "github.com/ipfs/go-ipfs-blockstore"
dshelp "github.com/ipfs/go-ipfs-ds-help" dshelp "github.com/ipfs/go-ipfs-ds-help"
@ -62,7 +64,7 @@ func setupPGX(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
} }
test_helpers.ExpectEqual(t, tx.(*sql.BatchTx).BlockNumber, mocks.BlockNumber.Uint64()) require.Equal(t, mocks.BlockNumber.String(), tx.(*sql.BatchTx).BlockNumber)
} }
func TestPGXIndexer(t *testing.T) { func TestPGXIndexer(t *testing.T) {
@ -91,10 +93,10 @@ func TestPGXIndexer(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
test_helpers.ExpectEqual(t, header.CID, headerCID.String()) require.Equal(t, headerCID.String(), header.CID)
test_helpers.ExpectEqual(t, header.TD, mocks.MockBlock.Difficulty().String()) require.Equal(t, mocks.MockBlock.Difficulty().String(), header.TD)
test_helpers.ExpectEqual(t, header.Reward, "2000000000000021250") require.Equal(t, "2000000000000021250", header.Reward)
test_helpers.ExpectEqual(t, header.Coinbase, mocks.MockHeader.Coinbase.String()) require.Equal(t, mocks.MockHeader.Coinbase.String(), header.Coinbase)
dc, err := cid.Decode(header.CID) dc, err := cid.Decode(header.CID)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@ -102,11 +104,11 @@ func TestPGXIndexer(t *testing.T) {
mhKey := dshelp.MultihashToDsKey(dc.Hash()) mhKey := dshelp.MultihashToDsKey(dc.Hash())
prefixedKey := blockstore.BlockPrefix.String() + mhKey.String() prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
var data []byte var data []byte
err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey) err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64())
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
test_helpers.ExpectEqual(t, data, mocks.MockHeaderRlp) require.Equal(t, mocks.MockHeaderRlp, data)
}) })
t.Run("Publish and index transaction IPLDs in a single tx", func(t *testing.T) { t.Run("Publish and index transaction IPLDs in a single tx", func(t *testing.T) {
@ -121,7 +123,7 @@ func TestPGXIndexer(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
test_helpers.ExpectEqual(t, len(trxs), 5) require.Equal(t, 5, len(trxs))
expectTrue(t, test_helpers.ListContainsString(trxs, trx1CID.String())) expectTrue(t, test_helpers.ListContainsString(trxs, trx1CID.String()))
expectTrue(t, test_helpers.ListContainsString(trxs, trx2CID.String())) expectTrue(t, test_helpers.ListContainsString(trxs, trx2CID.String()))
expectTrue(t, test_helpers.ListContainsString(trxs, trx3CID.String())) expectTrue(t, test_helpers.ListContainsString(trxs, trx3CID.String()))
@ -141,14 +143,14 @@ func TestPGXIndexer(t *testing.T) {
mhKey := dshelp.MultihashToDsKey(dc.Hash()) mhKey := dshelp.MultihashToDsKey(dc.Hash())
prefixedKey := blockstore.BlockPrefix.String() + mhKey.String() prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
var data []byte var data []byte
err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey) err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64())
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
txTypeAndValueStr := `SELECT tx_type, CAST(value as TEXT) FROM eth.transaction_cids WHERE cid = $1` txTypeAndValueStr := `SELECT tx_type, CAST(value as TEXT) FROM eth.transaction_cids WHERE cid = $1`
switch c { switch c {
case trx1CID.String(): case trx1CID.String():
test_helpers.ExpectEqual(t, data, tx1) require.Equal(t, tx1, data)
txRes := new(txResult) txRes := new(txResult)
err = db.QueryRow(context.Background(), txTypeAndValueStr, c).Scan(&txRes.TxType, &txRes.Value) err = db.QueryRow(context.Background(), txTypeAndValueStr, c).Scan(&txRes.TxType, &txRes.Value)
if err != nil { if err != nil {
@ -161,7 +163,7 @@ func TestPGXIndexer(t *testing.T) {
t.Fatalf("expected tx value %s got %s", transactions[0].Value().String(), txRes.Value) t.Fatalf("expected tx value %s got %s", transactions[0].Value().String(), txRes.Value)
} }
case trx2CID.String(): case trx2CID.String():
test_helpers.ExpectEqual(t, data, tx2) require.Equal(t, tx2, data)
txRes := new(txResult) txRes := new(txResult)
err = db.QueryRow(context.Background(), txTypeAndValueStr, c).Scan(&txRes.TxType, &txRes.Value) err = db.QueryRow(context.Background(), txTypeAndValueStr, c).Scan(&txRes.TxType, &txRes.Value)
if err != nil { if err != nil {
@ -174,7 +176,7 @@ func TestPGXIndexer(t *testing.T) {
t.Fatalf("expected tx value %s got %s", transactions[1].Value().String(), txRes.Value) t.Fatalf("expected tx value %s got %s", transactions[1].Value().String(), txRes.Value)
} }
case trx3CID.String(): case trx3CID.String():
test_helpers.ExpectEqual(t, data, tx3) require.Equal(t, tx3, data)
txRes := new(txResult) txRes := new(txResult)
err = db.QueryRow(context.Background(), txTypeAndValueStr, c).Scan(&txRes.TxType, &txRes.Value) err = db.QueryRow(context.Background(), txTypeAndValueStr, c).Scan(&txRes.TxType, &txRes.Value)
if err != nil { if err != nil {
@ -187,7 +189,7 @@ func TestPGXIndexer(t *testing.T) {
t.Fatalf("expected tx value %s got %s", transactions[2].Value().String(), txRes.Value) t.Fatalf("expected tx value %s got %s", transactions[2].Value().String(), txRes.Value)
} }
case trx4CID.String(): case trx4CID.String():
test_helpers.ExpectEqual(t, data, tx4) require.Equal(t, tx4, data)
txRes := new(txResult) txRes := new(txResult)
err = db.QueryRow(context.Background(), txTypeAndValueStr, c).Scan(&txRes.TxType, &txRes.Value) err = db.QueryRow(context.Background(), txTypeAndValueStr, c).Scan(&txRes.TxType, &txRes.Value)
if err != nil { if err != nil {
@ -199,8 +201,18 @@ func TestPGXIndexer(t *testing.T) {
if txRes.Value != transactions[3].Value().String() { if txRes.Value != transactions[3].Value().String() {
t.Fatalf("expected tx value %s got %s", transactions[3].Value().String(), txRes.Value) t.Fatalf("expected tx value %s got %s", transactions[3].Value().String(), txRes.Value)
} }
// AccessListElementModel is the db model for eth.access_list_entry
type AccessListElementModel struct {
BlockNumber string `db:"block_number"`
Index int64 `db:"index"`
TxID string `db:"tx_id"`
Address string `db:"address"`
StorageKeys pq.StringArray `db:"storage_keys"`
}
accessListElementModels := make([]models.AccessListElementModel, 0) accessListElementModels := make([]models.AccessListElementModel, 0)
pgStr = `SELECT access_list_elements.* FROM eth.access_list_elements INNER JOIN eth.transaction_cids ON (tx_id = transaction_cids.tx_hash) WHERE cid = $1 ORDER BY access_list_elements.index ASC` pgStr = "SELECT cast(access_list_elements.block_number AS TEXT), access_list_elements.index, access_list_elements.tx_id, " +
"access_list_elements.address, access_list_elements.storage_keys FROM eth.access_list_elements " +
"INNER JOIN eth.transaction_cids ON (tx_id = transaction_cids.tx_hash) WHERE cid = $1 ORDER BY access_list_elements.index ASC"
err = db.Select(context.Background(), &accessListElementModels, pgStr, c) err = db.Select(context.Background(), &accessListElementModels, pgStr, c)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@ -209,18 +221,20 @@ func TestPGXIndexer(t *testing.T) {
t.Fatalf("expected two access list entries, got %d", len(accessListElementModels)) t.Fatalf("expected two access list entries, got %d", len(accessListElementModels))
} }
model1 := models.AccessListElementModel{ model1 := models.AccessListElementModel{
Index: accessListElementModels[0].Index, BlockNumber: mocks.BlockNumber.String(),
Address: accessListElementModels[0].Address, Index: accessListElementModels[0].Index,
Address: accessListElementModels[0].Address,
} }
model2 := models.AccessListElementModel{ model2 := models.AccessListElementModel{
BlockNumber: mocks.BlockNumber.String(),
Index: accessListElementModels[1].Index, Index: accessListElementModels[1].Index,
Address: accessListElementModels[1].Address, Address: accessListElementModels[1].Address,
StorageKeys: accessListElementModels[1].StorageKeys, StorageKeys: accessListElementModels[1].StorageKeys,
} }
test_helpers.ExpectEqual(t, model1, mocks.AccessListEntry1Model) require.Equal(t, mocks.AccessListEntry1Model, model1)
test_helpers.ExpectEqual(t, model2, mocks.AccessListEntry2Model) require.Equal(t, mocks.AccessListEntry2Model, model2)
case trx5CID.String(): case trx5CID.String():
test_helpers.ExpectEqual(t, data, tx5) require.Equal(t, tx5, data)
txRes := new(txResult) txRes := new(txResult)
err = db.QueryRow(context.Background(), txTypeAndValueStr, c).Scan(&txRes.TxType, &txRes.Value) err = db.QueryRow(context.Background(), txTypeAndValueStr, c).Scan(&txRes.TxType, &txRes.Value)
if err != nil { if err != nil {
@ -272,7 +286,7 @@ func TestPGXIndexer(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
expectedLogs := mocks.MockReceipts[i].Logs expectedLogs := mocks.MockReceipts[i].Logs
test_helpers.ExpectEqual(t, len(results), len(expectedLogs)) require.Equal(t, len(expectedLogs), len(results))
var nodeElements []interface{} var nodeElements []interface{}
for idx, r := range results { for idx, r := range results {
@ -283,12 +297,12 @@ func TestPGXIndexer(t *testing.T) {
logRaw, err := rlp.EncodeToBytes(expectedLogs[idx]) logRaw, err := rlp.EncodeToBytes(expectedLogs[idx])
require.NoError(t, err) require.NoError(t, err)
// 2nd element of the leaf node contains the encoded log data. // 2nd element of the leaf node contains the encoded log data.
test_helpers.ExpectEqual(t, logRaw, nodeElements[1].([]byte)) require.Equal(t, nodeElements[1].([]byte), logRaw)
} else { } else {
logRaw, err := rlp.EncodeToBytes(expectedLogs[idx]) logRaw, err := rlp.EncodeToBytes(expectedLogs[idx])
require.NoError(t, err) require.NoError(t, err)
// raw log was IPLDized // raw log was IPLDized
test_helpers.ExpectEqual(t, logRaw, r.Data) require.Equal(t, r.Data, logRaw)
} }
} }
} }
@ -309,7 +323,7 @@ func TestPGXIndexer(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
test_helpers.ExpectEqual(t, len(rcts), 5) require.Equal(t, 5, len(rcts))
expectTrue(t, test_helpers.ListContainsString(rcts, rct1CID.String())) expectTrue(t, test_helpers.ListContainsString(rcts, rct1CID.String()))
expectTrue(t, test_helpers.ListContainsString(rcts, rct2CID.String())) expectTrue(t, test_helpers.ListContainsString(rcts, rct2CID.String()))
expectTrue(t, test_helpers.ListContainsString(rcts, rct3CID.String())) expectTrue(t, test_helpers.ListContainsString(rcts, rct3CID.String()))
@ -335,7 +349,7 @@ func TestPGXIndexer(t *testing.T) {
expectedRct, err := mocks.MockReceipts[idx].MarshalBinary() expectedRct, err := mocks.MockReceipts[idx].MarshalBinary()
require.NoError(t, err) require.NoError(t, err)
test_helpers.ExpectEqual(t, expectedRct, nodeElements[1].([]byte)) require.Equal(t, nodeElements[1].([]byte), expectedRct)
dc, err := cid.Decode(c) dc, err := cid.Decode(c)
if err != nil { if err != nil {
@ -344,7 +358,7 @@ func TestPGXIndexer(t *testing.T) {
mhKey := dshelp.MultihashToDsKey(dc.Hash()) mhKey := dshelp.MultihashToDsKey(dc.Hash())
prefixedKey := blockstore.BlockPrefix.String() + mhKey.String() prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
var data []byte var data []byte
err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey) err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64())
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -352,46 +366,46 @@ func TestPGXIndexer(t *testing.T) {
postStatePgStr := `SELECT post_state FROM eth.receipt_cids WHERE leaf_cid = $1` postStatePgStr := `SELECT post_state FROM eth.receipt_cids WHERE leaf_cid = $1`
switch c { switch c {
case rct1CID.String(): case rct1CID.String():
test_helpers.ExpectEqual(t, data, rctLeaf1) require.Equal(t, rctLeaf1, data)
var postStatus uint64 var postStatus uint64
pgStr = `SELECT post_status FROM eth.receipt_cids WHERE leaf_cid = $1` pgStr = `SELECT post_status FROM eth.receipt_cids WHERE leaf_cid = $1`
err = db.Get(context.Background(), &postStatus, pgStr, c) err = db.Get(context.Background(), &postStatus, pgStr, c)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
test_helpers.ExpectEqual(t, postStatus, mocks.ExpectedPostStatus) require.Equal(t, mocks.ExpectedPostStatus, postStatus)
case rct2CID.String(): case rct2CID.String():
test_helpers.ExpectEqual(t, data, rctLeaf2) require.Equal(t, rctLeaf2, data)
var postState string var postState string
err = db.Get(context.Background(), &postState, postStatePgStr, c) err = db.Get(context.Background(), &postState, postStatePgStr, c)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
test_helpers.ExpectEqual(t, postState, mocks.ExpectedPostState1) require.Equal(t, mocks.ExpectedPostState1, postState)
case rct3CID.String(): case rct3CID.String():
test_helpers.ExpectEqual(t, data, rctLeaf3) require.Equal(t, rctLeaf3, data)
var postState string var postState string
err = db.Get(context.Background(), &postState, postStatePgStr, c) err = db.Get(context.Background(), &postState, postStatePgStr, c)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
test_helpers.ExpectEqual(t, postState, mocks.ExpectedPostState2) require.Equal(t, mocks.ExpectedPostState2, postState)
case rct4CID.String(): case rct4CID.String():
test_helpers.ExpectEqual(t, data, rctLeaf4) require.Equal(t, rctLeaf4, data)
var postState string var postState string
err = db.Get(context.Background(), &postState, postStatePgStr, c) err = db.Get(context.Background(), &postState, postStatePgStr, c)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
test_helpers.ExpectEqual(t, postState, mocks.ExpectedPostState3) require.Equal(t, mocks.ExpectedPostState3, postState)
case rct5CID.String(): case rct5CID.String():
test_helpers.ExpectEqual(t, data, rctLeaf5) require.Equal(t, rctLeaf5, data)
var postState string var postState string
err = db.Get(context.Background(), &postState, postStatePgStr, c) err = db.Get(context.Background(), &postState, postStatePgStr, c)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
test_helpers.ExpectEqual(t, postState, mocks.ExpectedPostState3) require.Equal(t, mocks.ExpectedPostState3, postState)
} }
} }
}) })
@ -409,7 +423,7 @@ func TestPGXIndexer(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
test_helpers.ExpectEqual(t, len(stateNodes), 2) require.Equal(t, 2, len(stateNodes))
for _, stateNode := range stateNodes { for _, stateNode := range stateNodes {
var data []byte var data []byte
dc, err := cid.Decode(stateNode.CID) dc, err := cid.Decode(stateNode.CID)
@ -418,43 +432,45 @@ func TestPGXIndexer(t *testing.T) {
} }
mhKey := dshelp.MultihashToDsKey(dc.Hash()) mhKey := dshelp.MultihashToDsKey(dc.Hash())
prefixedKey := blockstore.BlockPrefix.String() + mhKey.String() prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey) err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64())
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
pgStr = `SELECT header_id, state_path, cast(balance AS TEXT), nonce, code_hash, storage_root from eth.state_accounts WHERE header_id = $1 AND state_path = $2` pgStr = `SELECT cast(block_number AS TEXT), header_id, state_path, cast(balance AS TEXT), nonce, code_hash, storage_root from eth.state_accounts WHERE header_id = $1 AND state_path = $2`
var account models.StateAccountModel var account models.StateAccountModel
err = db.Get(context.Background(), &account, pgStr, stateNode.HeaderID, stateNode.Path) err = db.Get(context.Background(), &account, pgStr, stateNode.HeaderID, stateNode.Path)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
if stateNode.CID == state1CID.String() { if stateNode.CID == state1CID.String() {
test_helpers.ExpectEqual(t, stateNode.NodeType, 2) require.Equal(t, 2, stateNode.NodeType)
test_helpers.ExpectEqual(t, stateNode.StateKey, common.BytesToHash(mocks.ContractLeafKey).Hex()) require.Equal(t, common.BytesToHash(mocks.ContractLeafKey).Hex(), stateNode.StateKey)
test_helpers.ExpectEqual(t, stateNode.Path, []byte{'\x06'}) require.Equal(t, []byte{'\x06'}, stateNode.Path)
test_helpers.ExpectEqual(t, data, mocks.ContractLeafNode) require.Equal(t, mocks.ContractLeafNode, data)
test_helpers.ExpectEqual(t, account, models.StateAccountModel{ require.Equal(t, models.StateAccountModel{
BlockNumber: mocks.BlockNumber.String(),
HeaderID: account.HeaderID, HeaderID: account.HeaderID,
StatePath: stateNode.Path, StatePath: stateNode.Path,
Balance: "0", Balance: "0",
CodeHash: mocks.ContractCodeHash.Bytes(), CodeHash: mocks.ContractCodeHash.Bytes(),
StorageRoot: mocks.ContractRoot, StorageRoot: mocks.ContractRoot,
Nonce: 1, Nonce: 1,
}) }, account)
} }
if stateNode.CID == state2CID.String() { if stateNode.CID == state2CID.String() {
test_helpers.ExpectEqual(t, stateNode.NodeType, 2) require.Equal(t, 2, stateNode.NodeType)
test_helpers.ExpectEqual(t, stateNode.StateKey, common.BytesToHash(mocks.AccountLeafKey).Hex()) require.Equal(t, common.BytesToHash(mocks.AccountLeafKey).Hex(), stateNode.StateKey)
test_helpers.ExpectEqual(t, stateNode.Path, []byte{'\x0c'}) require.Equal(t, []byte{'\x0c'}, stateNode.Path)
test_helpers.ExpectEqual(t, data, mocks.AccountLeafNode) require.Equal(t, mocks.AccountLeafNode, data)
test_helpers.ExpectEqual(t, account, models.StateAccountModel{ require.Equal(t, models.StateAccountModel{
BlockNumber: mocks.BlockNumber.String(),
HeaderID: account.HeaderID, HeaderID: account.HeaderID,
StatePath: stateNode.Path, StatePath: stateNode.Path,
Balance: "1000", Balance: "1000",
CodeHash: mocks.AccountCodeHash.Bytes(), CodeHash: mocks.AccountCodeHash.Bytes(),
StorageRoot: mocks.AccountRoot, StorageRoot: mocks.AccountRoot,
Nonce: 0, Nonce: 0,
}) }, account)
} }
} }
@ -467,7 +483,7 @@ func TestPGXIndexer(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
test_helpers.ExpectEqual(t, len(stateNodes), 1) require.Equal(t, 1, len(stateNodes))
stateNode := stateNodes[0] stateNode := stateNodes[0]
var data []byte var data []byte
dc, err := cid.Decode(stateNode.CID) dc, err := cid.Decode(stateNode.CID)
@ -476,14 +492,14 @@ func TestPGXIndexer(t *testing.T) {
} }
mhKey := dshelp.MultihashToDsKey(dc.Hash()) mhKey := dshelp.MultihashToDsKey(dc.Hash())
prefixedKey := blockstore.BlockPrefix.String() + mhKey.String() prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
test_helpers.ExpectEqual(t, prefixedKey, shared.RemovedNodeMhKey) require.Equal(t, shared.RemovedNodeMhKey, prefixedKey)
err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey) err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64())
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
test_helpers.ExpectEqual(t, stateNode.CID, shared.RemovedNodeStateCID) require.Equal(t, shared.RemovedNodeStateCID, stateNode.CID)
test_helpers.ExpectEqual(t, stateNode.Path, []byte{'\x02'}) require.Equal(t, []byte{'\x02'}, stateNode.Path)
test_helpers.ExpectEqual(t, data, []byte{}) require.Equal(t, []byte{}, data)
}) })
t.Run("Publish and index storage IPLDs in a single tx", func(t *testing.T) { t.Run("Publish and index storage IPLDs in a single tx", func(t *testing.T) {
@ -492,7 +508,7 @@ func TestPGXIndexer(t *testing.T) {
defer checkTxClosure(t, 1, 0, 1) defer checkTxClosure(t, 1, 0, 1)
// check that storage nodes were properly indexed // check that storage nodes were properly indexed
storageNodes := make([]models.StorageNodeWithStateKeyModel, 0) storageNodes := make([]models.StorageNodeWithStateKeyModel, 0)
pgStr := `SELECT storage_cids.cid, state_cids.state_leaf_key, storage_cids.storage_leaf_key, storage_cids.node_type, storage_cids.storage_path pgStr := `SELECT cast(storage_cids.block_number AS TEXT), storage_cids.cid, state_cids.state_leaf_key, storage_cids.storage_leaf_key, storage_cids.node_type, storage_cids.storage_path
FROM eth.storage_cids, eth.state_cids, eth.header_cids FROM eth.storage_cids, eth.state_cids, eth.header_cids
WHERE (storage_cids.state_path, storage_cids.header_id) = (state_cids.state_path, state_cids.header_id) WHERE (storage_cids.state_path, storage_cids.header_id) = (state_cids.state_path, state_cids.header_id)
AND state_cids.header_id = header_cids.block_hash AND state_cids.header_id = header_cids.block_hash
@ -502,14 +518,15 @@ func TestPGXIndexer(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
test_helpers.ExpectEqual(t, len(storageNodes), 1) require.Equal(t, 1, len(storageNodes))
test_helpers.ExpectEqual(t, storageNodes[0], models.StorageNodeWithStateKeyModel{ require.Equal(t, models.StorageNodeWithStateKeyModel{
CID: storageCID.String(), BlockNumber: mocks.BlockNumber.String(),
NodeType: 2, CID: storageCID.String(),
StorageKey: common.BytesToHash(mocks.StorageLeafKey).Hex(), NodeType: 2,
StateKey: common.BytesToHash(mocks.ContractLeafKey).Hex(), StorageKey: common.BytesToHash(mocks.StorageLeafKey).Hex(),
Path: []byte{}, StateKey: common.BytesToHash(mocks.ContractLeafKey).Hex(),
}) Path: []byte{},
}, storageNodes[0])
var data []byte var data []byte
dc, err := cid.Decode(storageNodes[0].CID) dc, err := cid.Decode(storageNodes[0].CID)
if err != nil { if err != nil {
@ -517,15 +534,15 @@ func TestPGXIndexer(t *testing.T) {
} }
mhKey := dshelp.MultihashToDsKey(dc.Hash()) mhKey := dshelp.MultihashToDsKey(dc.Hash())
prefixedKey := blockstore.BlockPrefix.String() + mhKey.String() prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey) err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64())
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
test_helpers.ExpectEqual(t, data, mocks.StorageLeafNode) require.Equal(t, mocks.StorageLeafNode, data)
// check that Removed storage nodes were properly indexed // check that Removed storage nodes were properly indexed
storageNodes = make([]models.StorageNodeWithStateKeyModel, 0) storageNodes = make([]models.StorageNodeWithStateKeyModel, 0)
pgStr = `SELECT storage_cids.cid, state_cids.state_leaf_key, storage_cids.storage_leaf_key, storage_cids.node_type, storage_cids.storage_path pgStr = `SELECT cast(storage_cids.block_number AS TEXT), storage_cids.cid, state_cids.state_leaf_key, storage_cids.storage_leaf_key, storage_cids.node_type, storage_cids.storage_path
FROM eth.storage_cids, eth.state_cids, eth.header_cids FROM eth.storage_cids, eth.state_cids, eth.header_cids
WHERE (storage_cids.state_path, storage_cids.header_id) = (state_cids.state_path, state_cids.header_id) WHERE (storage_cids.state_path, storage_cids.header_id) = (state_cids.state_path, state_cids.header_id)
AND state_cids.header_id = header_cids.block_hash AND state_cids.header_id = header_cids.block_hash
@ -535,25 +552,26 @@ func TestPGXIndexer(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
test_helpers.ExpectEqual(t, len(storageNodes), 1) require.Equal(t, 1, len(storageNodes))
test_helpers.ExpectEqual(t, storageNodes[0], models.StorageNodeWithStateKeyModel{ require.Equal(t, models.StorageNodeWithStateKeyModel{
CID: shared.RemovedNodeStorageCID, BlockNumber: mocks.BlockNumber.String(),
NodeType: 3, CID: shared.RemovedNodeStorageCID,
StorageKey: common.BytesToHash(mocks.RemovedLeafKey).Hex(), NodeType: 3,
StateKey: common.BytesToHash(mocks.ContractLeafKey).Hex(), StorageKey: common.BytesToHash(mocks.RemovedLeafKey).Hex(),
Path: []byte{'\x03'}, StateKey: common.BytesToHash(mocks.ContractLeafKey).Hex(),
}) Path: []byte{'\x03'},
}, storageNodes[0])
dc, err = cid.Decode(storageNodes[0].CID) dc, err = cid.Decode(storageNodes[0].CID)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
mhKey = dshelp.MultihashToDsKey(dc.Hash()) mhKey = dshelp.MultihashToDsKey(dc.Hash())
prefixedKey = blockstore.BlockPrefix.String() + mhKey.String() prefixedKey = blockstore.BlockPrefix.String() + mhKey.String()
test_helpers.ExpectEqual(t, prefixedKey, shared.RemovedNodeMhKey) require.Equal(t, shared.RemovedNodeMhKey, prefixedKey)
err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey) err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64())
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
test_helpers.ExpectEqual(t, data, []byte{}) require.Equal(t, []byte{}, data)
}) })
} }

6
statediff/indexer/database/sql/postgres/config.go
View File

@ -49,9 +49,9 @@ func ResolveDriverType(str string) (DriverType, error) {
var DefaultConfig = Config{ var DefaultConfig = Config{
Hostname: "localhost", Hostname: "localhost",
Port: 5432, Port: 5432,
DatabaseName: "vulcanize_testing_v3", DatabaseName: "vulcanize_testing_v4",
Username: "vdbm", Username: "postgres",
Password: "password", Password: "",
} }
// Config holds params for a Postgres db // Config holds params for a Postgres db

24
statediff/indexer/database/sql/postgres/database.go
View File

@ -44,58 +44,58 @@ func (db *DB) InsertHeaderStm() string {
// InsertUncleStm satisfies the sql.Statements interface // InsertUncleStm satisfies the sql.Statements interface
func (db *DB) InsertUncleStm() string { func (db *DB) InsertUncleStm() string {
return `INSERT INTO eth.uncle_cids (block_hash, header_id, parent_hash, cid, reward, mh_key) VALUES ($1, $2, $3, $4, $5, $6) return `INSERT INTO eth.uncle_cids (block_number, block_hash, header_id, parent_hash, cid, reward, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7)
ON CONFLICT (block_hash) DO NOTHING` ON CONFLICT (block_hash) DO NOTHING`
} }
// InsertTxStm satisfies the sql.Statements interface // InsertTxStm satisfies the sql.Statements interface
func (db *DB) InsertTxStm() string { func (db *DB) InsertTxStm() string {
return `INSERT INTO eth.transaction_cids (header_id, tx_hash, cid, dst, src, index, mh_key, tx_data, tx_type, value) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) return `INSERT INTO eth.transaction_cids (block_number, header_id, tx_hash, cid, dst, src, index, mh_key, tx_data, tx_type, value) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
ON CONFLICT (tx_hash) DO NOTHING` ON CONFLICT (tx_hash) DO NOTHING`
} }
// InsertAccessListElementStm satisfies the sql.Statements interface // InsertAccessListElementStm satisfies the sql.Statements interface
func (db *DB) InsertAccessListElementStm() string { func (db *DB) InsertAccessListElementStm() string {
return `INSERT INTO eth.access_list_elements (tx_id, index, address, storage_keys) VALUES ($1, $2, $3, $4) return `INSERT INTO eth.access_list_elements (block_number, tx_id, index, address, storage_keys) VALUES ($1, $2, $3, $4, $5)
ON CONFLICT (tx_id, index) DO NOTHING` ON CONFLICT (tx_id, index) DO NOTHING`
} }
// InsertRctStm satisfies the sql.Statements interface // InsertRctStm satisfies the sql.Statements interface
func (db *DB) InsertRctStm() string { func (db *DB) InsertRctStm() string {
return `INSERT INTO eth.receipt_cids (tx_id, leaf_cid, contract, contract_hash, leaf_mh_key, post_state, post_status, log_root) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) return `INSERT INTO eth.receipt_cids (block_number, tx_id, leaf_cid, contract, contract_hash, leaf_mh_key, post_state, post_status, log_root) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
ON CONFLICT (tx_id) DO NOTHING` ON CONFLICT (tx_id) DO NOTHING`
} }
// InsertLogStm satisfies the sql.Statements interface // InsertLogStm satisfies the sql.Statements interface
func (db *DB) InsertLogStm() string { func (db *DB) InsertLogStm() string {
return `INSERT INTO eth.log_cids (leaf_cid, leaf_mh_key, rct_id, address, index, topic0, topic1, topic2, topic3, log_data) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) return `INSERT INTO eth.log_cids (block_number, leaf_cid, leaf_mh_key, rct_id, address, index, topic0, topic1, topic2, topic3, log_data) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
ON CONFLICT (rct_id, index) DO NOTHING` ON CONFLICT (rct_id, index) DO NOTHING`
} }
// InsertStateStm satisfies the sql.Statements interface // InsertStateStm satisfies the sql.Statements interface
func (db *DB) InsertStateStm() string { func (db *DB) InsertStateStm() string {
return `INSERT INTO eth.state_cids (header_id, state_leaf_key, cid, state_path, node_type, diff, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7) return `INSERT INTO eth.state_cids (block_number, header_id, state_leaf_key, cid, state_path, node_type, diff, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
ON CONFLICT (header_id, state_path) DO UPDATE SET (state_leaf_key, cid, node_type, diff, mh_key) = ($2, $3, $5, $6, $7)` ON CONFLICT (header_id, state_path) DO UPDATE SET (block_number, state_leaf_key, cid, node_type, diff, mh_key) = ($1, $3, $4, $6, $7, $8)`
} }
// InsertAccountStm satisfies the sql.Statements interface // InsertAccountStm satisfies the sql.Statements interface
func (db *DB) InsertAccountStm() string { func (db *DB) InsertAccountStm() string {
return `INSERT INTO eth.state_accounts (header_id, state_path, balance, nonce, code_hash, storage_root) VALUES ($1, $2, $3, $4, $5, $6) return `INSERT INTO eth.state_accounts (block_number, header_id, state_path, balance, nonce, code_hash, storage_root) VALUES ($1, $2, $3, $4, $5, $6, $7)
ON CONFLICT (header_id, state_path) DO NOTHING` ON CONFLICT (header_id, state_path) DO NOTHING`
} }
// InsertStorageStm satisfies the sql.Statements interface // InsertStorageStm satisfies the sql.Statements interface
func (db *DB) InsertStorageStm() string { func (db *DB) InsertStorageStm() string {
return `INSERT INTO eth.storage_cids (header_id, state_path, storage_leaf_key, cid, storage_path, node_type, diff, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) return `INSERT INTO eth.storage_cids (block_number, header_id, state_path, storage_leaf_key, cid, storage_path, node_type, diff, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
ON CONFLICT (header_id, state_path, storage_path) DO UPDATE SET (storage_leaf_key, cid, node_type, diff, mh_key) = ($3, $4, $6, $7, $8)` ON CONFLICT (header_id, state_path, storage_path) DO UPDATE SET (block_number, storage_leaf_key, cid, node_type, diff, mh_key) = ($1, $4, $5, $7, $8, $9)`
} }
// InsertIPLDStm satisfies the sql.Statements interface // InsertIPLDStm satisfies the sql.Statements interface
func (db *DB) InsertIPLDStm() string { func (db *DB) InsertIPLDStm() string {
return `INSERT INTO public.blocks (key, data) VALUES ($1, $2) ON CONFLICT (key) DO NOTHING` return `INSERT INTO public.blocks (block_number, key, data) VALUES ($1, $2, $3) ON CONFLICT (block_number, key) DO NOTHING`
} }
// InsertIPLDsStm satisfies the sql.Statements interface // InsertIPLDsStm satisfies the sql.Statements interface
func (db *DB) InsertIPLDsStm() string { func (db *DB) InsertIPLDsStm() string {
return `INSERT INTO public.blocks (key, data) VALUES (unnest($1::TEXT[]), unnest($2::BYTEA[])) ON CONFLICT (key) DO NOTHING` return `INSERT INTO public.blocks (block_number, key, data) VALUES (unnest($1::BIGINT[]), unnest($2::TEXT[]), unnest($3::BYTEA[])) ON CONFLICT (block_number, key) DO NOTHING`
} }

11
statediff/indexer/database/sql/sqlx_indexer_legacy_test.go
View File

@ -31,7 +31,6 @@ import (
"github.com/ethereum/go-ethereum/statediff/indexer/interfaces" "github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
"github.com/ethereum/go-ethereum/statediff/indexer/ipld" "github.com/ethereum/go-ethereum/statediff/indexer/ipld"
"github.com/ethereum/go-ethereum/statediff/indexer/mocks" "github.com/ethereum/go-ethereum/statediff/indexer/mocks"
"github.com/ethereum/go-ethereum/statediff/indexer/test_helpers"
) )
var ( var (
@ -66,7 +65,7 @@ func setupLegacySQLX(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
} }
test_helpers.ExpectEqual(t, tx.(*sql.BatchTx).BlockNumber, legacyData.BlockNumber.Uint64()) require.Equal(t, legacyData.BlockNumber.String(), tx.(*sql.BatchTx).BlockNumber)
} }
func TestLegacySQLXIndexer(t *testing.T) { func TestLegacySQLXIndexer(t *testing.T) {
@ -89,10 +88,10 @@ func TestLegacySQLXIndexer(t *testing.T) {
err = db.QueryRow(context.Background(), pgStr, legacyData.BlockNumber.Uint64()).(*sqlx.Row).StructScan(header) err = db.QueryRow(context.Background(), pgStr, legacyData.BlockNumber.Uint64()).(*sqlx.Row).StructScan(header)
require.NoError(t, err) require.NoError(t, err)
test_helpers.ExpectEqual(t, header.CID, legacyHeaderCID.String()) require.Equal(t, legacyHeaderCID.String(), header.CID)
test_helpers.ExpectEqual(t, header.TD, legacyData.MockBlock.Difficulty().String()) require.Equal(t, legacyData.MockBlock.Difficulty().String(), header.TD)
test_helpers.ExpectEqual(t, header.Reward, "5000000000000011250") require.Equal(t, "5000000000000011250", header.Reward)
test_helpers.ExpectEqual(t, header.Coinbase, legacyData.MockHeader.Coinbase.String()) require.Equal(t, legacyData.MockHeader.Coinbase.String(), header.Coinbase)
require.Nil(t, legacyData.MockHeader.BaseFee) require.Nil(t, legacyData.MockHeader.BaseFee)
}) })
} }

164
statediff/indexer/database/sql/sqlx_indexer_test.go
View File

@ -63,7 +63,7 @@ func setupSQLX(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
} }
test_helpers.ExpectEqual(t, tx.(*sql.BatchTx).BlockNumber, mocks.BlockNumber.Uint64()) require.Equal(t, mocks.BlockNumber.String(), tx.(*sql.BatchTx).BlockNumber)
} }
func TestSQLXIndexer(t *testing.T) { func TestSQLXIndexer(t *testing.T) {
@ -87,10 +87,10 @@ func TestSQLXIndexer(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
test_helpers.ExpectEqual(t, header.CID, headerCID.String()) require.Equal(t, headerCID.String(), header.CID)
test_helpers.ExpectEqual(t, header.TD, mocks.MockBlock.Difficulty().String()) require.Equal(t, mocks.MockBlock.Difficulty().String(), header.TD)
test_helpers.ExpectEqual(t, header.Reward, "2000000000000021250") require.Equal(t, "2000000000000021250", header.Reward)
test_helpers.ExpectEqual(t, header.Coinbase, mocks.MockHeader.Coinbase.String()) require.Equal(t, mocks.MockHeader.Coinbase.String(), header.Coinbase)
dc, err := cid.Decode(header.CID) dc, err := cid.Decode(header.CID)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@ -98,11 +98,11 @@ func TestSQLXIndexer(t *testing.T) {
mhKey := dshelp.MultihashToDsKey(dc.Hash()) mhKey := dshelp.MultihashToDsKey(dc.Hash())
prefixedKey := blockstore.BlockPrefix.String() + mhKey.String() prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
var data []byte var data []byte
err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey) err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64())
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
test_helpers.ExpectEqual(t, data, mocks.MockHeaderRlp) require.Equal(t, mocks.MockHeaderRlp, data)
}) })
t.Run("Publish and index transaction IPLDs in a single tx", func(t *testing.T) { t.Run("Publish and index transaction IPLDs in a single tx", func(t *testing.T) {
@ -117,7 +117,7 @@ func TestSQLXIndexer(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
test_helpers.ExpectEqual(t, len(trxs), 5) require.Equal(t, 5, len(trxs))
expectTrue(t, test_helpers.ListContainsString(trxs, trx1CID.String())) expectTrue(t, test_helpers.ListContainsString(trxs, trx1CID.String()))
expectTrue(t, test_helpers.ListContainsString(trxs, trx2CID.String())) expectTrue(t, test_helpers.ListContainsString(trxs, trx2CID.String()))
expectTrue(t, test_helpers.ListContainsString(trxs, trx3CID.String())) expectTrue(t, test_helpers.ListContainsString(trxs, trx3CID.String()))
@ -137,14 +137,14 @@ func TestSQLXIndexer(t *testing.T) {
mhKey := dshelp.MultihashToDsKey(dc.Hash()) mhKey := dshelp.MultihashToDsKey(dc.Hash())
prefixedKey := blockstore.BlockPrefix.String() + mhKey.String() prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
var data []byte var data []byte
err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey) err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64())
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
txTypeAndValueStr := `SELECT tx_type, value FROM eth.transaction_cids WHERE cid = $1` txTypeAndValueStr := `SELECT tx_type, value FROM eth.transaction_cids WHERE cid = $1`
switch c { switch c {
case trx1CID.String(): case trx1CID.String():
test_helpers.ExpectEqual(t, data, tx1) require.Equal(t, tx1, data)
txRes := new(txResult) txRes := new(txResult)
err = db.QueryRow(context.Background(), txTypeAndValueStr, c).(*sqlx.Row).StructScan(txRes) err = db.QueryRow(context.Background(), txTypeAndValueStr, c).(*sqlx.Row).StructScan(txRes)
if err != nil { if err != nil {
@ -157,7 +157,7 @@ func TestSQLXIndexer(t *testing.T) {
t.Fatalf("expected tx value %s got %s", transactions[0].Value().String(), txRes.Value) t.Fatalf("expected tx value %s got %s", transactions[0].Value().String(), txRes.Value)
} }
case trx2CID.String(): case trx2CID.String():
test_helpers.ExpectEqual(t, data, tx2) require.Equal(t, tx2, data)
txRes := new(txResult) txRes := new(txResult)
err = db.QueryRow(context.Background(), txTypeAndValueStr, c).(*sqlx.Row).StructScan(txRes) err = db.QueryRow(context.Background(), txTypeAndValueStr, c).(*sqlx.Row).StructScan(txRes)
if err != nil { if err != nil {
@ -170,7 +170,7 @@ func TestSQLXIndexer(t *testing.T) {
t.Fatalf("expected tx value %s got %s", transactions[1].Value().String(), txRes.Value) t.Fatalf("expected tx value %s got %s", transactions[1].Value().String(), txRes.Value)
} }
case trx3CID.String(): case trx3CID.String():
test_helpers.ExpectEqual(t, data, tx3) require.Equal(t, tx3, data)
txRes := new(txResult) txRes := new(txResult)
err = db.QueryRow(context.Background(), txTypeAndValueStr, c).(*sqlx.Row).StructScan(txRes) err = db.QueryRow(context.Background(), txTypeAndValueStr, c).(*sqlx.Row).StructScan(txRes)
if err != nil { if err != nil {
@ -183,7 +183,7 @@ func TestSQLXIndexer(t *testing.T) {
t.Fatalf("expected tx value %s got %s", transactions[2].Value().String(), txRes.Value) t.Fatalf("expected tx value %s got %s", transactions[2].Value().String(), txRes.Value)
} }
case trx4CID.String(): case trx4CID.String():
test_helpers.ExpectEqual(t, data, tx4) require.Equal(t, tx4, data)
txRes := new(txResult) txRes := new(txResult)
err = db.QueryRow(context.Background(), txTypeAndValueStr, c).(*sqlx.Row).StructScan(txRes) err = db.QueryRow(context.Background(), txTypeAndValueStr, c).(*sqlx.Row).StructScan(txRes)
if err != nil { if err != nil {
@ -196,7 +196,9 @@ func TestSQLXIndexer(t *testing.T) {
t.Fatalf("expected tx value %s got %s", transactions[3].Value().String(), txRes.Value) t.Fatalf("expected tx value %s got %s", transactions[3].Value().String(), txRes.Value)
} }
accessListElementModels := make([]models.AccessListElementModel, 0) accessListElementModels := make([]models.AccessListElementModel, 0)
pgStr = `SELECT access_list_elements.* FROM eth.access_list_elements INNER JOIN eth.transaction_cids ON (tx_id = transaction_cids.tx_hash) WHERE cid = $1 ORDER BY access_list_elements.index ASC` pgStr = "SELECT cast(access_list_elements.block_number AS TEXT), access_list_elements.index, access_list_elements.tx_id, " +
"access_list_elements.address, access_list_elements.storage_keys FROM eth.access_list_elements " +
"INNER JOIN eth.transaction_cids ON (tx_id = transaction_cids.tx_hash) WHERE cid = $1 ORDER BY access_list_elements.index ASC"
err = db.Select(context.Background(), &accessListElementModels, pgStr, c) err = db.Select(context.Background(), &accessListElementModels, pgStr, c)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
@ -205,18 +207,20 @@ func TestSQLXIndexer(t *testing.T) {
t.Fatalf("expected two access list entries, got %d", len(accessListElementModels)) t.Fatalf("expected two access list entries, got %d", len(accessListElementModels))
} }
model1 := models.AccessListElementModel{ model1 := models.AccessListElementModel{
Index: accessListElementModels[0].Index, BlockNumber: mocks.BlockNumber.String(),
Address: accessListElementModels[0].Address, Index: accessListElementModels[0].Index,
Address: accessListElementModels[0].Address,
} }
model2 := models.AccessListElementModel{ model2 := models.AccessListElementModel{
BlockNumber: mocks.BlockNumber.String(),
Index: accessListElementModels[1].Index, Index: accessListElementModels[1].Index,
Address: accessListElementModels[1].Address, Address: accessListElementModels[1].Address,
StorageKeys: accessListElementModels[1].StorageKeys, StorageKeys: accessListElementModels[1].StorageKeys,
} }
test_helpers.ExpectEqual(t, model1, mocks.AccessListEntry1Model) require.Equal(t, mocks.AccessListEntry1Model, model1)
test_helpers.ExpectEqual(t, model2, mocks.AccessListEntry2Model) require.Equal(t, mocks.AccessListEntry2Model, model2)
case trx5CID.String(): case trx5CID.String():
test_helpers.ExpectEqual(t, data, tx5) require.Equal(t, tx5, data)
txRes := new(txResult) txRes := new(txResult)
err = db.QueryRow(context.Background(), txTypeAndValueStr, c).(*sqlx.Row).StructScan(txRes) err = db.QueryRow(context.Background(), txTypeAndValueStr, c).(*sqlx.Row).StructScan(txRes)
if err != nil { if err != nil {
@ -266,7 +270,7 @@ func TestSQLXIndexer(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
expectedLogs := mocks.MockReceipts[i].Logs expectedLogs := mocks.MockReceipts[i].Logs
test_helpers.ExpectEqual(t, len(results), len(expectedLogs)) require.Equal(t, len(expectedLogs), len(results))
var nodeElements []interface{} var nodeElements []interface{}
for idx, r := range results { for idx, r := range results {
@ -277,12 +281,12 @@ func TestSQLXIndexer(t *testing.T) {
logRaw, err := rlp.EncodeToBytes(expectedLogs[idx]) logRaw, err := rlp.EncodeToBytes(expectedLogs[idx])
require.NoError(t, err) require.NoError(t, err)
// 2nd element of the leaf node contains the encoded log data. // 2nd element of the leaf node contains the encoded log data.
test_helpers.ExpectEqual(t, logRaw, nodeElements[1].([]byte)) require.Equal(t, nodeElements[1].([]byte), logRaw)
} else { } else {
logRaw, err := rlp.EncodeToBytes(expectedLogs[idx]) logRaw, err := rlp.EncodeToBytes(expectedLogs[idx])
require.NoError(t, err) require.NoError(t, err)
// raw log was IPLDized // raw log was IPLDized
test_helpers.ExpectEqual(t, logRaw, r.Data) require.Equal(t, r.Data, logRaw)
} }
} }
} }
@ -303,7 +307,7 @@ func TestSQLXIndexer(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
test_helpers.ExpectEqual(t, len(rcts), 5) require.Equal(t, 5, len(rcts))
expectTrue(t, test_helpers.ListContainsString(rcts, rct1CID.String())) expectTrue(t, test_helpers.ListContainsString(rcts, rct1CID.String()))
expectTrue(t, test_helpers.ListContainsString(rcts, rct2CID.String())) expectTrue(t, test_helpers.ListContainsString(rcts, rct2CID.String()))
expectTrue(t, test_helpers.ListContainsString(rcts, rct3CID.String())) expectTrue(t, test_helpers.ListContainsString(rcts, rct3CID.String()))
@ -329,7 +333,7 @@ func TestSQLXIndexer(t *testing.T) {
expectedRct, err := mocks.MockReceipts[idx].MarshalBinary() expectedRct, err := mocks.MockReceipts[idx].MarshalBinary()
require.NoError(t, err) require.NoError(t, err)
test_helpers.ExpectEqual(t, expectedRct, nodeElements[1].([]byte)) require.Equal(t, nodeElements[1].([]byte), expectedRct)
dc, err := cid.Decode(c) dc, err := cid.Decode(c)
if err != nil { if err != nil {
@ -338,53 +342,53 @@ func TestSQLXIndexer(t *testing.T) {
mhKey := dshelp.MultihashToDsKey(dc.Hash()) mhKey := dshelp.MultihashToDsKey(dc.Hash())
prefixedKey := blockstore.BlockPrefix.String() + mhKey.String() prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
var data []byte var data []byte
err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey) err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64())
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
postStatePgStr := `SELECT post_state FROM eth.receipt_cids WHERE leaf_cid = $1` postStatePgStr := `SELECT post_state FROM eth.receipt_cids WHERE leaf_cid = $1`
switch c { switch c {
case rct1CID.String(): case rct1CID.String():
test_helpers.ExpectEqual(t, data, rctLeaf1) require.Equal(t, rctLeaf1, data)
var postStatus uint64 var postStatus uint64
pgStr = `SELECT post_status FROM eth.receipt_cids WHERE leaf_cid = $1` pgStr = `SELECT post_status FROM eth.receipt_cids WHERE leaf_cid = $1`
err = db.Get(context.Background(), &postStatus, pgStr, c) err = db.Get(context.Background(), &postStatus, pgStr, c)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
test_helpers.ExpectEqual(t, postStatus, mocks.ExpectedPostStatus) require.Equal(t, mocks.ExpectedPostStatus, postStatus)
case rct2CID.String(): case rct2CID.String():
test_helpers.ExpectEqual(t, data, rctLeaf2) require.Equal(t, rctLeaf2, data)
var postState string var postState string
err = db.Get(context.Background(), &postState, postStatePgStr, c) err = db.Get(context.Background(), &postState, postStatePgStr, c)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
test_helpers.ExpectEqual(t, postState, mocks.ExpectedPostState1) require.Equal(t, mocks.ExpectedPostState1, postState)
case rct3CID.String(): case rct3CID.String():
test_helpers.ExpectEqual(t, data, rctLeaf3) require.Equal(t, rctLeaf3, data)
var postState string var postState string
err = db.Get(context.Background(), &postState, postStatePgStr, c) err = db.Get(context.Background(), &postState, postStatePgStr, c)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
test_helpers.ExpectEqual(t, postState, mocks.ExpectedPostState2) require.Equal(t, mocks.ExpectedPostState2, postState)
case rct4CID.String(): case rct4CID.String():
test_helpers.ExpectEqual(t, data, rctLeaf4) require.Equal(t, rctLeaf4, data)
var postState string var postState string
err = db.Get(context.Background(), &postState, postStatePgStr, c) err = db.Get(context.Background(), &postState, postStatePgStr, c)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
test_helpers.ExpectEqual(t, postState, mocks.ExpectedPostState3) require.Equal(t, mocks.ExpectedPostState3, postState)
case rct5CID.String(): case rct5CID.String():
test_helpers.ExpectEqual(t, data, rctLeaf5) require.Equal(t, rctLeaf5, data)
var postState string var postState string
err = db.Get(context.Background(), &postState, postStatePgStr, c) err = db.Get(context.Background(), &postState, postStatePgStr, c)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
test_helpers.ExpectEqual(t, postState, mocks.ExpectedPostState3) require.Equal(t, mocks.ExpectedPostState3, postState)
} }
} }
}) })
@ -402,7 +406,7 @@ func TestSQLXIndexer(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
test_helpers.ExpectEqual(t, len(stateNodes), 2) require.Equal(t, 2, len(stateNodes))
for _, stateNode := range stateNodes { for _, stateNode := range stateNodes {
var data []byte var data []byte
dc, err := cid.Decode(stateNode.CID) dc, err := cid.Decode(stateNode.CID)
@ -411,7 +415,7 @@ func TestSQLXIndexer(t *testing.T) {
} }
mhKey := dshelp.MultihashToDsKey(dc.Hash()) mhKey := dshelp.MultihashToDsKey(dc.Hash())
prefixedKey := blockstore.BlockPrefix.String() + mhKey.String() prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey) err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64())
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
@ -422,32 +426,34 @@ func TestSQLXIndexer(t *testing.T) {
t.Fatal(err) t.Fatal(err)
} }
if stateNode.CID == state1CID.String() { if stateNode.CID == state1CID.String() {
test_helpers.ExpectEqual(t, stateNode.NodeType, 2) require.Equal(t, 2, stateNode.NodeType)
test_helpers.ExpectEqual(t, stateNode.StateKey, common.BytesToHash(mocks.ContractLeafKey).Hex()) require.Equal(t, common.BytesToHash(mocks.ContractLeafKey).Hex(), stateNode.StateKey)
test_helpers.ExpectEqual(t, stateNode.Path, []byte{'\x06'}) require.Equal(t, []byte{'\x06'}, stateNode.Path)
test_helpers.ExpectEqual(t, data, mocks.ContractLeafNode) require.Equal(t, mocks.ContractLeafNode, data)
test_helpers.ExpectEqual(t, account, models.StateAccountModel{ require.Equal(t, models.StateAccountModel{
BlockNumber: mocks.BlockNumber.String(),
HeaderID: account.HeaderID, HeaderID: account.HeaderID,
StatePath: stateNode.Path, StatePath: stateNode.Path,
Balance: "0", Balance: "0",
CodeHash: mocks.ContractCodeHash.Bytes(), CodeHash: mocks.ContractCodeHash.Bytes(),
StorageRoot: mocks.ContractRoot, StorageRoot: mocks.ContractRoot,
Nonce: 1, Nonce: 1,
}) }, account)
} }
if stateNode.CID == state2CID.String() { if stateNode.CID == state2CID.String() {
test_helpers.ExpectEqual(t, stateNode.NodeType, 2) require.Equal(t, 2, stateNode.NodeType)
test_helpers.ExpectEqual(t, stateNode.StateKey, common.BytesToHash(mocks.AccountLeafKey).Hex()) require.Equal(t, common.BytesToHash(mocks.AccountLeafKey).Hex(), stateNode.StateKey)
test_helpers.ExpectEqual(t, stateNode.Path, []byte{'\x0c'}) require.Equal(t, []byte{'\x0c'}, stateNode.Path)
test_helpers.ExpectEqual(t, data, mocks.AccountLeafNode) require.Equal(t, mocks.AccountLeafNode, data)
test_helpers.ExpectEqual(t, account, models.StateAccountModel{ require.Equal(t, models.StateAccountModel{
BlockNumber: mocks.BlockNumber.String(),
HeaderID: account.HeaderID, HeaderID: account.HeaderID,
StatePath: stateNode.Path, StatePath: stateNode.Path,
Balance: "1000", Balance: "1000",
CodeHash: mocks.AccountCodeHash.Bytes(), CodeHash: mocks.AccountCodeHash.Bytes(),
StorageRoot: mocks.AccountRoot, StorageRoot: mocks.AccountRoot,
Nonce: 0, Nonce: 0,
}) }, account)
} }
} }
@ -460,7 +466,7 @@ func TestSQLXIndexer(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
test_helpers.ExpectEqual(t, len(stateNodes), 1) require.Equal(t, 1, len(stateNodes))
stateNode := stateNodes[0] stateNode := stateNodes[0]
var data []byte var data []byte
dc, err := cid.Decode(stateNode.CID) dc, err := cid.Decode(stateNode.CID)
@ -469,14 +475,14 @@ func TestSQLXIndexer(t *testing.T) {
} }
mhKey := dshelp.MultihashToDsKey(dc.Hash()) mhKey := dshelp.MultihashToDsKey(dc.Hash())
prefixedKey := blockstore.BlockPrefix.String() + mhKey.String() prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
test_helpers.ExpectEqual(t, prefixedKey, shared.RemovedNodeMhKey) require.Equal(t, shared.RemovedNodeMhKey, prefixedKey)
err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey) err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64())
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
test_helpers.ExpectEqual(t, stateNode.CID, shared.RemovedNodeStateCID) require.Equal(t, shared.RemovedNodeStateCID, stateNode.CID)
test_helpers.ExpectEqual(t, stateNode.Path, []byte{'\x02'}) require.Equal(t, []byte{'\x02'}, stateNode.Path)
test_helpers.ExpectEqual(t, data, []byte{}) require.Equal(t, []byte{}, data)
}) })
t.Run("Publish and index storage IPLDs in a single tx", func(t *testing.T) { t.Run("Publish and index storage IPLDs in a single tx", func(t *testing.T) {
@ -485,7 +491,7 @@ func TestSQLXIndexer(t *testing.T) {
defer checkTxClosure(t, 0, 0, 0) defer checkTxClosure(t, 0, 0, 0)
// check that storage nodes were properly indexed // check that storage nodes were properly indexed
storageNodes := make([]models.StorageNodeWithStateKeyModel, 0) storageNodes := make([]models.StorageNodeWithStateKeyModel, 0)
pgStr := `SELECT storage_cids.cid, state_cids.state_leaf_key, storage_cids.storage_leaf_key, storage_cids.node_type, storage_cids.storage_path pgStr := `SELECT cast(storage_cids.block_number AS TEXT), storage_cids.cid, state_cids.state_leaf_key, storage_cids.storage_leaf_key, storage_cids.node_type, storage_cids.storage_path
FROM eth.storage_cids, eth.state_cids, eth.header_cids FROM eth.storage_cids, eth.state_cids, eth.header_cids
WHERE (storage_cids.state_path, storage_cids.header_id) = (state_cids.state_path, state_cids.header_id) WHERE (storage_cids.state_path, storage_cids.header_id) = (state_cids.state_path, state_cids.header_id)
AND state_cids.header_id = header_cids.block_hash AND state_cids.header_id = header_cids.block_hash
@ -495,14 +501,15 @@ func TestSQLXIndexer(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
test_helpers.ExpectEqual(t, len(storageNodes), 1) require.Equal(t, 1, len(storageNodes))
test_helpers.ExpectEqual(t, storageNodes[0], models.StorageNodeWithStateKeyModel{ require.Equal(t, models.StorageNodeWithStateKeyModel{
CID: storageCID.String(), BlockNumber: mocks.BlockNumber.String(),
NodeType: 2, CID: storageCID.String(),
StorageKey: common.BytesToHash(mocks.StorageLeafKey).Hex(), NodeType: 2,
StateKey: common.BytesToHash(mocks.ContractLeafKey).Hex(), StorageKey: common.BytesToHash(mocks.StorageLeafKey).Hex(),
Path: []byte{}, StateKey: common.BytesToHash(mocks.ContractLeafKey).Hex(),
}) Path: []byte{},
}, storageNodes[0])
var data []byte var data []byte
dc, err := cid.Decode(storageNodes[0].CID) dc, err := cid.Decode(storageNodes[0].CID)
if err != nil { if err != nil {
@ -510,15 +517,15 @@ func TestSQLXIndexer(t *testing.T) {
} }
mhKey := dshelp.MultihashToDsKey(dc.Hash()) mhKey := dshelp.MultihashToDsKey(dc.Hash())
prefixedKey := blockstore.BlockPrefix.String() + mhKey.String() prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey) err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64())
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
test_helpers.ExpectEqual(t, data, mocks.StorageLeafNode) require.Equal(t, mocks.StorageLeafNode, data)
// check that Removed storage nodes were properly indexed // check that Removed storage nodes were properly indexed
storageNodes = make([]models.StorageNodeWithStateKeyModel, 0) storageNodes = make([]models.StorageNodeWithStateKeyModel, 0)
pgStr = `SELECT storage_cids.cid, state_cids.state_leaf_key, storage_cids.storage_leaf_key, storage_cids.node_type, storage_cids.storage_path pgStr = `SELECT cast(storage_cids.block_number AS TEXT), storage_cids.cid, state_cids.state_leaf_key, storage_cids.storage_leaf_key, storage_cids.node_type, storage_cids.storage_path
FROM eth.storage_cids, eth.state_cids, eth.header_cids FROM eth.storage_cids, eth.state_cids, eth.header_cids
WHERE (storage_cids.state_path, storage_cids.header_id) = (state_cids.state_path, state_cids.header_id) WHERE (storage_cids.state_path, storage_cids.header_id) = (state_cids.state_path, state_cids.header_id)
AND state_cids.header_id = header_cids.block_hash AND state_cids.header_id = header_cids.block_hash
@ -528,25 +535,26 @@ func TestSQLXIndexer(t *testing.T) {
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
test_helpers.ExpectEqual(t, len(storageNodes), 1) require.Equal(t, 1, len(storageNodes))
test_helpers.ExpectEqual(t, storageNodes[0], models.StorageNodeWithStateKeyModel{ require.Equal(t, models.StorageNodeWithStateKeyModel{
CID: shared.RemovedNodeStorageCID, BlockNumber: mocks.BlockNumber.String(),
NodeType: 3, CID: shared.RemovedNodeStorageCID,
StorageKey: common.BytesToHash(mocks.RemovedLeafKey).Hex(), NodeType: 3,
StateKey: common.BytesToHash(mocks.ContractLeafKey).Hex(), StorageKey: common.BytesToHash(mocks.RemovedLeafKey).Hex(),
Path: []byte{'\x03'}, StateKey: common.BytesToHash(mocks.ContractLeafKey).Hex(),
}) Path: []byte{'\x03'},
}, storageNodes[0])
dc, err = cid.Decode(storageNodes[0].CID) dc, err = cid.Decode(storageNodes[0].CID)
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
mhKey = dshelp.MultihashToDsKey(dc.Hash()) mhKey = dshelp.MultihashToDsKey(dc.Hash())
prefixedKey = blockstore.BlockPrefix.String() + mhKey.String() prefixedKey = blockstore.BlockPrefix.String() + mhKey.String()
test_helpers.ExpectEqual(t, prefixedKey, shared.RemovedNodeMhKey) require.Equal(t, shared.RemovedNodeMhKey, prefixedKey)
err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey) err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64())
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
test_helpers.ExpectEqual(t, data, []byte{}) require.Equal(t, []byte{}, data)
}) })
} }

47
statediff/indexer/database/sql/writer.go
View File

@ -62,12 +62,12 @@ func (w *Writer) upsertHeaderCID(tx Tx, header models.HeaderModel) error {
} }
/* /*
INSERT INTO eth.uncle_cids (block_hash, header_id, parent_hash, cid, reward, mh_key) VALUES ($1, $2, $3, $4, $5, $6) INSERT INTO eth.uncle_cids (block_number, block_hash, header_id, parent_hash, cid, reward, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7)
ON CONFLICT (block_hash) DO NOTHING ON CONFLICT (block_hash) DO NOTHING
*/ */
func (w *Writer) upsertUncleCID(tx Tx, uncle models.UncleModel) error { func (w *Writer) upsertUncleCID(tx Tx, uncle models.UncleModel) error {
_, err := tx.Exec(w.db.Context(), w.db.InsertUncleStm(), _, err := tx.Exec(w.db.Context(), w.db.InsertUncleStm(),
uncle.BlockHash, uncle.HeaderID, uncle.ParentHash, uncle.CID, uncle.Reward, uncle.MhKey) uncle.BlockNumber, uncle.BlockHash, uncle.HeaderID, uncle.ParentHash, uncle.CID, uncle.Reward, uncle.MhKey)
if err != nil { if err != nil {
return fmt.Errorf("error upserting uncle_cids entry: %v", err) return fmt.Errorf("error upserting uncle_cids entry: %v", err)
} }
@ -75,13 +75,13 @@ func (w *Writer) upsertUncleCID(tx Tx, uncle models.UncleModel) error {
} }
/* /*
INSERT INTO eth.transaction_cids (header_id, tx_hash, cid, dst, src, index, mh_key, tx_data, tx_type, value) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) INSERT INTO eth.transaction_cids (block_number, header_id, tx_hash, cid, dst, src, index, mh_key, tx_data, tx_type, value) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
ON CONFLICT (tx_hash) DO NOTHING ON CONFLICT (tx_hash) DO NOTHING
*/ */
func (w *Writer) upsertTransactionCID(tx Tx, transaction models.TxModel) error { func (w *Writer) upsertTransactionCID(tx Tx, transaction models.TxModel) error {
_, err := tx.Exec(w.db.Context(), w.db.InsertTxStm(), _, err := tx.Exec(w.db.Context(), w.db.InsertTxStm(),
transaction.HeaderID, transaction.TxHash, transaction.CID, transaction.Dst, transaction.Src, transaction.Index, transaction.BlockNumber, transaction.HeaderID, transaction.TxHash, transaction.CID, transaction.Dst, transaction.Src,
transaction.MhKey, transaction.Data, transaction.Type, transaction.Value) transaction.Index, transaction.MhKey, transaction.Data, transaction.Type, transaction.Value)
if err != nil { if err != nil {
return fmt.Errorf("error upserting transaction_cids entry: %v", err) return fmt.Errorf("error upserting transaction_cids entry: %v", err)
} }
@ -90,12 +90,13 @@ func (w *Writer) upsertTransactionCID(tx Tx, transaction models.TxModel) error {
} }
/* /*
INSERT INTO eth.access_list_elements (tx_id, index, address, storage_keys) VALUES ($1, $2, $3, $4) INSERT INTO eth.access_list_elements (block_number, tx_id, index, address, storage_keys) VALUES ($1, $2, $3, $4, $5)
ON CONFLICT (tx_id, index) DO NOTHING ON CONFLICT (tx_id, index) DO NOTHING
*/ */
func (w *Writer) upsertAccessListElement(tx Tx, accessListElement models.AccessListElementModel) error { func (w *Writer) upsertAccessListElement(tx Tx, accessListElement models.AccessListElementModel) error {
_, err := tx.Exec(w.db.Context(), w.db.InsertAccessListElementStm(), _, err := tx.Exec(w.db.Context(), w.db.InsertAccessListElementStm(),
accessListElement.TxID, accessListElement.Index, accessListElement.Address, accessListElement.StorageKeys) accessListElement.BlockNumber, accessListElement.TxID, accessListElement.Index, accessListElement.Address,
accessListElement.StorageKeys)
if err != nil { if err != nil {
return fmt.Errorf("error upserting access_list_element entry: %v", err) return fmt.Errorf("error upserting access_list_element entry: %v", err)
} }
@ -104,12 +105,13 @@ func (w *Writer) upsertAccessListElement(tx Tx, accessListElement models.AccessL
} }
/* /*
INSERT INTO eth.receipt_cids (tx_id, leaf_cid, contract, contract_hash, leaf_mh_key, post_state, post_status, log_root) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) INSERT INTO eth.receipt_cids (block_number, tx_id, leaf_cid, contract, contract_hash, leaf_mh_key, post_state, post_status, log_root) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
ON CONFLICT (tx_id) DO NOTHING ON CONFLICT (tx_id) DO NOTHING
*/ */
func (w *Writer) upsertReceiptCID(tx Tx, rct *models.ReceiptModel) error { func (w *Writer) upsertReceiptCID(tx Tx, rct *models.ReceiptModel) error {
_, err := tx.Exec(w.db.Context(), w.db.InsertRctStm(), _, err := tx.Exec(w.db.Context(), w.db.InsertRctStm(),
rct.TxID, rct.LeafCID, rct.Contract, rct.ContractHash, rct.LeafMhKey, rct.PostState, rct.PostStatus, rct.LogRoot) rct.BlockNumber, rct.TxID, rct.LeafCID, rct.Contract, rct.ContractHash, rct.LeafMhKey, rct.PostState,
rct.PostStatus, rct.LogRoot)
if err != nil { if err != nil {
return fmt.Errorf("error upserting receipt_cids entry: %w", err) return fmt.Errorf("error upserting receipt_cids entry: %w", err)
} }
@ -118,14 +120,14 @@ func (w *Writer) upsertReceiptCID(tx Tx, rct *models.ReceiptModel) error {
} }
/* /*
INSERT INTO eth.log_cids (leaf_cid, leaf_mh_key, rct_id, address, index, topic0, topic1, topic2, topic3, log_data) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10) INSERT INTO eth.log_cids (block_number, leaf_cid, leaf_mh_key, rct_id, address, index, topic0, topic1, topic2, topic3, log_data) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
ON CONFLICT (rct_id, index) DO NOTHING ON CONFLICT (rct_id, index) DO NOTHING
*/ */
func (w *Writer) upsertLogCID(tx Tx, logs []*models.LogsModel) error { func (w *Writer) upsertLogCID(tx Tx, logs []*models.LogsModel) error {
for _, log := range logs { for _, log := range logs {
_, err := tx.Exec(w.db.Context(), w.db.InsertLogStm(), _, err := tx.Exec(w.db.Context(), w.db.InsertLogStm(),
log.LeafCID, log.LeafMhKey, log.ReceiptID, log.Address, log.Index, log.Topic0, log.Topic1, log.Topic2, log.BlockNumber, log.LeafCID, log.LeafMhKey, log.ReceiptID, log.Address, log.Index, log.Topic0, log.Topic1,
log.Topic3, log.Data) log.Topic2, log.Topic3, log.Data)
if err != nil { if err != nil {
return fmt.Errorf("error upserting logs entry: %w", err) return fmt.Errorf("error upserting logs entry: %w", err)
} }
@ -135,8 +137,8 @@ func (w *Writer) upsertLogCID(tx Tx, logs []*models.LogsModel) error {
} }
/* /*
INSERT INTO eth.state_cids (header_id, state_leaf_key, cid, state_path, node_type, diff, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7) INSERT INTO eth.state_cids (block_number, header_id, state_leaf_key, cid, state_path, node_type, diff, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
ON CONFLICT (header_id, state_path) DO UPDATE SET (state_leaf_key, cid, node_type, diff, mh_key) = ($2, $3, $5, $6, $7) ON CONFLICT (header_id, state_path) DO UPDATE SET (block_number, state_leaf_key, cid, node_type, diff, mh_key) = ($1 $3, $4, $6, $7, $8)
*/ */
func (w *Writer) upsertStateCID(tx Tx, stateNode models.StateNodeModel) error { func (w *Writer) upsertStateCID(tx Tx, stateNode models.StateNodeModel) error {
var stateKey string var stateKey string
@ -144,7 +146,8 @@ func (w *Writer) upsertStateCID(tx Tx, stateNode models.StateNodeModel) error {
stateKey = stateNode.StateKey stateKey = stateNode.StateKey
} }
_, err := tx.Exec(w.db.Context(), w.db.InsertStateStm(), _, err := tx.Exec(w.db.Context(), w.db.InsertStateStm(),
stateNode.HeaderID, stateKey, stateNode.CID, stateNode.Path, stateNode.NodeType, true, stateNode.MhKey) stateNode.BlockNumber, stateNode.HeaderID, stateKey, stateNode.CID, stateNode.Path, stateNode.NodeType, true,
stateNode.MhKey)
if err != nil { if err != nil {
return fmt.Errorf("error upserting state_cids entry: %v", err) return fmt.Errorf("error upserting state_cids entry: %v", err)
} }
@ -152,13 +155,13 @@ func (w *Writer) upsertStateCID(tx Tx, stateNode models.StateNodeModel) error {
} }
/* /*
INSERT INTO eth.state_accounts (header_id, state_path, balance, nonce, code_hash, storage_root) VALUES ($1, $2, $3, $4, $5, $6) INSERT INTO eth.state_accounts (block_number, header_id, state_path, balance, nonce, code_hash, storage_root) VALUES ($1, $2, $3, $4, $5, $6, $7)
ON CONFLICT (header_id, state_path) DO NOTHING ON CONFLICT (header_id, state_path) DO NOTHING
*/ */
func (w *Writer) upsertStateAccount(tx Tx, stateAccount models.StateAccountModel) error { func (w *Writer) upsertStateAccount(tx Tx, stateAccount models.StateAccountModel) error {
_, err := tx.Exec(w.db.Context(), w.db.InsertAccountStm(), _, err := tx.Exec(w.db.Context(), w.db.InsertAccountStm(),
stateAccount.HeaderID, stateAccount.StatePath, stateAccount.Balance, stateAccount.Nonce, stateAccount.CodeHash, stateAccount.BlockNumber, stateAccount.HeaderID, stateAccount.StatePath, stateAccount.Balance,
stateAccount.StorageRoot) stateAccount.Nonce, stateAccount.CodeHash, stateAccount.StorageRoot)
if err != nil { if err != nil {
return fmt.Errorf("error upserting state_accounts entry: %v", err) return fmt.Errorf("error upserting state_accounts entry: %v", err)
} }
@ -166,8 +169,8 @@ func (w *Writer) upsertStateAccount(tx Tx, stateAccount models.StateAccountModel
} }
/* /*
INSERT INTO eth.storage_cids (header_id, state_path, storage_leaf_key, cid, storage_path, node_type, diff, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) INSERT INTO eth.storage_cids (block_number, header_id, state_path, storage_leaf_key, cid, storage_path, node_type, diff, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
ON CONFLICT (header_id, state_path, storage_path) DO UPDATE SET (storage_leaf_key, cid, node_type, diff, mh_key) = ($3, $4, $6, $7, $8) ON CONFLICT (header_id, state_path, storage_path) DO UPDATE SET (block_number, storage_leaf_key, cid, node_type, diff, mh_key) = ($1, $4, $5, $7, $8, $9)
*/ */
func (w *Writer) upsertStorageCID(tx Tx, storageCID models.StorageNodeModel) error { func (w *Writer) upsertStorageCID(tx Tx, storageCID models.StorageNodeModel) error {
var storageKey string var storageKey string
@ -175,8 +178,8 @@ func (w *Writer) upsertStorageCID(tx Tx, storageCID models.StorageNodeModel) err
storageKey = storageCID.StorageKey storageKey = storageCID.StorageKey
} }
_, err := tx.Exec(w.db.Context(), w.db.InsertStorageStm(), _, err := tx.Exec(w.db.Context(), w.db.InsertStorageStm(),
storageCID.HeaderID, storageCID.StatePath, storageKey, storageCID.CID, storageCID.Path, storageCID.NodeType, storageCID.BlockNumber, storageCID.HeaderID, storageCID.StatePath, storageKey, storageCID.CID, storageCID.Path,
true, storageCID.MhKey) storageCID.NodeType, true, storageCID.MhKey)
if err != nil { if err != nil {
return fmt.Errorf("error upserting storage_cids entry: %v", err) return fmt.Errorf("error upserting storage_cids entry: %v", err)
} }

6
statediff/indexer/mocks/test_data.go
View File

@ -106,10 +106,12 @@ var (
StorageKeys: []common.Hash{common.BytesToHash(StorageLeafKey), common.BytesToHash(MockStorageLeafKey)}, StorageKeys: []common.Hash{common.BytesToHash(StorageLeafKey), common.BytesToHash(MockStorageLeafKey)},
} }
AccessListEntry1Model = models.AccessListElementModel{ AccessListEntry1Model = models.AccessListElementModel{
Index: 0, BlockNumber: BlockNumber.String(),
Address: Address.Hex(), Index: 0,
Address: Address.Hex(),
} }
AccessListEntry2Model = models.AccessListElementModel{ AccessListEntry2Model = models.AccessListElementModel{
BlockNumber: BlockNumber.String(),
Index: 1, Index: 1,
Address: AnotherAddress.Hex(), Address: AnotherAddress.Hex(),
StorageKeys: []string{common.BytesToHash(StorageLeafKey).Hex(), common.BytesToHash(MockStorageLeafKey).Hex()}, StorageKeys: []string{common.BytesToHash(StorageLeafKey).Hex(), common.BytesToHash(MockStorageLeafKey).Hex()},

81
statediff/indexer/models/batch.go
View File

@ -20,12 +20,14 @@ import "github.com/lib/pq"
// IPLDBatch holds the arguments for a batch insert of IPLD data // IPLDBatch holds the arguments for a batch insert of IPLD data
type IPLDBatch struct { type IPLDBatch struct {
Keys []string BlockNumbers []string
Values [][]byte Keys []string
Values [][]byte
} }
// UncleBatch holds the arguments for a batch insert of uncle data // UncleBatch holds the arguments for a batch insert of uncle data
type UncleBatch struct { type UncleBatch struct {
BlockNumbers []string
HeaderID []string HeaderID []string
BlockHashes []string BlockHashes []string
ParentHashes []string ParentHashes []string
@ -36,19 +38,21 @@ type UncleBatch struct {
// TxBatch holds the arguments for a batch insert of tx data // TxBatch holds the arguments for a batch insert of tx data
type TxBatch struct { type TxBatch struct {
HeaderID string BlockNumbers []string
Indexes []int64 HeaderID string
TxHashes []string Indexes []int64
CIDs []string TxHashes []string
MhKeys []string CIDs []string
Dsts []string MhKeys []string
Srcs []string Dsts []string
Datas [][]byte Srcs []string
Types []uint8 Datas [][]byte
Types []uint8
} }
// AccessListBatch holds the arguments for a batch insert of access list data // AccessListBatch holds the arguments for a batch insert of access list data
type AccessListBatch struct { type AccessListBatch struct {
BlockNumbers []string
Indexes []int64 Indexes []int64
TxIDs []string TxIDs []string
Addresses []string Addresses []string
@ -57,6 +61,7 @@ type AccessListBatch struct {
// ReceiptBatch holds the arguments for a batch insert of receipt data // ReceiptBatch holds the arguments for a batch insert of receipt data
type ReceiptBatch struct { type ReceiptBatch struct {
BlockNumbers []string
TxIDs []string TxIDs []string
LeafCIDs []string LeafCIDs []string
LeafMhKeys []string LeafMhKeys []string
@ -69,31 +74,34 @@ type ReceiptBatch struct {
// LogBatch holds the arguments for a batch insert of log data // LogBatch holds the arguments for a batch insert of log data
type LogBatch struct { type LogBatch struct {
LeafCIDs []string BlockNumbers []string
LeafMhKeys []string LeafCIDs []string
ReceiptIDs []string LeafMhKeys []string
Addresses []string ReceiptIDs []string
Indexes []int64 Addresses []string
Datas [][]byte Indexes []int64
Topic0s []string Datas [][]byte
Topic1s []string Topic0s []string
Topic2s []string Topic1s []string
Topic3s []string Topic2s []string
Topic3s []string
} }
// StateBatch holds the arguments for a batch insert of state data // StateBatch holds the arguments for a batch insert of state data
type StateBatch struct { type StateBatch struct {
HeaderID string BlockNumbers []string
Paths [][]byte HeaderID string
StateKeys []string Paths [][]byte
NodeTypes []int StateKeys []string
CIDs []string NodeTypes []int
MhKeys []string CIDs []string
Diff bool MhKeys []string
Diff bool
} }
// AccountBatch holds the arguments for a batch insert of account data // AccountBatch holds the arguments for a batch insert of account data
type AccountBatch struct { type AccountBatch struct {
BlockNumbers []string
HeaderID string HeaderID string
StatePaths [][]byte StatePaths [][]byte
Balances []string Balances []string
@ -104,12 +112,13 @@ type AccountBatch struct {
// StorageBatch holds the arguments for a batch insert of storage data // StorageBatch holds the arguments for a batch insert of storage data
type StorageBatch struct { type StorageBatch struct {
HeaderID string BlockNumbers []string
StatePaths [][]string HeaderID string
Paths [][]byte StatePaths [][]string
StorageKeys []string Paths [][]byte
NodeTypes []int StorageKeys []string
CIDs []string NodeTypes []int
MhKeys []string CIDs []string
Diff bool MhKeys []string
Diff bool
} }

114
statediff/indexer/models/models.go
View File

@ -20,8 +20,9 @@ import "github.com/lib/pq"
// IPLDModel is the db model for public.blocks // IPLDModel is the db model for public.blocks
type IPLDModel struct { type IPLDModel struct {
Key string `db:"key"` BlockNumber string `db:"block_number"`
Data []byte `db:"data"` Key string `db:"key"`
Data []byte `db:"data"`
} }
// HeaderModel is the db model for eth.header_cids // HeaderModel is the db model for eth.header_cids
@ -46,30 +47,33 @@ type HeaderModel struct {
// UncleModel is the db model for eth.uncle_cids // UncleModel is the db model for eth.uncle_cids
type UncleModel struct { type UncleModel struct {
HeaderID string `db:"header_id"` BlockNumber string `db:"block_number"`
BlockHash string `db:"block_hash"` HeaderID string `db:"header_id"`
ParentHash string `db:"parent_hash"` BlockHash string `db:"block_hash"`
CID string `db:"cid"` ParentHash string `db:"parent_hash"`
MhKey string `db:"mh_key"` CID string `db:"cid"`
Reward string `db:"reward"` MhKey string `db:"mh_key"`
Reward string `db:"reward"`
} }
// TxModel is the db model for eth.transaction_cids // TxModel is the db model for eth.transaction_cids
type TxModel struct { type TxModel struct {
HeaderID string `db:"header_id"` BlockNumber string `db:"block_number"`
Index int64 `db:"index"` HeaderID string `db:"header_id"`
TxHash string `db:"tx_hash"` Index int64 `db:"index"`
CID string `db:"cid"` TxHash string `db:"tx_hash"`
MhKey string `db:"mh_key"` CID string `db:"cid"`
Dst string `db:"dst"` MhKey string `db:"mh_key"`
Src string `db:"src"` Dst string `db:"dst"`
Data []byte `db:"tx_data"` Src string `db:"src"`
Type uint8 `db:"tx_type"` Data []byte `db:"tx_data"`
Value string `db:"value"` Type uint8 `db:"tx_type"`
Value string `db:"value"`
} }
// AccessListElementModel is the db model for eth.access_list_entry // AccessListElementModel is the db model for eth.access_list_entry
type AccessListElementModel struct { type AccessListElementModel struct {
BlockNumber string `db:"block_number"`
Index int64 `db:"index"` Index int64 `db:"index"`
TxID string `db:"tx_id"` TxID string `db:"tx_id"`
Address string `db:"address"` Address string `db:"address"`
@ -78,6 +82,7 @@ type AccessListElementModel struct {
// ReceiptModel is the db model for eth.receipt_cids // ReceiptModel is the db model for eth.receipt_cids
type ReceiptModel struct { type ReceiptModel struct {
BlockNumber string `db:"block_number"`
TxID string `db:"tx_id"` TxID string `db:"tx_id"`
LeafCID string `db:"leaf_cid"` LeafCID string `db:"leaf_cid"`
LeafMhKey string `db:"leaf_mh_key"` LeafMhKey string `db:"leaf_mh_key"`
@ -90,42 +95,46 @@ type ReceiptModel struct {
// StateNodeModel is the db model for eth.state_cids // StateNodeModel is the db model for eth.state_cids
type StateNodeModel struct { type StateNodeModel struct {
HeaderID string `db:"header_id"` BlockNumber string `db:"block_number"`
Path []byte `db:"state_path"` HeaderID string `db:"header_id"`
StateKey string `db:"state_leaf_key"` Path []byte `db:"state_path"`
NodeType int `db:"node_type"` StateKey string `db:"state_leaf_key"`
CID string `db:"cid"` NodeType int `db:"node_type"`
MhKey string `db:"mh_key"` CID string `db:"cid"`
Diff bool `db:"diff"` MhKey string `db:"mh_key"`
Diff bool `db:"diff"`
} }
// StorageNodeModel is the db model for eth.storage_cids // StorageNodeModel is the db model for eth.storage_cids
type StorageNodeModel struct { type StorageNodeModel struct {
HeaderID string `db:"header_id"` BlockNumber string `db:"block_number"`
StatePath []byte `db:"state_path"` HeaderID string `db:"header_id"`
Path []byte `db:"storage_path"` StatePath []byte `db:"state_path"`
StorageKey string `db:"storage_leaf_key"` Path []byte `db:"storage_path"`
NodeType int `db:"node_type"` StorageKey string `db:"storage_leaf_key"`
CID string `db:"cid"` NodeType int `db:"node_type"`
MhKey string `db:"mh_key"` CID string `db:"cid"`
Diff bool `db:"diff"` MhKey string `db:"mh_key"`
Diff bool `db:"diff"`
} }
// StorageNodeWithStateKeyModel is a db model for eth.storage_cids + eth.state_cids.state_key // StorageNodeWithStateKeyModel is a db model for eth.storage_cids + eth.state_cids.state_key
type StorageNodeWithStateKeyModel struct { type StorageNodeWithStateKeyModel struct {
HeaderID string `db:"header_id"` BlockNumber string `db:"block_number"`
StatePath []byte `db:"state_path"` HeaderID string `db:"header_id"`
Path []byte `db:"storage_path"` StatePath []byte `db:"state_path"`
StateKey string `db:"state_leaf_key"` Path []byte `db:"storage_path"`
StorageKey string `db:"storage_leaf_key"` StateKey string `db:"state_leaf_key"`
NodeType int `db:"node_type"` StorageKey string `db:"storage_leaf_key"`
CID string `db:"cid"` NodeType int `db:"node_type"`
MhKey string `db:"mh_key"` CID string `db:"cid"`
Diff bool `db:"diff"` MhKey string `db:"mh_key"`
Diff bool `db:"diff"`
} }
// StateAccountModel is a db model for an eth state account (decoded value of state leaf node) // StateAccountModel is a db model for an eth state account (decoded value of state leaf node)
type StateAccountModel struct { type StateAccountModel struct {
BlockNumber string `db:"block_number"`
HeaderID string `db:"header_id"` HeaderID string `db:"header_id"`
StatePath []byte `db:"state_path"` StatePath []byte `db:"state_path"`
Balance string `db:"balance"` Balance string `db:"balance"`
@ -136,14 +145,15 @@ type StateAccountModel struct {
// LogsModel is the db model for eth.logs // LogsModel is the db model for eth.logs
type LogsModel struct { type LogsModel struct {
ReceiptID string `db:"rct_id"` BlockNumber string `db:"block_number"`
LeafCID string `db:"leaf_cid"` ReceiptID string `db:"rct_id"`
LeafMhKey string `db:"leaf_mh_key"` LeafCID string `db:"leaf_cid"`
Address string `db:"address"` LeafMhKey string `db:"leaf_mh_key"`
Index int64 `db:"index"` Address string `db:"address"`
Data []byte `db:"log_data"` Index int64 `db:"index"`
Topic0 string `db:"topic0"` Data []byte `db:"log_data"`
Topic1 string `db:"topic1"` Topic0 string `db:"topic0"`
Topic2 string `db:"topic2"` Topic1 string `db:"topic1"`
Topic3 string `db:"topic3"` Topic2 string `db:"topic2"`
Topic3 string `db:"topic3"`
} }

12
statediff/indexer/test_helpers/test_helpers.go
View File

@ -16,18 +16,6 @@
package test_helpers package test_helpers
import (
"reflect"
"testing"
)
// ExpectEqual asserts the provided interfaces are deep equal
func ExpectEqual(t *testing.T, got interface{}, want interface{}) {
if !reflect.DeepEqual(got, want) {
t.Fatalf("Expected: %v\nActual: %v", want, got)
}
}
// ListContainsString used to check if a list of strings contains a particular string // ListContainsString used to check if a list of strings contains a particular string
func ListContainsString(sss []string, s string) bool { func ListContainsString(sss []string, s string) bool {
for _, str := range sss { for _, str := range sss {