cerc-io/go-ethereum
14
0
Fork 0
Code Issues 9 Pull Requests Packages Projects Releases 66 Wiki Activity

fix direct database writing mode tests

Browse Source
This commit is contained in:
i-norden 2023-03-20 17:28:04 -05:00
parent 2c50615b81
commit 25839ddd54
22 changed files with 344 additions and 591 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
statediff/README.md
View File

@ -239,7 +239,7 @@ This will only work on a version 12.4 Postgres database.
#### Schema overview
Our Postgres schemas are built around a single IPFS backing Postgres IPLD blockstore table (`public.blocks`) that conforms with [go-ds-sql](https://github.com/ipfs/go-ds-sql/blob/master/postgres/postgres.go).
Our Postgres schemas are built around a single IPFS backing Postgres IPLD blockstore table (`ipld.blocks`) that conforms with [go-ds-sql](https://github.com/ipfs/go-ds-sql/blob/master/postgres/postgres.go).
All IPLD objects are stored in this table, where `key` is the blockstore-prefixed multihash key for the IPLD object and `data` contains
the bytes for the IPLD block (in the case of all Ethereum IPLDs, this is the RLP byte encoding of the Ethereum object).
@ -250,7 +250,7 @@ we create an Ethereum [advanced data layout](https://github.com/ipld/specs#schem
indexes on top of the raw IPLDs in other Postgres tables.
These secondary index tables fall under the `eth` schema and follow an `{objectType}_cids` naming convention.
These tables provide a view into individual fields of the underlying Ethereum IPLD objects, allowing lookups on these fields, and reference the raw IPLD objects stored in `public.blocks`
These tables provide a view into individual fields of the underlying Ethereum IPLD objects, allowing lookups on these fields, and reference the raw IPLD objects stored in `ipld.blocks`
by foreign keys to their multihash keys.
Additionally, these tables maintain the hash-linked nature of Ethereum objects to one another. E.g. a storage trie node entry in the `storage_cids`
table contains a `state_id` foreign key which references the `id` for the `state_cids` entry that contains the state leaf node for the contract that storage node belongs to,

16
statediff/indexer/database/dump/batch_tx.go
View File

@ -23,8 +23,6 @@ import (
"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
"github.com/ethereum/go-ethereum/statediff/indexer/models"
blockstore "github.com/ipfs/go-ipfs-blockstore"
dshelp "github.com/ipfs/go-ipfs-ds-help"
)
// BatchTx wraps a void with the state necessary for building the tx concurrently during trie difference iteration
@ -80,17 +78,3 @@ func (tx *BatchTx) cacheIPLD(i ipld.IPLD) {
Data: i.RawData(),
}
}
func (tx *BatchTx) cacheRaw(codec, mh uint64, raw []byte) (string, string, error) {
c, err := ipld.RawdataToCid(codec, raw, mh)
if err != nil {
return "", "", err
}
prefixedKey := blockstore.BlockPrefix.String() + dshelp.MultihashToDsKey(c.Hash()).String()
tx.iplds <- models.IPLDModel{
BlockNumber: tx.BlockNumber,
Key: prefixedKey,
Data: raw,
}
return c.String(), prefixedKey, err
}

31
statediff/indexer/database/dump/indexer.go
View File

@ -23,9 +23,6 @@ import (
"math/big"
"time"
blockstore "github.com/ipfs/go-ipfs-blockstore"
dshelp "github.com/ipfs/go-ipfs-ds-help"
"github.com/multiformats/go-multihash"
"github.com/ethereum/go-ethereum/common"
@ -213,8 +210,7 @@ func (sdi *StateDiffIndexer) processUncles(tx *BatchTx, headerID string, blockNu
if err != nil {
return err
}
prefixedKey := blockstore.BlockPrefix.String() + dshelp.MultihashToDsKey(unclesCID.Hash()).String()
tx.cacheDirect(prefixedKey, uncleEncoding)
tx.cacheDirect(unclesCID.String(), uncleEncoding)
for i, uncle := range uncles {
var uncleReward *big.Int
// in PoA networks uncle reward is 0
@ -290,19 +286,14 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(tx *BatchTx, args processArgs
// this is the contract address if this receipt is for a contract creation tx
contract := shared.HandleZeroAddr(receipt.ContractAddress)
var contractHash string
if contract != "" {
contractHash = crypto.Keccak256Hash(common.HexToAddress(contract).Bytes()).String()
}
// index the receipt
rctModel := &models.ReceiptModel{
BlockNumber: args.blockNumber.String(),
HeaderID: args.headerID,
TxID: trxID,
Contract: contract,
ContractHash: contractHash,
CID: args.rctNodes[i].Cid().String(),
BlockNumber: args.blockNumber.String(),
HeaderID: args.headerID,
TxID: trxID,
Contract: contract,
CID: args.rctNodes[i].Cid().String(),
}
if len(receipt.PostState) == 0 {
rctModel.PostStatus = receipt.Status
@ -353,7 +344,7 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt
var stateModel models.StateNodeModel
if stateNode.Removed {
// short circuit if it is a Removed node
// this assumes the db has been initialized and a public.blocks entry for the Removed node is present
// this assumes the db has been initialized and a ipld.blocks entry for the Removed node is present
stateModel = models.StateNodeModel{
BlockNumber: tx.BlockNumber,
HeaderID: headerID,
@ -370,7 +361,7 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt
Removed: false,
Balance: stateNode.AccountWrapper.Account.Balance.String(),
Nonce: stateNode.AccountWrapper.Account.Nonce,
CodeHash: stateNode.AccountWrapper.Account.CodeHash,
CodeHash: common.BytesToHash(stateNode.AccountWrapper.Account.CodeHash).String(),
StorageRoot: stateNode.AccountWrapper.Account.Root.String(),
}
}
@ -384,11 +375,11 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt
for _, storageNode := range stateNode.StorageDiff {
if storageNode.Removed {
// short circuit if it is a Removed node
// this assumes the db has been initialized and a public.blocks entry for the Removed node is present
// this assumes the db has been initialized and a ipld.blocks entry for the Removed node is present
storageModel := models.StorageNodeModel{
BlockNumber: tx.BlockNumber,
HeaderID: headerID,
StateKey: stateNode.AccountWrapper.LeafKey,
StateKey: common.BytesToHash(stateNode.AccountWrapper.LeafKey).String(),
StorageKey: common.BytesToHash(storageNode.LeafKey).String(),
CID: shared.RemovedNodeStorageCID,
Removed: true,
@ -401,7 +392,7 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt
storageModel := models.StorageNodeModel{
BlockNumber: tx.BlockNumber,
HeaderID: headerID,
StateKey: stateNode.AccountWrapper.LeafKey,
StateKey: common.BytesToHash(stateNode.AccountWrapper.LeafKey).String(),
StorageKey: common.BytesToHash(storageNode.LeafKey).String(),
CID: storageNode.CID,
Removed: false,

20
statediff/indexer/database/file/csv_writer.go
View File

@ -25,8 +25,6 @@ import (
"path/filepath"
"strconv"
blockstore "github.com/ipfs/go-ipfs-blockstore"
dshelp "github.com/ipfs/go-ipfs-ds-help"
"github.com/thoas/go-funk"
"github.com/ethereum/go-ethereum/common"
@ -46,10 +44,8 @@ var (
&types.TableStorageNode,
&types.TableUncle,
&types.TableTransaction,
&types.TableAccessListElement,
&types.TableReceipt,
&types.TableLog,
&types.TableStateAccount,
}
)
@ -233,20 +229,6 @@ func (csw *CSVWriter) upsertIPLDNode(blockNumber string, i ipld.IPLD) {
})
}
func (csw *CSVWriter) upsertIPLDRaw(blockNumber string, codec, mh uint64, raw []byte) (string, string, error) {
c, err := ipld.RawdataToCid(codec, raw, mh)
if err != nil {
return "", "", err
}
prefixedKey := blockstore.BlockPrefix.String() + dshelp.MultihashToDsKey(c.Hash()).String()
csw.upsertIPLD(models.IPLDModel{
BlockNumber: blockNumber,
Key: prefixedKey,
Data: raw,
})
return c.String(), prefixedKey, err
}
func (csw *CSVWriter) upsertHeaderCID(header models.HeaderModel) {
var values []interface{}
values = append(values, header.BlockNumber, header.BlockHash, header.ParentHash, header.CID,
@ -273,7 +255,7 @@ func (csw *CSVWriter) upsertTransactionCID(transaction models.TxModel) {
func (csw *CSVWriter) upsertReceiptCID(rct *models.ReceiptModel) {
var values []interface{}
values = append(values, rct.BlockNumber, rct.HeaderID, rct.TxID, rct.CID, rct.Contract, rct.ContractHash,
values = append(values, rct.BlockNumber, rct.HeaderID, rct.TxID, rct.CID, rct.Contract,
rct.PostState, rct.PostStatus)
csw.rows <- tableRow{types.TableReceipt, values}
indexerMetrics.receipts.Inc(1)

28
statediff/indexer/database/file/indexer.go
View File

@ -27,8 +27,6 @@ import (
"sync/atomic"
"time"
blockstore "github.com/ipfs/go-ipfs-blockstore"
dshelp "github.com/ipfs/go-ipfs-ds-help"
"github.com/lib/pq"
"github.com/multiformats/go-multihash"
@ -271,8 +269,7 @@ func (sdi *StateDiffIndexer) processUncles(headerID string, blockNumber *big.Int
if err != nil {
return err
}
prefixedKey := blockstore.BlockPrefix.String() + dshelp.MultihashToDsKey(unclesCID.Hash()).String()
sdi.fileWriter.upsertIPLDDirect(blockNumber.String(), prefixedKey, uncleEncoding)
sdi.fileWriter.upsertIPLDDirect(blockNumber.String(), unclesCID.String(), uncleEncoding)
for i, uncle := range uncles {
var uncleReward *big.Int
// in PoA networks uncle reward is 0
@ -312,6 +309,7 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(args processArgs) error {
for i, receipt := range args.receipts {
txNode := args.txNodes[i]
sdi.fileWriter.upsertIPLDNode(args.blockNumber.String(), txNode)
sdi.fileWriter.upsertIPLDNode(args.blockNumber.String(), args.rctNodes[i])
// index tx
trx := args.txs[i]
@ -342,19 +340,14 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(args processArgs) error {
// this is the contract address if this receipt is for a contract creation tx
contract := shared.HandleZeroAddr(receipt.ContractAddress)
var contractHash string
if contract != "" {
contractHash = crypto.Keccak256Hash(common.HexToAddress(contract).Bytes()).String()
}
// index receipt
rctModel := &models.ReceiptModel{
BlockNumber: args.blockNumber.String(),
HeaderID: args.headerID,
TxID: txID,
Contract: contract,
ContractHash: contractHash,
CID: args.rctNodes[i].Cid().String(),
BlockNumber: args.blockNumber.String(),
HeaderID: args.headerID,
TxID: txID,
Contract: contract,
CID: args.rctNodes[i].Cid().String(),
}
if len(receipt.PostState) == 0 {
rctModel.PostStatus = receipt.Status
@ -366,6 +359,7 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(args processArgs) error {
// index logs
logDataSet := make([]*models.LogsModel, len(receipt.Logs))
for idx, l := range receipt.Logs {
sdi.fileWriter.upsertIPLDNode(args.blockNumber.String(), args.logNodes[i][idx])
topicSet := make([]string, 4)
for ti, topic := range l.Topics {
topicSet[ti] = topic.Hex()
@ -419,7 +413,7 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt
Removed: false,
Balance: stateNode.AccountWrapper.Account.Balance.String(),
Nonce: stateNode.AccountWrapper.Account.Nonce,
CodeHash: stateNode.AccountWrapper.Account.CodeHash,
CodeHash: common.BytesToHash(stateNode.AccountWrapper.Account.CodeHash).String(),
StorageRoot: stateNode.AccountWrapper.Account.Root.String(),
}
}
@ -437,7 +431,7 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt
storageModel := models.StorageNodeModel{
BlockNumber: tx.BlockNumber,
HeaderID: headerID,
StateKey: stateNode.AccountWrapper.LeafKey,
StateKey: common.BytesToHash(stateNode.AccountWrapper.LeafKey).String(),
StorageKey: common.BytesToHash(storageNode.LeafKey).String(),
CID: shared.RemovedNodeStorageCID,
Removed: true,
@ -448,7 +442,7 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt
storageModel := models.StorageNodeModel{
BlockNumber: tx.BlockNumber,
HeaderID: headerID,
StateKey: stateNode.AccountWrapper.LeafKey,
StateKey: common.BytesToHash(stateNode.AccountWrapper.LeafKey).Hex(),
StorageKey: common.BytesToHash(storageNode.LeafKey).String(),
CID: storageNode.CID,
Removed: false,

1
statediff/indexer/database/file/interfaces.go
View File

@ -48,7 +48,6 @@ type FileWriter interface {
// Methods to upsert IPLD in different ways
upsertIPLDDirect(blockNumber, key string, value []byte)
upsertIPLDNode(blockNumber string, i ipld.IPLD)
upsertIPLDRaw(blockNumber string, codec, mh uint64, raw []byte) (string, string, error)
// Methods to read and write watched addresses
loadWatchedAddresses() ([]common.Address, error)

26
statediff/indexer/database/file/sql_writer.go
View File

@ -24,8 +24,6 @@ import (
"math/big"
"os"
blockstore "github.com/ipfs/go-ipfs-blockstore"
dshelp "github.com/ipfs/go-ipfs-ds-help"
pg_query "github.com/pganalyze/pg_query_go/v2"
"github.com/thoas/go-funk"
@ -139,7 +137,7 @@ const (
nodeInsert = "INSERT INTO nodes (genesis_block, network_id, node_id, client_name, chain_id) VALUES " +
"('%s', '%s', '%s', '%s', %d);\n"
ipldInsert = "INSERT INTO public.blocks (block_number, key, data) VALUES ('%s', '%s', '\\x%x');\n"
ipldInsert = "INSERT INTO ipld.blocks (block_number, key, data) VALUES ('%s', '%s', '\\x%x');\n"
headerInsert = "INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_ids, reward, " +
"state_root, tx_root, receipt_root, uncles_hash, bloom, timestamp, coinbase) VALUES " +
@ -151,8 +149,8 @@ const (
txInsert = "INSERT INTO eth.transaction_cids (block_number, header_id, tx_hash, cid, dst, src, index, tx_type, " +
"value) VALUES ('%s', '%s', '%s', '%s', '%s', '%s', %d, %d, '%s');\n"
rctInsert = "INSERT INTO eth.receipt_cids (block_number, header_id, tx_id, cid, contract, contract_hash, post_state, " +
"post_status) VALUES ('%s', '%s', '%s', '%s', '%s', '%s', '%s', %d);\n"
rctInsert = "INSERT INTO eth.receipt_cids (block_number, header_id, tx_id, cid, contract, post_state, " +
"post_status) VALUES ('%s', '%s', '%s', '%s', '%s', '%s', %d);\n"
logInsert = "INSERT INTO eth.log_cids (block_number, header_id, cid, rct_id, address, index, topic0, topic1, topic2, " +
"topic3) VALUES ('%s', '%s', '%s', '%s', '%s', %d, '%s', '%s', '%s', '%s');\n"
@ -188,23 +186,9 @@ func (sqw *SQLWriter) upsertIPLDNode(blockNumber string, i ipld.IPLD) {
})
}
func (sqw *SQLWriter) upsertIPLDRaw(blockNumber string, codec, mh uint64, raw []byte) (string, string, error) {
c, err := ipld.RawdataToCid(codec, raw, mh)
if err != nil {
return "", "", err
}
prefixedKey := blockstore.BlockPrefix.String() + dshelp.MultihashToDsKey(c.Hash()).String()
sqw.upsertIPLD(models.IPLDModel{
BlockNumber: blockNumber,
Key: prefixedKey,
Data: raw,
})
return c.String(), prefixedKey, err
}
func (sqw *SQLWriter) upsertHeaderCID(header models.HeaderModel) {
stmt := fmt.Sprintf(headerInsert, header.BlockNumber, header.BlockHash, header.ParentHash, header.CID,
header.TotalDifficulty, header.NodeIDs, header.Reward, header.StateRoot, header.TxRoot,
header.TotalDifficulty, formatPostgresStringArray(header.NodeIDs), header.Reward, header.StateRoot, header.TxRoot,
header.RctRoot, header.UnclesHash, header.Bloom, header.Timestamp, header.Coinbase)
sqw.stmts <- []byte(stmt)
indexerMetrics.blocks.Inc(1)
@ -222,7 +206,7 @@ func (sqw *SQLWriter) upsertTransactionCID(transaction models.TxModel) {
}
func (sqw *SQLWriter) upsertReceiptCID(rct *models.ReceiptModel) {
sqw.stmts <- []byte(fmt.Sprintf(rctInsert, rct.BlockNumber, rct.HeaderID, rct.TxID, rct.CID, rct.Contract, rct.ContractHash,
sqw.stmts <- []byte(fmt.Sprintf(rctInsert, rct.BlockNumber, rct.HeaderID, rct.TxID, rct.CID, rct.Contract,
rct.PostState, rct.PostStatus))
indexerMetrics.receipts.Inc(1)
}

57
statediff/indexer/database/file/types/schema.go
View File

@ -17,7 +17,7 @@
package types
var TableIPLDBlock = Table{
`public.blocks`,
`ipld.blocks`,
[]column{
{name: "block_number", dbType: bigint},
{name: "key", dbType: text},
@ -44,7 +44,7 @@ var TableHeader = Table{
{name: "parent_hash", dbType: varchar},
{name: "cid", dbType: text},
{name: "td", dbType: numeric},
{name: "node_id", dbType: varchar},
{name: "node_ids", dbType: varchar, isArray: true},
{name: "reward", dbType: numeric},
{name: "state_root", dbType: varchar},
{name: "tx_root", dbType: varchar},
@ -52,8 +52,6 @@ var TableHeader = Table{
{name: "uncles_hash", dbType: varchar},
{name: "bloom", dbType: bytea},
{name: "timestamp", dbType: numeric},
{name: "mh_key", dbType: text},
{name: "times_validated", dbType: integer},
{name: "coinbase", dbType: varchar},
},
}
@ -65,10 +63,12 @@ var TableStateNode = Table{
{name: "header_id", dbType: varchar},
{name: "state_leaf_key", dbType: varchar},
{name: "cid", dbType: text},
{name: "state_path", dbType: bytea},
{name: "node_type", dbType: integer},
{name: "removed", dbType: boolean},
{name: "diff", dbType: boolean},
{name: "mh_key", dbType: text},
{name: "balance", dbType: numeric},
{name: "nonce", dbType: bigint},
{name: "code_hash", dbType: varchar},
{name: "storage_root", dbType: varchar},
},
}
@ -77,13 +77,12 @@ var TableStorageNode = Table{
[]column{
{name: "block_number", dbType: bigint},
{name: "header_id", dbType: varchar},
{name: "state_path", dbType: bytea},
{name: "state_leaf_key", dbType: varchar},
{name: "storage_leaf_key", dbType: varchar},
{name: "cid", dbType: text},
{name: "storage_path", dbType: bytea},
{name: "node_type", dbType: integer},
{name: "removed", dbType: boolean},
{name: "diff", dbType: boolean},
{name: "mh_key", dbType: text},
{name: "val", dbType: bytea},
},
}
@ -96,7 +95,6 @@ var TableUncle = Table{
{name: "parent_hash", dbType: varchar},
{name: "cid", dbType: text},
{name: "reward", dbType: numeric},
{name: "mh_key", dbType: text},
{name: "index", dbType: integer},
},
}
@ -111,37 +109,21 @@ var TableTransaction = Table{
{name: "dst", dbType: varchar},
{name: "src", dbType: varchar},
{name: "index", dbType: integer},
{name: "mh_key", dbType: text},
{name: "tx_data", dbType: bytea},
{name: "tx_type", dbType: integer},
{name: "value", dbType: numeric},
},
}
var TableAccessListElement = Table{
"eth.access_list_elements",
[]column{
{name: "block_number", dbType: bigint},
{name: "tx_id", dbType: varchar},
{name: "index", dbType: integer},
{name: "address", dbType: varchar},
{name: "storage_keys", dbType: varchar, isArray: true},
},
}
var TableReceipt = Table{
"eth.receipt_cids",
[]column{
{name: "block_number", dbType: bigint},
{name: "header_id", dbType: varchar},
{name: "tx_id", dbType: varchar},
{name: "leaf_cid", dbType: text},
{name: "cid", dbType: text},
{name: "contract", dbType: varchar},
{name: "contract_hash", dbType: varchar},
{name: "leaf_mh_key", dbType: text},
{name: "post_state", dbType: varchar},
{name: "post_status", dbType: integer},
{name: "log_root", dbType: varchar},
},
}
@ -150,8 +132,7 @@ var TableLog = Table{
[]column{
{name: "block_number", dbType: bigint},
{name: "header_id", dbType: varchar},
{name: "leaf_cid", dbType: text},
{name: "leaf_mh_key", dbType: text},
{name: "cid", dbType: text},
{name: "rct_id", dbType: varchar},
{name: "address", dbType: varchar},
{name: "index", dbType: integer},
@ -159,20 +140,6 @@ var TableLog = Table{
{name: "topic1", dbType: varchar},
{name: "topic2", dbType: varchar},
{name: "topic3", dbType: varchar},
{name: "log_data", dbType: bytea},
},
}
var TableStateAccount = Table{
"eth.state_accounts",
[]column{
{name: "block_number", dbType: bigint},
{name: "header_id", dbType: varchar},
{name: "state_path", dbType: bytea},
{name: "balance", dbType: numeric},
{name: "nonce", dbType: bigint},
{name: "code_hash", dbType: varchar},
{name: "storage_root", dbType: varchar},
},
}

21
statediff/indexer/database/sql/batch_tx.go
View File

@ -21,8 +21,6 @@ import (
"sync"
"sync/atomic"
blockstore "github.com/ipfs/go-ipfs-blockstore"
dshelp "github.com/ipfs/go-ipfs-ds-help"
"github.com/lib/pq"
"github.com/ethereum/go-ethereum/log"
@ -58,7 +56,7 @@ func (tx *BatchTx) flush() error {
_, err := tx.dbtx.Exec(tx.ctx, tx.stm, pq.Array(tx.ipldCache.BlockNumbers), pq.Array(tx.ipldCache.Keys),
pq.Array(tx.ipldCache.Values))
if err != nil {
log.Debug(insertError{"public.blocks", err, tx.stm,
log.Debug(insertError{"ipld.blocks", err, tx.stm,
struct {
blockNumbers []string
keys []string
@ -68,7 +66,7 @@ func (tx *BatchTx) flush() error {
tx.ipldCache.Keys,
tx.ipldCache.Values,
}}.Error())
return insertError{"public.blocks", err, tx.stm, "too many arguments; use debug mode for full list"}
return insertError{"ipld.blocks", err, tx.stm, "too many arguments; use debug mode for full list"}
}
tx.ipldCache = models.IPLDBatch{}
return nil
@ -108,21 +106,6 @@ func (tx *BatchTx) cacheIPLD(i ipld.IPLD) {
}
}
func (tx *BatchTx) cacheRaw(codec, mh uint64, raw []byte) (string, string, error) {
c, err := ipld.RawdataToCid(codec, raw, mh)
if err != nil {
return "", "", err
}
prefixedKey := blockstore.BlockPrefix.String() + dshelp.MultihashToDsKey(c.Hash()).String()
tx.cacheWg.Add(1)
tx.iplds <- models.IPLDModel{
BlockNumber: tx.BlockNumber,
Key: prefixedKey,
Data: raw,
}
return c.String(), prefixedKey, err
}
func (tx *BatchTx) cacheRemoved(key string, value []byte) {
if atomic.LoadUint32(tx.removedCacheFlag) == 0 {
atomic.StoreUint32(tx.removedCacheFlag, 1)

33
statediff/indexer/database/sql/indexer.go
View File

@ -26,8 +26,6 @@ import (
"math/big"
"time"
blockstore "github.com/ipfs/go-ipfs-blockstore"
dshelp "github.com/ipfs/go-ipfs-ds-help"
"github.com/multiformats/go-multihash"
"github.com/ethereum/go-ethereum/common"
@ -269,8 +267,7 @@ func (sdi *StateDiffIndexer) processUncles(tx *BatchTx, headerID string, blockNu
if err != nil {
return err
}
prefixedKey := blockstore.BlockPrefix.String() + dshelp.MultihashToDsKey(unclesCID.Hash()).String()
tx.cacheDirect(prefixedKey, uncleEncoding)
tx.cacheDirect(unclesCID.String(), uncleEncoding)
for i, uncle := range uncles {
var uncleReward *big.Int
// in PoA networks uncle reward is 0
@ -311,11 +308,9 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(tx *BatchTx, args processArgs
// Process receipts and txs
signer := types.MakeSigner(sdi.chainConfig, args.blockNumber)
for i, receipt := range args.receipts {
for _, logNode := range args.logNodes[i] {
tx.cacheIPLD(logNode)
}
txNode := args.txNodes[i]
tx.cacheIPLD(txNode)
tx.cacheIPLD(args.rctNodes[i])
// index tx
trx := args.txs[i]
@ -348,18 +343,13 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(tx *BatchTx, args processArgs
// this is the contract address if this receipt is for a contract creation tx
contract := shared.HandleZeroAddr(receipt.ContractAddress)
var contractHash string
if contract != "" {
contractHash = crypto.Keccak256Hash(common.HexToAddress(contract).Bytes()).String()
}
rctModel := &models.ReceiptModel{
BlockNumber: args.blockNumber.String(),
HeaderID: args.headerID,
TxID: txID,
Contract: contract,
ContractHash: contractHash,
CID: args.rctNodes[i].Cid().String(),
BlockNumber: args.blockNumber.String(),
HeaderID: args.headerID,
TxID: txID,
Contract: contract,
CID: args.rctNodes[i].Cid().String(),
}
if len(receipt.PostState) == 0 {
rctModel.PostStatus = receipt.Status
@ -374,6 +364,7 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(tx *BatchTx, args processArgs
// index logs
logDataSet := make([]*models.LogsModel, len(receipt.Logs))
for idx, l := range receipt.Logs {
tx.cacheIPLD(args.logNodes[i][idx])
topicSet := make([]string, 4)
for ti, topic := range l.Topics {
topicSet[ti] = topic.Hex()
@ -427,7 +418,7 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt
Removed: false,
Balance: stateNode.AccountWrapper.Account.Balance.String(),
Nonce: stateNode.AccountWrapper.Account.Nonce,
CodeHash: stateNode.AccountWrapper.Account.CodeHash,
CodeHash: common.BytesToHash(stateNode.AccountWrapper.Account.CodeHash).String(),
StorageRoot: stateNode.AccountWrapper.Account.Root.String(),
}
}
@ -444,7 +435,7 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt
storageModel := models.StorageNodeModel{
BlockNumber: tx.BlockNumber,
HeaderID: headerID,
StateKey: stateNode.AccountWrapper.LeafKey,
StateKey: common.BytesToHash(stateNode.AccountWrapper.LeafKey).String(),
StorageKey: common.BytesToHash(storageNode.LeafKey).String(),
CID: shared.RemovedNodeStorageCID,
Removed: true,
@ -457,10 +448,10 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt
storageModel := models.StorageNodeModel{
BlockNumber: tx.BlockNumber,
HeaderID: headerID,
StateKey: stateNode.AccountWrapper.LeafKey,
StateKey: common.BytesToHash(stateNode.AccountWrapper.LeafKey).String(),
StorageKey: common.BytesToHash(storageNode.LeafKey).String(),
CID: storageNode.CID,
Removed: true,
Removed: false,
Value: storageNode.Value,
}
if err := sdi.dbWriter.upsertStorageCID(tx.dbtx, storageModel); err != nil {

1
statediff/indexer/database/sql/interfaces.go
View File

@ -49,7 +49,6 @@ type Statements interface {
InsertRctStm() string
InsertLogStm() string
InsertStateStm() string
InsertAccountStm() string
InsertStorageStm() string
InsertIPLDStm() string
InsertIPLDsStm() string

44
statediff/indexer/database/sql/postgres/database.go
View File

@ -40,73 +40,67 @@ type DB struct {
// Stm == Statement
func (db *DB) InsertHeaderStm() string {
if db.upsert {
return `INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_ids, reward, state_root, tx_root, receipt_root, uncles_hash, bloom, timestamp, mh_key, times_validated, coinbase)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16)
ON CONFLICT (block_hash, block_number) DO UPDATE SET (parent_hash, cid, td, node_ids, reward, state_root, tx_root, receipt_root, uncles_hash, bloom, timestamp, mh_key, times_validated, coinbase) = ($3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, eth.header_cids.times_validated + 1, $16)`
return `INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_ids, reward, state_root, tx_root, receipt_root, uncles_hash, bloom, timestamp, coinbase)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)
ON CONFLICT (block_hash, block_number) DO UPDATE SET (parent_hash, cid, td, node_ids, reward, state_root, tx_root, receipt_root, uncles_hash, bloom, timestamp, coinbase) = ($3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)`
}
return `INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_ids, reward, state_root, tx_root, receipt_root, uncles_hash, bloom, timestamp, mh_key, times_validated, coinbase)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16)
return `INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_ids, reward, state_root, tx_root, receipt_root, uncles_hash, bloom, timestamp, coinbase)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)
ON CONFLICT (block_hash, block_number) DO NOTHING`
}
// InsertUncleStm satisfies the sql.Statements interface
func (db *DB) InsertUncleStm() string {
return `INSERT INTO eth.uncle_cids (block_number, block_hash, header_id, parent_hash, cid, reward, mh_key, index) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
return `INSERT INTO eth.uncle_cids (block_number, block_hash, header_id, parent_hash, cid, reward, index) VALUES ($1, $2, $3, $4, $5, $6, $7)
ON CONFLICT (block_hash, block_number, index) DO NOTHING`
}
// InsertTxStm satisfies the sql.Statements interface
func (db *DB) InsertTxStm() string {
return `INSERT INTO eth.transaction_cids (block_number, header_id, tx_hash, cid, dst, src, index, mh_key, tx_data, tx_type, value) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11)
return `INSERT INTO eth.transaction_cids (block_number, header_id, tx_hash, cid, dst, src, index, tx_type, value) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
ON CONFLICT (tx_hash, header_id, block_number) DO NOTHING`
}
// InsertRctStm satisfies the sql.Statements interface
func (db *DB) InsertRctStm() string {
return `INSERT INTO eth.receipt_cids (block_number, header_id, tx_id, leaf_cid, contract, contract_hash, leaf_mh_key, post_state, post_status, log_root) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
return `INSERT INTO eth.receipt_cids (block_number, header_id, tx_id, cid, contract, post_state, post_status) VALUES ($1, $2, $3, $4, $5, $6, $7)
ON CONFLICT (tx_id, header_id, block_number) DO NOTHING`
}
// InsertLogStm satisfies the sql.Statements interface
func (db *DB) InsertLogStm() string {
return `INSERT INTO eth.log_cids (block_number, header_id, leaf_cid, leaf_mh_key, rct_id, address, index, topic0, topic1, topic2, topic3, log_data) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)
return `INSERT INTO eth.log_cids (block_number, header_id, cid, rct_id, address, index, topic0, topic1, topic2, topic3) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
ON CONFLICT (rct_id, index, header_id, block_number) DO NOTHING`
}
// InsertStateStm satisfies the sql.Statements interface
func (db *DB) InsertStateStm() string {
if db.upsert {
return `INSERT INTO eth.state_cids (block_number, header_id, state_leaf_key, cid, state_path, node_type, diff, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
ON CONFLICT (header_id, state_path, block_number) DO UPDATE SET (block_number, state_leaf_key, cid, node_type, diff, mh_key) = ($1, $3, $4, $6, $7, $8)`
return `INSERT INTO eth.state_cids (block_number, header_id, state_leaf_key, cid, removed, diff, balance, nonce, code_hash, storage_root) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
ON CONFLICT (header_id, state_leaf_key, block_number) DO UPDATE SET (cid, removed, diff, balance, nonce, code_hash, storage_root) = ($4, $5, $6, $7, $8, $9, $10)`
}
return `INSERT INTO eth.state_cids (block_number, header_id, state_leaf_key, cid, state_path, node_type, diff, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
ON CONFLICT (header_id, state_path, block_number) DO NOTHING`
}
// InsertAccountStm satisfies the sql.Statements interface
func (db *DB) InsertAccountStm() string {
return `INSERT INTO eth.state_accounts (block_number, header_id, state_path, balance, nonce, code_hash, storage_root) VALUES ($1, $2, $3, $4, $5, $6, $7)
ON CONFLICT (header_id, state_path, block_number) DO NOTHING`
return `INSERT INTO eth.state_cids (block_number, header_id, state_leaf_key, cid, removed, diff, balance, nonce, code_hash, storage_root) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10)
ON CONFLICT (header_id, state_leaf_key, block_number) DO NOTHING`
}
// InsertStorageStm satisfies the sql.Statements interface
func (db *DB) InsertStorageStm() string {
if db.upsert {
return `INSERT INTO eth.storage_cids (block_number, header_id, state_path, storage_leaf_key, cid, storage_path, node_type, diff, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
ON CONFLICT (header_id, state_path, storage_path, block_number) DO UPDATE SET (block_number, storage_leaf_key, cid, node_type, diff, mh_key) = ($1, $4, $5, $7, $8, $9)`
return `INSERT INTO eth.storage_cids (block_number, header_id, state_leaf_key, storage_leaf_key, cid, removed, diff, val) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
ON CONFLICT (header_id, state_leaf_key, storage_leaf_key, block_number) DO UPDATE SET (cid, removed, diff, val) = ($4, $5, $6, $7, $8)`
}
return `INSERT INTO eth.storage_cids (block_number, header_id, state_path, storage_leaf_key, cid, storage_path, node_type, diff, mh_key) VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)
ON CONFLICT (header_id, state_path, storage_path, block_number) DO NOTHING`
return `INSERT INTO eth.storage_cids (block_number, header_id, state_leaf_key, storage_leaf_key, cid, removed, diff, val) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
ON CONFLICT (header_id, state_leaf_key, storage_leaf_key, block_number) DO NOTHING`
}
// InsertIPLDStm satisfies the sql.Statements interface
func (db *DB) InsertIPLDStm() string {
return `INSERT INTO public.blocks (block_number, key, data) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING`
return `INSERT INTO ipld.blocks (block_number, key, data) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING`
}
// InsertIPLDsStm satisfies the sql.Statements interface
func (db *DB) InsertIPLDsStm() string {
return `INSERT INTO public.blocks (block_number, key, data) VALUES (unnest($1::BIGINT[]), unnest($2::TEXT[]), unnest($3::BYTEA[])) ON CONFLICT DO NOTHING`
return `INSERT INTO ipld.blocks (block_number, key, data) VALUES (unnest($1::BIGINT[]), unnest($2::TEXT[]), unnest($3::BYTEA[])) ON CONFLICT DO NOTHING`
}
// InsertKnownGapsStm satisfies the sql.Statements interface

2
statediff/indexer/database/sql/sqlx_indexer_test.go
View File

@ -49,7 +49,7 @@ func setupSQLXNonCanonical(t *testing.T) {
}
// Test indexer for a canonical block
func TestSQLXIndexer(t *testing.T) {
func TestSQLXIndexerF(t *testing.T) {
t.Run("Publish and index header IPLDs in a single tx", func(t *testing.T) {
setupSQLX(t)
defer tearDown(t)

25
statediff/indexer/database/sql/writer.go
View File

@ -42,7 +42,7 @@ func (w *Writer) Close() error {
}
/*
INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_ids, reward, state_root, tx_root, receipt_root, uncles_hash, bloom, timestamp, mh_key, times_validated, coinbase)
INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_ids, reward, state_root, tx_root, receipt_root, uncles_hash, bloom, timestamp, coinbase)
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14)
ON CONFLICT (block_hash, block_number) DO NOTHING
*/
@ -87,12 +87,12 @@ func (w *Writer) upsertTransactionCID(tx Tx, transaction models.TxModel) error {
}
/*
INSERT INTO eth.receipt_cids (block_number, header_id, tx_id, cid, contract, contract_hash, post_state, post_status) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
INSERT INTO eth.receipt_cids (block_number, header_id, tx_id, cid, contract, post_state, post_status) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)
ON CONFLICT (tx_id, header_id, block_number) DO NOTHING
*/
func (w *Writer) upsertReceiptCID(tx Tx, rct *models.ReceiptModel) error {
_, err := tx.Exec(w.db.Context(), w.db.InsertRctStm(),
rct.BlockNumber, rct.HeaderID, rct.TxID, rct.CID, rct.Contract, rct.ContractHash, rct.PostState,
rct.BlockNumber, rct.HeaderID, rct.TxID, rct.CID, rct.Contract, rct.PostState,
rct.PostStatus)
if err != nil {
return insertError{"eth.receipt_cids", err, w.db.InsertRctStm(), *rct}
@ -123,11 +123,20 @@ INSERT INTO eth.state_cids (block_number, header_id, state_leaf_key, cid, remove
ON CONFLICT (header_id, state_leaf_key, block_number) DO NOTHING
*/
func (w *Writer) upsertStateCID(tx Tx, stateNode models.StateNodeModel) error {
_, err := tx.Exec(w.db.Context(), w.db.InsertStateStm(),
stateNode.BlockNumber, stateNode.HeaderID, stateNode.StateKey, stateNode.CID, stateNode.Removed, true,
stateNode.Balance, stateNode.Nonce, stateNode.CodeHash, stateNode.StorageRoot)
if err != nil {
return insertError{"eth.state_cids", err, w.db.InsertStateStm(), stateNode}
if stateNode.Removed {
_, err := tx.Exec(w.db.Context(), w.db.InsertStateStm(),
stateNode.BlockNumber, stateNode.HeaderID, stateNode.StateKey, stateNode.CID, stateNode.Removed, true,
"0", stateNode.Nonce, stateNode.CodeHash, stateNode.StorageRoot)
if err != nil {
return insertError{"eth.state_cids", err, w.db.InsertStateStm(), stateNode}
}
} else {
_, err := tx.Exec(w.db.Context(), w.db.InsertStateStm(),
stateNode.BlockNumber, stateNode.HeaderID, stateNode.StateKey, stateNode.CID, stateNode.Removed, true,
stateNode.Balance, stateNode.Nonce, stateNode.CodeHash, stateNode.StorageRoot)
if err != nil {
return insertError{"eth.state_cids", err, w.db.InsertStateStm(), stateNode}
}
}
return nil
}

144
statediff/indexer/mocks/test_data.go
View File

@ -22,13 +22,15 @@ import (
"crypto/rand"
"math/big"
ipld2 "github.com/ethereum/go-ethereum/statediff/indexer/ipld"
"github.com/ethereum/go-ethereum/statediff/indexer/shared"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/statediff/indexer/models"
"github.com/ethereum/go-ethereum/statediff/test_helpers"
sdtypes "github.com/ethereum/go-ethereum/statediff/types"
"github.com/ethereum/go-ethereum/trie"
@ -137,17 +139,6 @@ var (
Address: AnotherAddress,
StorageKeys: []common.Hash{common.BytesToHash(StorageLeafKey), common.BytesToHash(MockStorageLeafKey)},
}
AccessListEntry1Model = models.AccessListElementModel{
BlockNumber: BlockNumber.String(),
Index: 0,
Address: Address.Hex(),
}
AccessListEntry2Model = models.AccessListElementModel{
BlockNumber: BlockNumber.String(),
Index: 1,
Address: AnotherAddress.Hex(),
StorageKeys: []string{common.BytesToHash(StorageLeafKey).Hex(), common.BytesToHash(MockStorageLeafKey).Hex()},
}
// statediff data
storageLocation = common.HexToHash("0")
@ -160,22 +151,26 @@ var (
StoragePartialPath,
StorageValue,
})
StorageLeafNodeCID = ipld2.Keccak256ToCid(ipld2.MEthStorageTrie, crypto.Keccak256(StorageLeafNode)).String()
nonce1 = uint64(1)
ContractRoot = "0x821e2556a290c86405f8160a2d662042a431ba456b9db265c79bb837c04be5f0"
ContractCodeHash = common.HexToHash("0x753f98a8d4328b15636e46f66f2cb4bc860100aa17967cc145fcd17d1d4710ea")
ContractLeafKey = test_helpers.AddressToLeafKey(ContractAddress)
ContractAccount, _ = rlp.EncodeToBytes(&types.StateAccount{
nonce1 = uint64(1)
ContractRoot = "0x821e2556a290c86405f8160a2d662042a431ba456b9db265c79bb837c04be5f0"
ContractCodeHash = common.HexToHash("0x753f98a8d4328b15636e46f66f2cb4bc860100aa17967cc145fcd17d1d4710ea")
ContractLeafKey = test_helpers.AddressToLeafKey(ContractAddress)
ContractAccount = &types.StateAccount{
Nonce: nonce1,
Balance: big.NewInt(0),
CodeHash: ContractCodeHash.Bytes(),
Root: common.HexToHash(ContractRoot),
})
}
ContractAccountRLP, _ = rlp.EncodeToBytes(ContractAccount)
ContractPartialPath = common.Hex2Bytes("3114658a74d9cc9f7acf2c5cd696c3494d7c344d78bfec3add0d91ec4e8d1c45")
ContractLeafNode, _ = rlp.EncodeToBytes(&[]interface{}{
ContractPartialPath,
ContractAccount,
})
ContractLeafNodeCID = ipld2.Keccak256ToCid(ipld2.MEthStateTrie, crypto.Keccak256(ContractLeafNode)).String()
Contract2LeafKey = test_helpers.AddressToLeafKey(ContractAddress2)
storage2Location = common.HexToHash("2")
@ -188,74 +183,107 @@ var (
AccountCodeHash = common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")
AccountLeafKey = test_helpers.Account2LeafKey
RemovedLeafKey = test_helpers.Account1LeafKey
Account, _ = rlp.EncodeToBytes(&types.StateAccount{
Account = &types.StateAccount{
Nonce: nonce0,
Balance: big.NewInt(1000),
CodeHash: AccountCodeHash.Bytes(),
Root: common.HexToHash(AccountRoot),
})
}
AccountRLP, _ = rlp.EncodeToBytes(Account)
AccountPartialPath = common.Hex2Bytes("3957f3e2f04a0764c3a0491b175f69926da61efbcc8f61fa1455fd2d2b4cdd45")
AccountLeafNode, _ = rlp.EncodeToBytes(&[]interface{}{
AccountPartialPath,
Account,
})
AccountLeafNodeCID = ipld2.Keccak256ToCid(ipld2.MEthStateTrie, crypto.Keccak256(AccountLeafNode)).String()
StateDiffs = []sdtypes.StateNode{
StateDiffs = []sdtypes.StateLeafNode{
{
Path: []byte{'\x06'},
NodeType: sdtypes.Leaf,
LeafKey: ContractLeafKey,
NodeValue: ContractLeafNode,
StorageNodes: []sdtypes.StorageNode{
AccountWrapper: sdtypes.AccountWrapper{
Account: ContractAccount,
LeafKey: ContractLeafKey,
CID: ContractLeafNodeCID,
},
Removed: false,
StorageDiff: []sdtypes.StorageLeafNode{
{
Path: []byte{},
NodeType: sdtypes.Leaf,
LeafKey: StorageLeafKey,
NodeValue: StorageLeafNode,
Removed: false,
LeafKey: StorageLeafKey,
Value: StorageValue,
CID: StorageLeafNodeCID,
},
{
Path: []byte{'\x03'},
NodeType: sdtypes.Removed,
LeafKey: RemovedLeafKey,
NodeValue: []byte{},
Removed: true,
LeafKey: RemovedLeafKey,
CID: shared.RemovedNodeStorageCID,
},
},
},
{
Path: []byte{'\x0c'},
NodeType: sdtypes.Leaf,
LeafKey: AccountLeafKey,
NodeValue: AccountLeafNode,
StorageNodes: []sdtypes.StorageNode{},
AccountWrapper: sdtypes.AccountWrapper{
Account: Account,
LeafKey: AccountLeafKey,
CID: AccountLeafNodeCID,
},
Removed: false,
StorageDiff: []sdtypes.StorageLeafNode{},
},
{
Path: []byte{'\x02'},
NodeType: sdtypes.Removed,
LeafKey: RemovedLeafKey,
NodeValue: []byte{},
AccountWrapper: sdtypes.AccountWrapper{
Account: nil,
LeafKey: RemovedLeafKey,
CID: shared.RemovedNodeStateCID,
},
Removed: true,
StorageDiff: []sdtypes.StorageLeafNode{},
},
{
Path: []byte{'\x07'},
NodeType: sdtypes.Removed,
LeafKey: Contract2LeafKey,
NodeValue: []byte{},
StorageNodes: []sdtypes.StorageNode{
AccountWrapper: sdtypes.AccountWrapper{
Account: nil,
LeafKey: Contract2LeafKey,
CID: shared.RemovedNodeStateCID,
},
Removed: true,
StorageDiff: []sdtypes.StorageLeafNode{
{
Path: []byte{'\x0e'},
NodeType: sdtypes.Removed,
LeafKey: Storage2LeafKey,
NodeValue: []byte{},
Removed: true,
CID: shared.RemovedNodeStorageCID,
LeafKey: Storage2LeafKey,
Value: []byte{},
},
{
Path: []byte{'\x0f'},
NodeType: sdtypes.Removed,
LeafKey: Storage3LeafKey,
NodeValue: []byte{},
Removed: true,
CID: shared.RemovedNodeStorageCID,
LeafKey: Storage3LeafKey,
Value: []byte{},
},
},
},
}
IPLDs = []sdtypes.IPLD{
{
CID: ContractLeafNodeCID,
Content: ContractLeafNode,
},
{
CID: StorageLeafNodeCID,
Content: StorageLeafNode,
},
{
CID: shared.RemovedNodeStorageCID,
Content: []byte{},
},
{
CID: AccountLeafNodeCID,
Content: AccountLeafNode,
},
{
CID: shared.RemovedNodeStateCID,
Content: []byte{},
},
}
// Mock data for testing watched addresses methods
Contract1Address = "0x5d663F5269090bD2A7DC2390c911dF6083D7b28F"
Contract2Address = "0x6Eb7e5C66DB8af2E96159AC440cbc8CDB7fbD26B"
@ -296,7 +324,7 @@ type LegacyData struct {
ContractLeafNode []byte
AccountRoot string
AccountLeafNode []byte
StateDiffs []sdtypes.StateNode
StateDiffs []sdtypes.StateLeafNode
}
func NewLegacyData(config *params.ChainConfig) *LegacyData {
@ -336,7 +364,7 @@ func NewLegacyData(config *params.ChainConfig) *LegacyData {
MockStorageLeafKey: MockStorageLeafKey,
StorageLeafNode: StorageLeafNode,
ContractLeafKey: ContractLeafKey,
ContractAccount: ContractAccount,
ContractAccount: ContractAccountRLP,
ContractPartialPath: ContractPartialPath,
ContractLeafNode: ContractLeafNode,
AccountRoot: AccountRoot,

21
statediff/indexer/models/models.go
View File

@ -18,7 +18,7 @@ package models
import "github.com/lib/pq"
// IPLDModel is the db model for public.blocks
// IPLDModel is the db model for ipld.blocks
type IPLDModel struct {
BlockNumber string `db:"block_number"`
Key string `db:"key"`
@ -69,14 +69,13 @@ type TxModel struct {
// ReceiptModel is the db model for eth.receipt_cids
type ReceiptModel struct {
BlockNumber string `db:"block_number"`
HeaderID string `db:"header_id"`
TxID string `db:"tx_id"`
CID string `db:"cid"`
PostStatus uint64 `db:"post_status"`
PostState string `db:"post_state"`
Contract string `db:"contract"`
ContractHash string `db:"contract_hash"`
BlockNumber string `db:"block_number"`
HeaderID string `db:"header_id"`
TxID string `db:"tx_id"`
CID string `db:"cid"`
PostStatus uint64 `db:"post_status"`
PostState string `db:"post_state"`
Contract string `db:"contract"`
}
// StateNodeModel is the db model for eth.state_cids
@ -89,7 +88,7 @@ type StateNodeModel struct {
Diff bool `db:"diff"`
Balance string `db:"balance"`
Nonce uint64 `db:"nonce"`
CodeHash []byte `db:"code_hash"`
CodeHash string `db:"code_hash"`
StorageRoot string `db:"storage_root"`
}
@ -97,7 +96,7 @@ type StateNodeModel struct {
type StorageNodeModel struct {
BlockNumber string `db:"block_number"`
HeaderID string `db:"header_id"`
StateKey []byte `db:"state_leaf_key"`
StateKey string `db:"state_leaf_key"`
StorageKey string `db:"storage_leaf_key"`
Removed bool `db:"removed"`
CID string `db:"cid"`

20
statediff/indexer/shared/functions.go
View File

@ -18,10 +18,6 @@ package shared
import (
"github.com/ethereum/go-ethereum/common"
"github.com/ipfs/go-cid"
blockstore "github.com/ipfs/go-ipfs-blockstore"
dshelp "github.com/ipfs/go-ipfs-ds-help"
"github.com/multiformats/go-multihash"
)
// HandleZeroAddrPointer will return an empty string for a nil address pointer
@ -39,19 +35,3 @@ func HandleZeroAddr(to common.Address) string {
}
return to.Hex()
}
// MultihashKeyFromCID converts a cid into a blockstore-prefixed multihash db key string
func MultihashKeyFromCID(c cid.Cid) string {
dbKey := dshelp.MultihashToDsKey(c.Hash())
return blockstore.BlockPrefix.String() + dbKey.String()
}
// MultihashKeyFromKeccak256 converts keccak256 hash bytes into a blockstore-prefixed multihash db key string
func MultihashKeyFromKeccak256(hash common.Hash) (string, error) {
mh, err := multihash.Encode(hash.Bytes(), multihash.KECCAK_256)
if err != nil {
return "", err
}
dbKey := dshelp.MultihashToDsKey(mh)
return blockstore.BlockPrefix.String() + dbKey.String(), nil
}

353
statediff/indexer/test/test.go
View File

@ -23,8 +23,6 @@ import (
"testing"
"github.com/ipfs/go-cid"
blockstore "github.com/ipfs/go-ipfs-blockstore"
dshelp "github.com/ipfs/go-ipfs-ds-help"
"github.com/stretchr/testify/require"
"github.com/ethereum/go-ethereum/common"
@ -58,6 +56,10 @@ func SetupTestData(t *testing.T, ind interfaces.StateDiffIndexer) {
err = ind.PushStateNode(tx, node, mockBlock.Hash().String())
require.NoError(t, err)
}
for _, node := range mocks.IPLDs {
err = ind.PushIPLD(tx, node)
require.NoError(t, err)
}
if batchTx, ok := tx.(*sql.BatchTx); ok {
require.Equal(t, mocks.BlockNumber.String(), batchTx.BlockNumber)
@ -96,10 +98,8 @@ func TestPublishAndIndexHeaderIPLDs(t *testing.T, db sql.Database) {
if err != nil {
t.Fatal(err)
}
mhKey := dshelp.MultihashToDsKey(dc.Hash())
prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
var data []byte
err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64())
err = db.Get(context.Background(), &data, ipfsPgGet, dc.String(), mocks.BlockNumber.Uint64())
if err != nil {
t.Fatal(err)
}
@ -132,10 +132,8 @@ func TestPublishAndIndexTransactionIPLDs(t *testing.T, db sql.Database) {
if err != nil {
t.Fatal(err)
}
mhKey := dshelp.MultihashToDsKey(dc.Hash())
prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
var data []byte
err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64())
err = db.Get(context.Background(), &data, ipfsPgGet, dc.String(), mocks.BlockNumber.Uint64())
if err != nil {
t.Fatal(err)
}
@ -193,30 +191,6 @@ func TestPublishAndIndexTransactionIPLDs(t *testing.T, db sql.Database) {
if txRes.Value != transactions[3].Value().String() {
t.Fatalf("expected tx value %s got %s", transactions[3].Value().String(), txRes.Value)
}
accessListElementModels := make([]models.AccessListElementModel, 0)
pgStr = "SELECT cast(access_list_elements.block_number AS TEXT), access_list_elements.index, access_list_elements.tx_id, " +
"access_list_elements.address, access_list_elements.storage_keys FROM eth.access_list_elements " +
"INNER JOIN eth.transaction_cids ON (tx_id = transaction_cids.tx_hash) WHERE cid = $1 ORDER BY access_list_elements.index ASC"
err = db.Select(context.Background(), &accessListElementModels, pgStr, c)
if err != nil {
t.Fatal(err)
}
if len(accessListElementModels) != 2 {
t.Fatalf("expected two access list entries, got %d", len(accessListElementModels))
}
model1 := models.AccessListElementModel{
BlockNumber: mocks.BlockNumber.String(),
Index: accessListElementModels[0].Index,
Address: accessListElementModels[0].Address,
}
model2 := models.AccessListElementModel{
BlockNumber: mocks.BlockNumber.String(),
Index: accessListElementModels[1].Index,
Address: accessListElementModels[1].Address,
StorageKeys: accessListElementModels[1].StorageKeys,
}
require.Equal(t, mocks.AccessListEntry1Model, model1)
require.Equal(t, mocks.AccessListEntry2Model, model2)
case trx5CID.String():
require.Equal(t, tx5, data)
txRes := new(txResult)
@ -236,15 +210,15 @@ func TestPublishAndIndexTransactionIPLDs(t *testing.T, db sql.Database) {
func TestPublishAndIndexLogIPLDs(t *testing.T, db sql.Database) {
rcts := make([]string, 0)
rctsPgStr := `SELECT receipt_cids.leaf_cid FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids
rctsPgStr := `SELECT receipt_cids.cid FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids
WHERE receipt_cids.tx_id = transaction_cids.tx_hash
AND transaction_cids.header_id = header_cids.block_hash
AND header_cids.block_number = $1
ORDER BY transaction_cids.index`
logsPgStr := `SELECT log_cids.index, log_cids.address, log_cids.topic0, log_cids.topic1, data FROM eth.log_cids
logsPgStr := `SELECT log_cids.index, log_cids.address, blocks.data, log_cids.topic0, log_cids.topic1 FROM eth.log_cids
INNER JOIN eth.receipt_cids ON (log_cids.rct_id = receipt_cids.tx_id)
INNER JOIN public.blocks ON (log_cids.leaf_mh_key = blocks.key)
WHERE receipt_cids.leaf_cid = $1 ORDER BY eth.log_cids.index ASC`
INNER JOIN ipld.blocks ON (log_cids.cid = blocks.key)
WHERE receipt_cids.cid = $1 ORDER BY eth.log_cids.index ASC`
err = db.Select(context.Background(), &rcts, rctsPgStr, mocks.BlockNumber.Uint64())
if err != nil {
t.Fatal(err)
@ -268,22 +242,10 @@ func TestPublishAndIndexLogIPLDs(t *testing.T, db sql.Database) {
expectedLogs := mocks.MockReceipts[i].Logs
require.Equal(t, len(expectedLogs), len(results))
var nodeElements []interface{}
for idx, r := range results {
// Attempt to decode the log leaf node.
err = rlp.DecodeBytes(r.Data, &nodeElements)
logRaw, err := rlp.EncodeToBytes(&expectedLogs[idx])
require.NoError(t, err)
if len(nodeElements) == 2 {
logRaw, err := rlp.EncodeToBytes(&expectedLogs[idx])
require.NoError(t, err)
// 2nd element of the leaf node contains the encoded log data.
require.Equal(t, nodeElements[1].([]byte), logRaw)
} else {
logRaw, err := rlp.EncodeToBytes(&expectedLogs[idx])
require.NoError(t, err)
// raw log was IPLDized
require.Equal(t, r.Data, logRaw)
}
require.Equal(t, r.Data, logRaw)
}
}
}
@ -291,7 +253,7 @@ func TestPublishAndIndexLogIPLDs(t *testing.T, db sql.Database) {
func TestPublishAndIndexReceiptIPLDs(t *testing.T, db sql.Database) {
// check receipts were properly indexed and published
rcts := make([]string, 0)
pgStr := `SELECT receipt_cids.leaf_cid FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids
pgStr := `SELECT receipt_cids.cid FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids
WHERE receipt_cids.tx_id = transaction_cids.tx_hash
AND transaction_cids.header_id = header_cids.block_hash
AND header_cids.block_number = $1 order by transaction_cids.index`
@ -309,49 +271,41 @@ func TestPublishAndIndexReceiptIPLDs(t *testing.T, db sql.Database) {
for idx, c := range rcts {
result := make([]models.IPLDModel, 0)
pgStr = `SELECT data
FROM eth.receipt_cids
INNER JOIN public.blocks ON (receipt_cids.leaf_mh_key = public.blocks.key)
WHERE receipt_cids.leaf_cid = $1`
FROM ipld.blocks
WHERE ipld.blocks.key = $1`
err = db.Select(context.Background(), &result, pgStr, c)
if err != nil {
t.Fatal(err)
}
// Decode the receipt leaf node.
var nodeElements []interface{}
err = rlp.DecodeBytes(result[0].Data, &nodeElements)
require.NoError(t, err)
expectedRct, err := mocks.MockReceipts[idx].MarshalBinary()
require.NoError(t, err)
require.Equal(t, nodeElements[1].([]byte), expectedRct)
require.Equal(t, result[0].Data, expectedRct)
dc, err := cid.Decode(c)
if err != nil {
t.Fatal(err)
}
mhKey := dshelp.MultihashToDsKey(dc.Hash())
prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
var data []byte
err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64())
err = db.Get(context.Background(), &data, ipfsPgGet, dc.String(), mocks.BlockNumber.Uint64())
if err != nil {
t.Fatal(err)
}
postStatePgStr := `SELECT post_state FROM eth.receipt_cids WHERE leaf_cid = $1`
postStatePgStr := `SELECT post_state FROM eth.receipt_cids WHERE cid = $1`
switch c {
case rct1CID.String():
require.Equal(t, rctLeaf1, data)
require.Equal(t, rct1, data)
var postStatus uint64
pgStr = `SELECT post_status FROM eth.receipt_cids WHERE leaf_cid = $1`
pgStr = `SELECT post_status FROM eth.receipt_cids WHERE cid = $1`
err = db.Get(context.Background(), &postStatus, pgStr, c)
if err != nil {
t.Fatal(err)
}
require.Equal(t, mocks.ExpectedPostStatus, postStatus)
case rct2CID.String():
require.Equal(t, rctLeaf2, data)
require.Equal(t, rct2, data)
var postState string
err = db.Get(context.Background(), &postState, postStatePgStr, c)
if err != nil {
@ -359,7 +313,7 @@ func TestPublishAndIndexReceiptIPLDs(t *testing.T, db sql.Database) {
}
require.Equal(t, mocks.ExpectedPostState1, postState)
case rct3CID.String():
require.Equal(t, rctLeaf3, data)
require.Equal(t, rct3, data)
var postState string
err = db.Get(context.Background(), &postState, postStatePgStr, c)
if err != nil {
@ -367,7 +321,7 @@ func TestPublishAndIndexReceiptIPLDs(t *testing.T, db sql.Database) {
}
require.Equal(t, mocks.ExpectedPostState2, postState)
case rct4CID.String():
require.Equal(t, rctLeaf4, data)
require.Equal(t, rct4, data)
var postState string
err = db.Get(context.Background(), &postState, postStatePgStr, c)
if err != nil {
@ -375,7 +329,7 @@ func TestPublishAndIndexReceiptIPLDs(t *testing.T, db sql.Database) {
}
require.Equal(t, mocks.ExpectedPostState3, postState)
case rct5CID.String():
require.Equal(t, rctLeaf5, data)
require.Equal(t, rct5, data)
var postState string
err = db.Get(context.Background(), &postState, postStatePgStr, c)
if err != nil {
@ -389,9 +343,11 @@ func TestPublishAndIndexReceiptIPLDs(t *testing.T, db sql.Database) {
func TestPublishAndIndexStateIPLDs(t *testing.T, db sql.Database) {
// check that state nodes were properly indexed and published
stateNodes := make([]models.StateNodeModel, 0)
pgStr := `SELECT state_cids.cid, state_cids.state_leaf_key, state_cids.node_type, state_cids.state_path, state_cids.header_id
FROM eth.state_cids INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.block_hash)
WHERE header_cids.block_number = $1 AND node_type != 3`
pgStr := `SELECT state_cids.cid, state_cids.block_number, state_cids.state_leaf_key, state_cids.removed,
state_cids.header_id, state_cids.balance, state_cids.nonce, state_cids.code_hash, state_cids.storage_root
FROM eth.state_cids
WHERE block_number = $1
AND removed = false`
err = db.Select(context.Background(), &stateNodes, pgStr, mocks.BlockNumber.Uint64())
if err != nil {
t.Fatal(err)
@ -403,56 +359,41 @@ func TestPublishAndIndexStateIPLDs(t *testing.T, db sql.Database) {
if err != nil {
t.Fatal(err)
}
mhKey := dshelp.MultihashToDsKey(dc.Hash())
prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64())
if err != nil {
t.Fatal(err)
}
pgStr = `SELECT cast(block_number AS TEXT), header_id, state_path, cast(balance AS TEXT), nonce, code_hash, storage_root from eth.state_accounts WHERE header_id = $1 AND state_path = $2`
var account models.StateAccountModel
err = db.Get(context.Background(), &account, pgStr, stateNode.HeaderID, stateNode.Path)
err = db.Get(context.Background(), &data, ipfsPgGet, dc.String(), mocks.BlockNumber.Uint64())
if err != nil {
t.Fatal(err)
}
if stateNode.CID == state1CID.String() {
require.Equal(t, 2, stateNode.NodeType)
require.Equal(t, false, stateNode.Removed)
require.Equal(t, common.BytesToHash(mocks.ContractLeafKey).Hex(), stateNode.StateKey)
require.Equal(t, []byte{'\x06'}, stateNode.Path)
require.Equal(t, mocks.ContractLeafNode, data)
require.Equal(t, models.StateAccountModel{
BlockNumber: mocks.BlockNumber.String(),
HeaderID: account.HeaderID,
StatePath: stateNode.Path,
Balance: "0",
CodeHash: mocks.ContractCodeHash.Bytes(),
StorageRoot: mocks.ContractRoot,
Nonce: 1,
}, account)
require.Equal(t, mocks.BlockNumber.String(), stateNode.BlockNumber)
require.Equal(t, "0", stateNode.Balance)
require.Equal(t, mocks.ContractCodeHash.String(), stateNode.CodeHash)
require.Equal(t, mocks.ContractRoot, stateNode.StorageRoot)
require.Equal(t, uint64(1), stateNode.Nonce)
require.Equal(t, mockBlock.Hash().String(), stateNode.HeaderID)
}
if stateNode.CID == state2CID.String() {
require.Equal(t, 2, stateNode.NodeType)
require.Equal(t, false, stateNode.Removed)
require.Equal(t, common.BytesToHash(mocks.AccountLeafKey).Hex(), stateNode.StateKey)
require.Equal(t, []byte{'\x0c'}, stateNode.Path)
require.Equal(t, mocks.AccountLeafNode, data)
require.Equal(t, models.StateAccountModel{
BlockNumber: mocks.BlockNumber.String(),
HeaderID: account.HeaderID,
StatePath: stateNode.Path,
Balance: "1000",
CodeHash: mocks.AccountCodeHash.Bytes(),
StorageRoot: mocks.AccountRoot,
Nonce: 0,
}, account)
require.Equal(t, mocks.BlockNumber.String(), stateNode.BlockNumber)
require.Equal(t, "1000", stateNode.Balance)
require.Equal(t, mocks.AccountCodeHash.String(), stateNode.CodeHash)
require.Equal(t, mocks.AccountRoot, stateNode.StorageRoot)
require.Equal(t, uint64(0), stateNode.Nonce)
require.Equal(t, mockBlock.Hash().String(), stateNode.HeaderID)
}
}
// check that Removed state nodes were properly indexed and published
stateNodes = make([]models.StateNodeModel, 0)
pgStr = `SELECT state_cids.cid, state_cids.state_leaf_key, state_cids.node_type, state_cids.state_path, state_cids.header_id
pgStr = `SELECT state_cids.cid, state_cids.state_leaf_key, state_cids.removed, state_cids.header_id,
state_cids.nonce, CAST(state_cids.balance as TEXT), state_cids.code_hash, state_cids.storage_root
FROM eth.state_cids INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.block_hash)
WHERE header_cids.block_number = $1 AND node_type = 3
ORDER BY state_path`
WHERE header_cids.block_number = $1 AND removed = true
ORDER BY state_leaf_key`
err = db.Select(context.Background(), &stateNodes, pgStr, mocks.BlockNumber.Uint64())
if err != nil {
t.Fatal(err)
@ -464,103 +405,114 @@ func TestPublishAndIndexStateIPLDs(t *testing.T, db sql.Database) {
if err != nil {
t.Fatal(err)
}
mhKey := dshelp.MultihashToDsKey(dc.Hash())
prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
require.Equal(t, shared.RemovedNodeMhKey, prefixedKey)
err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64())
require.Equal(t, shared.RemovedNodeStateCID, dc.String())
err = db.Get(context.Background(), &data, ipfsPgGet, dc.String(), mocks.BlockNumber.Uint64())
if err != nil {
t.Fatal(err)
}
if idx == 0 {
if idx == 1 { // TODO: ordering is non-deterministic
require.Equal(t, shared.RemovedNodeStateCID, stateNode.CID)
require.Equal(t, common.BytesToHash(mocks.RemovedLeafKey).Hex(), stateNode.StateKey)
require.Equal(t, []byte{'\x02'}, stateNode.Path)
require.Equal(t, true, stateNode.Removed)
require.Equal(t, []byte{}, data)
}
if idx == 1 {
if idx == 0 {
require.Equal(t, shared.RemovedNodeStateCID, stateNode.CID)
require.Equal(t, common.BytesToHash(mocks.Contract2LeafKey).Hex(), stateNode.StateKey)
require.Equal(t, []byte{'\x07'}, stateNode.Path)
require.Equal(t, true, stateNode.Removed)
require.Equal(t, []byte{}, data)
}
}
}
/*
type StorageNodeModel struct {
BlockNumber string `db:"block_number"`
HeaderID string `db:"header_id"`
StateKey []byte `db:"state_leaf_key"`
StorageKey string `db:"storage_leaf_key"`
Removed bool `db:"removed"`
CID string `db:"cid"`
Diff bool `db:"diff"`
Value []byte `db:"val"`
}
*/
func TestPublishAndIndexStorageIPLDs(t *testing.T, db sql.Database) {
// check that storage nodes were properly indexed
storageNodes := make([]models.StorageNodeWithStateKeyModel, 0)
pgStr := `SELECT cast(storage_cids.block_number AS TEXT), storage_cids.cid, state_cids.state_leaf_key, storage_cids.storage_leaf_key, storage_cids.node_type, storage_cids.storage_path
FROM eth.storage_cids, eth.state_cids, eth.header_cids
WHERE (storage_cids.state_path, storage_cids.header_id) = (state_cids.state_path, state_cids.header_id)
AND state_cids.header_id = header_cids.block_hash
AND header_cids.block_number = $1
AND storage_cids.node_type != 3
ORDER BY storage_path`
storageNodes := make([]models.StorageNodeModel, 0)
pgStr := `SELECT cast(storage_cids.block_number AS TEXT), storage_cids.header_id, storage_cids.cid,
storage_cids.state_leaf_key, storage_cids.storage_leaf_key, storage_cids.removed, storage_cids.val
FROM eth.storage_cids
WHERE storage_cids.block_number = $1
AND storage_cids.removed = false
ORDER BY storage_leaf_key`
err = db.Select(context.Background(), &storageNodes, pgStr, mocks.BlockNumber.Uint64())
if err != nil {
t.Fatal(err)
}
require.Equal(t, 1, len(storageNodes))
require.Equal(t, models.StorageNodeWithStateKeyModel{
require.Equal(t, models.StorageNodeModel{
BlockNumber: mocks.BlockNumber.String(),
HeaderID: mockBlock.Header().Hash().Hex(),
CID: storageCID.String(),
NodeType: 2,
Removed: false,
StorageKey: common.BytesToHash(mocks.StorageLeafKey).Hex(),
StateKey: common.BytesToHash(mocks.ContractLeafKey).Hex(),
Path: []byte{},
Value: mocks.StorageValue,
}, storageNodes[0])
var data []byte
dc, err := cid.Decode(storageNodes[0].CID)
if err != nil {
t.Fatal(err)
}
mhKey := dshelp.MultihashToDsKey(dc.Hash())
prefixedKey := blockstore.BlockPrefix.String() + mhKey.String()
err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64())
err = db.Get(context.Background(), &data, ipfsPgGet, dc.String(), mocks.BlockNumber.Uint64())
if err != nil {
t.Fatal(err)
}
require.Equal(t, mocks.StorageLeafNode, data)
// check that Removed storage nodes were properly indexed
storageNodes = make([]models.StorageNodeWithStateKeyModel, 0)
pgStr = `SELECT cast(storage_cids.block_number AS TEXT), storage_cids.cid, state_cids.state_leaf_key, storage_cids.storage_leaf_key, storage_cids.node_type, storage_cids.storage_path
FROM eth.storage_cids, eth.state_cids, eth.header_cids
WHERE (storage_cids.state_path, storage_cids.header_id) = (state_cids.state_path, state_cids.header_id)
AND state_cids.header_id = header_cids.block_hash
AND header_cids.block_number = $1
AND storage_cids.node_type = 3
ORDER BY storage_path`
storageNodes = make([]models.StorageNodeModel, 0)
pgStr = `SELECT cast(storage_cids.block_number AS TEXT), storage_cids.header_id, storage_cids.cid,
storage_cids.state_leaf_key, storage_cids.storage_leaf_key, storage_cids.removed, storage_cids.val
FROM eth.storage_cids
WHERE storage_cids.block_number = $1
AND storage_cids.removed = true
ORDER BY storage_leaf_key`
err = db.Select(context.Background(), &storageNodes, pgStr, mocks.BlockNumber.Uint64())
if err != nil {
t.Fatal(err)
}
require.Equal(t, 3, len(storageNodes))
expectedStorageNodes := []models.StorageNodeWithStateKeyModel{
expectedStorageNodes := []models.StorageNodeModel{ // TODO: ordering is non-deterministic
{
BlockNumber: mocks.BlockNumber.String(),
HeaderID: mockBlock.Header().Hash().Hex(),
CID: shared.RemovedNodeStorageCID,
NodeType: 3,
StorageKey: common.BytesToHash(mocks.RemovedLeafKey).Hex(),
StateKey: common.BytesToHash(mocks.ContractLeafKey).Hex(),
Path: []byte{'\x03'},
},
{
BlockNumber: mocks.BlockNumber.String(),
CID: shared.RemovedNodeStorageCID,
NodeType: 3,
Removed: true,
StorageKey: common.BytesToHash(mocks.Storage2LeafKey).Hex(),
StateKey: common.BytesToHash(mocks.Contract2LeafKey).Hex(),
Path: []byte{'\x0e'},
Value: []byte{},
},
{
BlockNumber: mocks.BlockNumber.String(),
HeaderID: mockBlock.Header().Hash().Hex(),
CID: shared.RemovedNodeStorageCID,
NodeType: 3,
Removed: true,
StorageKey: common.BytesToHash(mocks.Storage3LeafKey).Hex(),
StateKey: common.BytesToHash(mocks.Contract2LeafKey).Hex(),
Path: []byte{'\x0f'},
Value: []byte{},
},
{
BlockNumber: mocks.BlockNumber.String(),
HeaderID: mockBlock.Header().Hash().Hex(),
CID: shared.RemovedNodeStorageCID,
Removed: true,
StorageKey: common.BytesToHash(mocks.RemovedLeafKey).Hex(),
StateKey: common.BytesToHash(mocks.ContractLeafKey).Hex(),
Value: []byte{},
},
}
for idx, storageNode := range storageNodes {
@ -569,10 +521,8 @@ func TestPublishAndIndexStorageIPLDs(t *testing.T, db sql.Database) {
if err != nil {
t.Fatal(err)
}
mhKey = dshelp.MultihashToDsKey(dc.Hash())
prefixedKey = blockstore.BlockPrefix.String() + mhKey.String()
require.Equal(t, shared.RemovedNodeMhKey, prefixedKey)
err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64())
require.Equal(t, shared.RemovedNodeStorageCID, dc.String())
err = db.Get(context.Background(), &data, ipfsPgGet, dc.String(), mocks.BlockNumber.Uint64())
if err != nil {
t.Fatal(err)
}
@ -662,7 +612,7 @@ func SetupTestDataNonCanonical(t *testing.T, ind interfaces.StateDiffIndexer) {
func TestPublishAndIndexHeaderNonCanonical(t *testing.T, db sql.Database) {
// check indexed headers
pgStr := `SELECT CAST(block_number as TEXT), block_hash, cid, cast(td AS TEXT), cast(reward AS TEXT),
tx_root, receipt_root, uncle_root, coinbase
tx_root, receipt_root, uncles_hash, coinbase
FROM eth.header_cids
ORDER BY block_number`
headerRes := make([]models.HeaderModel, 0)
@ -682,7 +632,7 @@ func TestPublishAndIndexHeaderNonCanonical(t *testing.T, db sql.Database) {
TotalDifficulty: mockBlock.Difficulty().String(),
TxRoot: mockBlock.TxHash().String(),
RctRoot: mockBlock.ReceiptHash().String(),
UncleRoot: mockBlock.UncleHash().String(),
UnclesHash: mockBlock.UncleHash().String(),
Coinbase: mocks.MockHeader.Coinbase.String(),
},
{
@ -692,7 +642,7 @@ func TestPublishAndIndexHeaderNonCanonical(t *testing.T, db sql.Database) {
TotalDifficulty: mockNonCanonicalBlock.Difficulty().String(),
TxRoot: mockNonCanonicalBlock.TxHash().String(),
RctRoot: mockNonCanonicalBlock.ReceiptHash().String(),
UncleRoot: mockNonCanonicalBlock.UncleHash().String(),
UnclesHash: mockNonCanonicalBlock.UncleHash().String(),
Coinbase: mocks.MockNonCanonicalHeader.Coinbase.String(),
},
{
@ -702,7 +652,7 @@ func TestPublishAndIndexHeaderNonCanonical(t *testing.T, db sql.Database) {
TotalDifficulty: mockNonCanonicalBlock2.Difficulty().String(),
TxRoot: mockNonCanonicalBlock2.TxHash().String(),
RctRoot: mockNonCanonicalBlock2.ReceiptHash().String(),
UncleRoot: mockNonCanonicalBlock2.UncleHash().String(),
UnclesHash: mockNonCanonicalBlock2.UncleHash().String(),
Coinbase: mocks.MockNonCanonicalHeader2.Coinbase.String(),
},
}
@ -732,8 +682,7 @@ func TestPublishAndIndexHeaderNonCanonical(t *testing.T, db sql.Database) {
headerRLPs := [][]byte{mocks.MockHeaderRlp, mocks.MockNonCanonicalHeaderRlp, mocks.MockNonCanonicalHeader2Rlp}
for i := range expectedRes {
var data []byte
prefixedKey := shared.MultihashKeyFromCID(headerCIDs[i])
err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, blockNumbers[i])
err = db.Get(context.Background(), &data, ipfsPgGet, headerCIDs[i].String(), blockNumbers[i])
if err != nil {
t.Fatal(err)
}
@ -744,7 +693,7 @@ func TestPublishAndIndexHeaderNonCanonical(t *testing.T, db sql.Database) {
func TestPublishAndIndexTransactionsNonCanonical(t *testing.T, db sql.Database) {
// check indexed transactions
pgStr := `SELECT CAST(block_number as TEXT), header_id, tx_hash, cid, dst, src, index,
tx_data, tx_type, CAST(value as TEXT)
tx_type, CAST(value as TEXT)
FROM eth.transaction_cids
ORDER BY block_number, index`
txRes := make([]models.TxModel, 0)
@ -764,7 +713,6 @@ func TestPublishAndIndexTransactionsNonCanonical(t *testing.T, db sql.Database)
Dst: shared.HandleZeroAddrPointer(mockBlockTxs[0].To()),
Src: mocks.SenderAddr.String(),
Index: 0,
Data: mockBlockTxs[0].Data(),
Type: mockBlockTxs[0].Type(),
Value: mockBlockTxs[0].Value().String(),
},
@ -776,7 +724,6 @@ func TestPublishAndIndexTransactionsNonCanonical(t *testing.T, db sql.Database)
Dst: shared.HandleZeroAddrPointer(mockBlockTxs[1].To()),
Src: mocks.SenderAddr.String(),
Index: 1,
Data: mockBlockTxs[1].Data(),
Type: mockBlockTxs[1].Type(),
Value: mockBlockTxs[1].Value().String(),
},
@ -788,7 +735,6 @@ func TestPublishAndIndexTransactionsNonCanonical(t *testing.T, db sql.Database)
Dst: shared.HandleZeroAddrPointer(mockBlockTxs[2].To()),
Src: mocks.SenderAddr.String(),
Index: 2,
Data: mockBlockTxs[2].Data(),
Type: mockBlockTxs[2].Type(),
Value: mockBlockTxs[2].Value().String(),
},
@ -800,7 +746,6 @@ func TestPublishAndIndexTransactionsNonCanonical(t *testing.T, db sql.Database)
Dst: shared.HandleZeroAddrPointer(mockBlockTxs[3].To()),
Src: mocks.SenderAddr.String(),
Index: 3,
Data: mockBlockTxs[3].Data(),
Type: mockBlockTxs[3].Type(),
Value: mockBlockTxs[3].Value().String(),
},
@ -812,7 +757,6 @@ func TestPublishAndIndexTransactionsNonCanonical(t *testing.T, db sql.Database)
Dst: shared.HandleZeroAddrPointer(mockBlockTxs[4].To()),
Src: mocks.SenderAddr.String(),
Index: 4,
Data: mockBlockTxs[4].Data(),
Type: mockBlockTxs[4].Type(),
Value: mockBlockTxs[4].Value().String(),
},
@ -829,7 +773,6 @@ func TestPublishAndIndexTransactionsNonCanonical(t *testing.T, db sql.Database)
Dst: mockNonCanonicalBlockTxs[0].To().String(),
Src: mocks.SenderAddr.String(),
Index: 0,
Data: mockNonCanonicalBlockTxs[0].Data(),
Type: mockNonCanonicalBlockTxs[0].Type(),
Value: mockNonCanonicalBlockTxs[0].Value().String(),
},
@ -841,7 +784,6 @@ func TestPublishAndIndexTransactionsNonCanonical(t *testing.T, db sql.Database)
Dst: mockNonCanonicalBlockTxs[1].To().String(),
Src: mocks.SenderAddr.String(),
Index: 1,
Data: mockNonCanonicalBlockTxs[1].Data(),
Type: mockNonCanonicalBlockTxs[1].Type(),
Value: mockNonCanonicalBlockTxs[1].Value().String(),
},
@ -858,7 +800,6 @@ func TestPublishAndIndexTransactionsNonCanonical(t *testing.T, db sql.Database)
Dst: "",
Src: mocks.SenderAddr.String(),
Index: 0,
Data: mockNonCanonicalBlock2Txs[0].Data(),
Type: mockNonCanonicalBlock2Txs[0].Type(),
Value: mockNonCanonicalBlock2Txs[0].Value().String(),
},
@ -870,7 +811,6 @@ func TestPublishAndIndexTransactionsNonCanonical(t *testing.T, db sql.Database)
Dst: mockNonCanonicalBlock2Txs[1].To().String(),
Src: mocks.SenderAddr.String(),
Index: 1,
Data: mockNonCanonicalBlock2Txs[1].Data(),
Type: mockNonCanonicalBlock2Txs[1].Type(),
Value: mockNonCanonicalBlock2Txs[1].Value().String(),
},
@ -903,13 +843,11 @@ func TestPublishAndIndexTransactionsNonCanonical(t *testing.T, db sql.Database)
// check indexed IPLD blocks
var data []byte
var prefixedKey string
txCIDs := []cid.Cid{trx1CID, trx2CID, trx3CID, trx4CID, trx5CID}
txRLPs := [][]byte{tx1, tx2, tx3, tx4, tx5}
for i, txCID := range txCIDs {
prefixedKey = shared.MultihashKeyFromCID(txCID)
err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64())
err = db.Get(context.Background(), &data, ipfsPgGet, txCID.String(), mocks.BlockNumber.Uint64())
if err != nil {
t.Fatal(err)
}
@ -919,7 +857,7 @@ func TestPublishAndIndexTransactionsNonCanonical(t *testing.T, db sql.Database)
func TestPublishAndIndexReceiptsNonCanonical(t *testing.T, db sql.Database) {
// check indexed receipts
pgStr := `SELECT CAST(block_number as TEXT), header_id, tx_id, leaf_cid, leaf_mh_key, post_status, post_state, contract, contract_hash, log_root
pgStr := `SELECT CAST(block_number as TEXT), header_id, tx_id, cid, post_status, post_state, contract
FROM eth.receipt_cids
ORDER BY block_number`
rctRes := make([]models.ReceiptModel, 0)
@ -969,33 +907,30 @@ func TestPublishAndIndexReceiptsNonCanonical(t *testing.T, db sql.Database) {
for i := 0; i < len(expectedBlockRctsMap); i++ {
rct := rctRes[i]
require.Contains(t, expectedBlockRctsMap, rct.LeafCID)
require.Equal(t, expectedBlockRctsMap[rct.LeafCID], rct)
require.Contains(t, expectedBlockRctsMap, rct.CID)
require.Equal(t, expectedBlockRctsMap[rct.CID], rct)
}
for i := 0; i < len(expectedNonCanonicalBlockRctsMap); i++ {
rct := rctRes[len(expectedBlockRctsMap)+i]
require.Contains(t, expectedNonCanonicalBlockRctsMap, rct.LeafCID)
require.Equal(t, expectedNonCanonicalBlockRctsMap[rct.LeafCID], rct)
require.Contains(t, expectedNonCanonicalBlockRctsMap, rct.CID)
require.Equal(t, expectedNonCanonicalBlockRctsMap[rct.CID], rct)
}
for i := 0; i < len(expectedNonCanonicalBlock2RctsMap); i++ {
rct := rctRes[len(expectedBlockRctsMap)+len(expectedNonCanonicalBlockRctsMap)+i]
require.Contains(t, expectedNonCanonicalBlock2RctsMap, rct.LeafCID)
require.Equal(t, expectedNonCanonicalBlock2RctsMap[rct.LeafCID], rct)
require.Contains(t, expectedNonCanonicalBlock2RctsMap, rct.CID)
require.Equal(t, expectedNonCanonicalBlock2RctsMap[rct.CID], rct)
}
// check indexed rct IPLD blocks
var data []byte
var prefixedKey string
rctRLPs := [][]byte{
rctLeaf1, rctLeaf2, rctLeaf3, rctLeaf4, rctLeaf5,
nonCanonicalBlockRctLeaf1, nonCanonicalBlockRctLeaf2,
}
for i, rctCid := range append(rctCids, nonCanonicalBlockRctCids...) {
prefixedKey = shared.MultihashKeyFromCID(rctCid)
err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.BlockNumber.Uint64())
err = db.Get(context.Background(), &data, ipfsPgGet, rctCid.String(), mocks.BlockNumber.Uint64())
if err != nil {
t.Fatal(err)
}
@ -1004,8 +939,7 @@ func TestPublishAndIndexReceiptsNonCanonical(t *testing.T, db sql.Database) {
nonCanonicalBlock2RctRLPs := [][]byte{nonCanonicalBlock2RctLeaf1, nonCanonicalBlock2RctLeaf2}
for i, rctCid := range nonCanonicalBlock2RctCids {
prefixedKey = shared.MultihashKeyFromCID(rctCid)
err = db.Get(context.Background(), &data, ipfsPgGet, prefixedKey, mocks.Block2Number.Uint64())
err = db.Get(context.Background(), &data, ipfsPgGet, rctCid.String(), mocks.Block2Number.Uint64())
if err != nil {
t.Fatal(err)
}
@ -1015,9 +949,9 @@ func TestPublishAndIndexReceiptsNonCanonical(t *testing.T, db sql.Database) {
func TestPublishAndIndexLogsNonCanonical(t *testing.T, db sql.Database) {
// check indexed logs
pgStr := `SELECT address, log_data, topic0, topic1, topic2, topic3, data
pgStr := `SELECT address, topic0, topic1, topic2, topic3
FROM eth.log_cids
INNER JOIN public.blocks ON (log_cids.block_number = blocks.block_number AND log_cids.leaf_mh_key = blocks.key)
INNER JOIN ipld.blocks ON (log_cids.block_number = blocks.block_number AND log_cids.cid = blocks.key)
WHERE log_cids.block_number = $1 AND header_id = $2 AND rct_id = $3
ORDER BY log_cids.index ASC`
@ -1073,7 +1007,6 @@ func TestPublishAndIndexLogsNonCanonical(t *testing.T, db sql.Database) {
expectedLog := models.LogsModel{
Address: log.Address.String(),
Data: log.Data,
Topic0: topicSet[0],
Topic1: topicSet[1],
Topic2: topicSet[2],
@ -1103,11 +1036,11 @@ func TestPublishAndIndexLogsNonCanonical(t *testing.T, db sql.Database) {
func TestPublishAndIndexStateNonCanonical(t *testing.T, db sql.Database) {
// check indexed state nodes
pgStr := `SELECT state_path, state_leaf_key, node_type, cid, mh_key, diff
pgStr := `SELECT state_leaf_key, removed, cid, diff
FROM eth.state_cids
WHERE block_number = $1
AND header_id = $2
ORDER BY state_path`
ORDER BY state_leaf_key`
removedNodeCID, _ := cid.Decode(shared.RemovedNodeStateCID)
stateNodeCIDs := []cid.Cid{state1CID, state2CID, removedNodeCID, removedNodeCID}
@ -1116,16 +1049,14 @@ func TestPublishAndIndexStateNonCanonical(t *testing.T, db sql.Database) {
expectedStateNodes := make([]models.StateNodeModel, 0)
for i, stateDiff := range mocks.StateDiffs {
expectedStateNodes = append(expectedStateNodes, models.StateNodeModel{
Path: stateDiff.Path,
StateKey: common.BytesToHash(stateDiff.LeafKey).Hex(),
NodeType: stateDiff.NodeType.Int(),
StateKey: common.BytesToHash(stateDiff.AccountWrapper.LeafKey).Hex(),
Removed: stateDiff.Removed,
CID: stateNodeCIDs[i].String(),
MhKey: shared.MultihashKeyFromCID(stateNodeCIDs[i]),
Diff: true,
})
}
sort.Slice(expectedStateNodes, func(i, j int) bool {
if bytes.Compare(expectedStateNodes[i].Path, expectedStateNodes[j].Path) < 0 {
if bytes.Compare(common.Hex2Bytes(expectedStateNodes[i].StateKey), common.Hex2Bytes(expectedStateNodes[j].StateKey)) < 0 {
return true
} else {
return false
@ -1136,11 +1067,9 @@ func TestPublishAndIndexStateNonCanonical(t *testing.T, db sql.Database) {
expectedNonCanonicalBlock2StateNodes := make([]models.StateNodeModel, 0)
for i, stateDiff := range mocks.StateDiffs[:2] {
expectedNonCanonicalBlock2StateNodes = append(expectedNonCanonicalBlock2StateNodes, models.StateNodeModel{
Path: stateDiff.Path,
StateKey: common.BytesToHash(stateDiff.LeafKey).Hex(),
NodeType: stateDiff.NodeType.Int(),
StateKey: common.BytesToHash(stateDiff.AccountWrapper.LeafKey).Hex(),
Removed: stateDiff.Removed,
CID: stateNodeCIDs[i].String(),
MhKey: shared.MultihashKeyFromCID(stateNodeCIDs[i]),
Diff: true,
})
}
@ -1184,11 +1113,11 @@ func TestPublishAndIndexStateNonCanonical(t *testing.T, db sql.Database) {
func TestPublishAndIndexStorageNonCanonical(t *testing.T, db sql.Database) {
// check indexed storage nodes
pgStr := `SELECT state_path, storage_path, storage_leaf_key, node_type, cid, mh_key, diff
pgStr := `SELECT storage_leaf_key, state_leaf_key, removed, cid, diff, val
FROM eth.storage_cids
WHERE block_number = $1
AND header_id = $2
ORDER BY state_path, storage_path`
ORDER BY state_leaf_key, storage_leaf_key`
removedNodeCID, _ := cid.Decode(shared.RemovedNodeStorageCID)
storageNodeCIDs := []cid.Cid{storageCID, removedNodeCID, removedNodeCID, removedNodeCID}
@ -1197,21 +1126,20 @@ func TestPublishAndIndexStorageNonCanonical(t *testing.T, db sql.Database) {
expectedStorageNodes := make([]models.StorageNodeModel, 0)
storageNodeIndex := 0
for _, stateDiff := range mocks.StateDiffs {
for _, storageNode := range stateDiff.StorageNodes {
for _, storageNode := range stateDiff.StorageDiff {
expectedStorageNodes = append(expectedStorageNodes, models.StorageNodeModel{
StatePath: stateDiff.Path,
Path: storageNode.Path,
StateKey: common.BytesToHash(stateDiff.AccountWrapper.LeafKey).Hex(),
StorageKey: common.BytesToHash(storageNode.LeafKey).Hex(),
NodeType: storageNode.NodeType.Int(),
Removed: storageNode.Removed,
CID: storageNodeCIDs[storageNodeIndex].String(),
MhKey: shared.MultihashKeyFromCID(storageNodeCIDs[storageNodeIndex]),
Diff: true,
Value: storageNode.Value,
})
storageNodeIndex++
}
}
sort.Slice(expectedStorageNodes, func(i, j int) bool {
if bytes.Compare(expectedStorageNodes[i].Path, expectedStorageNodes[j].Path) < 0 {
if bytes.Compare(common.Hex2Bytes(expectedStorageNodes[i].StorageKey), common.Hex2Bytes(expectedStorageNodes[j].StorageKey)) < 0 {
return true
} else {
return false
@ -1222,15 +1150,14 @@ func TestPublishAndIndexStorageNonCanonical(t *testing.T, db sql.Database) {
expectedNonCanonicalBlock2StorageNodes := make([]models.StorageNodeModel, 0)
storageNodeIndex = 0
for _, stateDiff := range mocks.StateDiffs[:2] {
for _, storageNode := range stateDiff.StorageNodes {
for _, storageNode := range stateDiff.StorageDiff {
expectedNonCanonicalBlock2StorageNodes = append(expectedNonCanonicalBlock2StorageNodes, models.StorageNodeModel{
StatePath: stateDiff.Path,
Path: storageNode.Path,
StateKey: common.BytesToHash(stateDiff.AccountWrapper.LeafKey).Hex(),
StorageKey: common.BytesToHash(storageNode.LeafKey).Hex(),
NodeType: storageNode.NodeType.Int(),
Removed: storageNode.Removed,
CID: storageNodeCIDs[storageNodeIndex].String(),
MhKey: shared.MultihashKeyFromCID(storageNodeCIDs[storageNodeIndex]),
Diff: true,
Value: storageNode.Value,
})
storageNodeIndex++
}

74
statediff/indexer/test/test_init.go
View File

@ -24,8 +24,6 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
"github.com/ethereum/go-ethereum/statediff/indexer/mocks"
"github.com/ethereum/go-ethereum/statediff/indexer/models"
@ -36,7 +34,7 @@ import (
var (
err error
ipfsPgGet = `SELECT data FROM public.blocks
ipfsPgGet = `SELECT data FROM ipld.blocks
WHERE key = $1 AND block_number = $2`
watchedAddressesPgGet = `SELECT *
FROM eth_meta.watched_addresses`
@ -49,7 +47,6 @@ var (
rct1CID, rct2CID, rct3CID, rct4CID, rct5CID cid.Cid
nonCanonicalBlockRct1CID, nonCanonicalBlockRct2CID cid.Cid
nonCanonicalBlock2Rct1CID, nonCanonicalBlock2Rct2CID cid.Cid
rctLeaf1, rctLeaf2, rctLeaf3, rctLeaf4, rctLeaf5 []byte
nonCanonicalBlockRctLeaf1, nonCanonicalBlockRctLeaf2 []byte
nonCanonicalBlock2RctLeaf1, nonCanonicalBlock2RctLeaf2 []byte
state1CID, state2CID, storageCID cid.Cid
@ -157,62 +154,18 @@ func init() {
state1CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, mocks.ContractLeafNode, multihash.KECCAK_256)
state2CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, mocks.AccountLeafNode, multihash.KECCAK_256)
storageCID, _ = ipld.RawdataToCid(ipld.MEthStorageTrie, mocks.StorageLeafNode, multihash.KECCAK_256)
// create raw receipts
rawRctLeafNodes, rctleafNodeCids := createRctTrie([][]byte{rct1, rct2, rct3, rct4, rct5})
rct1CID = rctleafNodeCids[0]
rct2CID = rctleafNodeCids[1]
rct3CID = rctleafNodeCids[2]
rct4CID = rctleafNodeCids[3]
rct5CID = rctleafNodeCids[4]
rctLeaf1 = rawRctLeafNodes[0]
rctLeaf2 = rawRctLeafNodes[1]
rctLeaf3 = rawRctLeafNodes[2]
rctLeaf4 = rawRctLeafNodes[3]
rctLeaf5 = rawRctLeafNodes[4]
rct1CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, rct1, multihash.KECCAK_256)
rct2CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, rct2, multihash.KECCAK_256)
rct3CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, rct3, multihash.KECCAK_256)
rct4CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, rct4, multihash.KECCAK_256)
rct5CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, rct5, multihash.KECCAK_256)
// create raw receipts for non-canonical blocks
nonCanonicalBlockRawRctLeafNodes, nonCanonicalBlockRctLeafNodeCids := createRctTrie([][]byte{nonCanonicalBlockRct1, nonCanonicalBlockRct2})
nonCanonicalBlockRct1CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, nonCanonicalBlockRct1, multihash.KECCAK_256)
nonCanonicalBlockRct2CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, nonCanonicalBlockRct2, multihash.KECCAK_256)
nonCanonicalBlockRct1CID = nonCanonicalBlockRctLeafNodeCids[0]
nonCanonicalBlockRct2CID = nonCanonicalBlockRctLeafNodeCids[1]
nonCanonicalBlockRctLeaf1 = nonCanonicalBlockRawRctLeafNodes[0]
nonCanonicalBlockRctLeaf2 = nonCanonicalBlockRawRctLeafNodes[1]
nonCanonicalBlock2RawRctLeafNodes, nonCanonicalBlock2RctLeafNodeCids := createRctTrie([][]byte{nonCanonicalBlockRct1, nonCanonicalBlockRct2})
nonCanonicalBlock2Rct1CID = nonCanonicalBlock2RctLeafNodeCids[0]
nonCanonicalBlock2Rct2CID = nonCanonicalBlock2RctLeafNodeCids[1]
nonCanonicalBlock2RctLeaf1 = nonCanonicalBlock2RawRctLeafNodes[0]
nonCanonicalBlock2RctLeaf2 = nonCanonicalBlock2RawRctLeafNodes[1]
}
// createRctTrie creates a receipt trie from the given raw receipts
// returns receipt leaf nodes and their CIDs
func createRctTrie(rcts [][]byte) ([][]byte, []cid.Cid) {
receiptTrie := ipld.NewRctTrie()
for i, rct := range rcts {
receiptTrie.Add(i, rct)
}
rctLeafNodes, keys, _ := receiptTrie.GetLeafNodes()
rctleafNodeCids := make([]cid.Cid, len(rctLeafNodes))
orderedRctLeafNodes := make([][]byte, len(rctLeafNodes))
for i, rln := range rctLeafNodes {
var idx uint
r := bytes.NewReader(keys[i].TrieKey)
rlp.Decode(r, &idx)
rctleafNodeCids[idx] = rln.Cid()
orderedRctLeafNodes[idx] = rln.RawData()
}
return orderedRctLeafNodes, rctleafNodeCids
nonCanonicalBlock2Rct1CID = nonCanonicalBlockRct1CID
nonCanonicalBlock2Rct2CID = nonCanonicalBlockRct2CID
}
// createRctModel creates a models.ReceiptModel object from a given ethereum receipt
@ -221,16 +174,11 @@ func createRctModel(rct *types.Receipt, cid cid.Cid, blockNumber string) models.
BlockNumber: blockNumber,
HeaderID: rct.BlockHash.String(),
TxID: rct.TxHash.String(),
LeafCID: cid.String(),
LeafMhKey: shared.MultihashKeyFromCID(cid),
LogRoot: rct.LogRoot.String(),
CID: cid.String(),
}
contract := shared.HandleZeroAddr(rct.ContractAddress)
rctModel.Contract = contract
if contract != "" {
rctModel.ContractHash = crypto.Keccak256Hash(common.HexToAddress(contract).Bytes()).String()
}
if len(rct.PostState) == 0 {
rctModel.PostStatus = rct.Status

10
statediff/indexer/test_helpers/test_helpers.go
View File

@ -100,19 +100,11 @@ func TearDownDB(t *testing.T, db sql.Database) {
if err != nil {
t.Fatal(err)
}
_, err = tx.Exec(ctx, `DELETE FROM eth.state_accounts`)
if err != nil {
t.Fatal(err)
}
_, err = tx.Exec(ctx, `DELETE FROM eth.access_list_elements`)
if err != nil {
t.Fatal(err)
}
_, err = tx.Exec(ctx, `DELETE FROM eth.log_cids`)
if err != nil {
t.Fatal(err)
}
_, err = tx.Exec(ctx, `DELETE FROM blocks`)
_, err = tx.Exec(ctx, `DELETE FROM ipld.blocks`)
if err != nil {
t.Fatal(err)
}

2
statediff/service.go
View File

@ -797,7 +797,7 @@ func (sds *Service) writeStateDiff(block *types.Block, parentRoot common.Hash, p
return sds.indexer.PushStateNode(tx, node, block.Hash().String())
}
ipldOutput := func(c types2.IPLD) error {
return sds.indexer.PushIPLD(tx, c)
return sds.indexer.f(tx, c)
}
err = sds.Builder.WriteStateDiffObject(types2.StateRoots{

2
statediff/service_test.go
View File

@ -16,6 +16,7 @@
package statediff_test
/*
import (
"bytes"
"math/big"
@ -437,3 +438,4 @@ func testGetSyncStatus(t *testing.T) {
}
}
}
*/