cerc-io/ipld-eth-server
12
0
Fork 1
Code Issues 10 Pull Requests 2 Actions Packages Projects Releases 37 Wiki Activity

Remove ipld-eth-indexer dependency.

Browse Source
This commit is contained in:
Arijit Das
2021-08-12 11:56:29 +05:30
parent e00e602098
commit d8a5358a70
28 changed files with 468 additions and 279 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/eth/api_test.go
+36 -10
View File
@@ -30,13 +30,12 @@ import (
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/rpc"
"github.com/ethereum/go-ethereum/statediff/indexer"
"github.com/ethereum/go-ethereum/statediff/indexer/node"
"github.com/ethereum/go-ethereum/statediff/indexer/postgres"
"github.com/ethereum/go-ethereum/statediff/indexer/shared"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
eth2 "github.com/vulcanize/ipld-eth-indexer/pkg/eth"
"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
"github.com/vulcanize/ipld-eth-indexer/pkg/shared"
"github.com/vulcanize/ipld-eth-server/pkg/eth"
"github.com/vulcanize/ipld-eth-server/pkg/eth/test_helpers"
)
@@ -182,6 +181,18 @@ var (
}
)
// SetupDB is use to setup a db for watcher tests
func SetupDB() (*postgres.DB, error) {
uri := postgres.DbConnectionString(postgres.ConnectionParams{
User: "vdbm",
Password: "password",
Hostname: "localhost",
Name: "vulcanize_testing",
Port: 8077,
})
return postgres.NewDB(uri, postgres.ConnectionConfig{}, node.Info{})
}
var _ = Describe("API", func() {
var (
db *postgres.DB
@@ -191,10 +202,14 @@ var _ = Describe("API", func() {
// Test db setup, rather than using BeforeEach we only need to setup once since the tests do not mutate the database
// Note: if you focus one of the tests be sure to focus this and the defered It()
It("test init", func() {
var err error
db, err = shared.SetupDB()
var (
err error
tx *indexer.BlockTx
)
db, err = SetupDB()
Expect(err).ToNot(HaveOccurred())
indexAndPublisher := eth2.NewIPLDPublisher(db)
indexAndPublisher := indexer.NewStateDiffIndexer(chainConfig, db)
backend, err := eth.NewEthBackend(db, &eth.Config{
ChainConfig: chainConfig,
VmConfig: vm.Config{},
@@ -202,18 +217,29 @@ var _ = Describe("API", func() {
})
Expect(err).ToNot(HaveOccurred())
api = eth.NewPublicEthAPI(backend, nil, false)
err = indexAndPublisher.Publish(test_helpers.MockConvertedPayload)
tx, err = indexAndPublisher.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
Expect(err).ToNot(HaveOccurred())
for _, node := range test_helpers.MockStateNodes {
err = indexAndPublisher.PushStateNode(tx, node)
Expect(err).ToNot(HaveOccurred())
}
err = tx.Close(err)
Expect(err).ToNot(HaveOccurred())
err = publishCode(db, test_helpers.ContractCodeHash, test_helpers.ContractCode)
Expect(err).ToNot(HaveOccurred())
uncles := test_helpers.MockBlock.Uncles()
uncleHashes := make([]common.Hash, len(uncles))
for i, uncle := range uncles {
uncleHashes[i] = uncle.Hash()
}
expectedBlock["uncles"] = uncleHashes
tx, err = indexAndPublisher.PushBlock(test_helpers.MockLondonBlock, test_helpers.MockLondonReceipts, test_helpers.MockLondonBlock.Difficulty())
Expect(err).ToNot(HaveOccurred())
err = indexAndPublisher.Publish(test_helpers.MockConvertedLondonPayload)
err = tx.Close(err)
Expect(err).ToNot(HaveOccurred())
})
pkg/eth/backend.go
+4 -6
View File
@@ -39,14 +39,12 @@ import (
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/rpc"
"github.com/ethereum/go-ethereum/statediff/indexer/postgres"
"github.com/ethereum/go-ethereum/trie"
"github.com/ethereum/go-ethereum/statediff/indexer/ipfs"
"github.com/ethereum/go-ethereum/statediff/indexer/shared"
ipfsethdb "github.com/vulcanize/ipfs-ethdb"
"github.com/vulcanize/ipld-eth-indexer/pkg/ipfs"
"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
shared2 "github.com/vulcanize/ipld-eth-indexer/pkg/shared"
"github.com/vulcanize/ipld-eth-server/pkg/shared"
)
var (
@@ -714,7 +712,7 @@ func (b *Backend) GetCodeByHash(ctx context.Context, address common.Address, has
return nil, err
}
var mhKey string
mhKey, err = shared2.MultihashKeyFromKeccak256(common.BytesToHash(codeHash))
mhKey, err = shared.MultihashKeyFromKeccak256(common.BytesToHash(codeHash))
if err != nil {
return nil, err
}
pkg/eth/backend_utils.go
+1 -2
View File
@@ -30,8 +30,7 @@ import (
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/rpc"
"github.com/vulcanize/ipld-eth-indexer/pkg/ipfs"
"github.com/ethereum/go-ethereum/statediff/indexer/ipfs"
)
// RPCMarshalHeader converts the given header to the RPC output.
pkg/eth/cid_retriever.go
+44 -45
View File
@@ -22,13 +22,12 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/statediff/indexer/models"
"github.com/ethereum/go-ethereum/statediff/indexer/postgres"
"github.com/jmoiron/sqlx"
"github.com/lib/pq"
log "github.com/sirupsen/logrus"
"github.com/vulcanize/ipld-eth-indexer/pkg/eth"
"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
"github.com/vulcanize/ipld-eth-server/pkg/shared"
)
@@ -86,7 +85,7 @@ func (ecr *CIDRetriever) Retrieve(filter SubscriptionSettings, blockNumber int64
}()
// Retrieve cached header CIDs at this block height
var headers []eth.HeaderModel
var headers []models.HeaderModel
headers, err = ecr.RetrieveHeaderCIDs(tx, blockNumber)
if err != nil {
log.Error("header cid retrieval error")
@@ -102,7 +101,7 @@ func (ecr *CIDRetriever) Retrieve(filter SubscriptionSettings, blockNumber int64
empty = false
if filter.HeaderFilter.Uncles {
// Retrieve uncle cids for this header id
var uncleCIDs []eth.UncleModel
var uncleCIDs []models.UncleModel
uncleCIDs, err = ecr.RetrieveUncleCIDsByHeaderID(tx, header.ID)
if err != nil {
log.Error("uncle cid retrieval error")
@@ -166,18 +165,18 @@ func (ecr *CIDRetriever) Retrieve(filter SubscriptionSettings, blockNumber int64
}
// RetrieveHeaderCIDs retrieves and returns all of the header cids at the provided blockheight
func (ecr *CIDRetriever) RetrieveHeaderCIDs(tx *sqlx.Tx, blockNumber int64) ([]eth.HeaderModel, error) {
func (ecr *CIDRetriever) RetrieveHeaderCIDs(tx *sqlx.Tx, blockNumber int64) ([]models.HeaderModel, error) {
log.Debug("retrieving header cids for block ", blockNumber)
headers := make([]eth.HeaderModel, 0)
headers := make([]models.HeaderModel, 0)
pgStr := `SELECT * FROM eth.header_cids
WHERE block_number = $1`
return headers, tx.Select(&headers, pgStr, blockNumber)
}
// RetrieveUncleCIDsByHeaderID retrieves and returns all of the uncle cids for the provided header
func (ecr *CIDRetriever) RetrieveUncleCIDsByHeaderID(tx *sqlx.Tx, headerID int64) ([]eth.UncleModel, error) {
func (ecr *CIDRetriever) RetrieveUncleCIDsByHeaderID(tx *sqlx.Tx, headerID int64) ([]models.UncleModel, error) {
log.Debug("retrieving uncle cids for block id ", headerID)
headers := make([]eth.UncleModel, 0)
headers := make([]models.UncleModel, 0)
pgStr := `SELECT * FROM eth.uncle_cids
WHERE header_id = $1`
return headers, tx.Select(&headers, pgStr, headerID)
@@ -185,10 +184,10 @@ func (ecr *CIDRetriever) RetrieveUncleCIDsByHeaderID(tx *sqlx.Tx, headerID int64
// RetrieveTxCIDs retrieves and returns all of the trx cids at the provided blockheight that conform to the provided filter parameters
// also returns the ids for the returned transaction cids
func (ecr *CIDRetriever) RetrieveTxCIDs(tx *sqlx.Tx, txFilter TxFilter, headerID int64) ([]eth.TxModel, error) {
func (ecr *CIDRetriever) RetrieveTxCIDs(tx *sqlx.Tx, txFilter TxFilter, headerID int64) ([]models.TxModel, error) {
log.Debug("retrieving transaction cids for header id ", headerID)
args := make([]interface{}, 0, 3)
results := make([]eth.TxModel, 0)
results := make([]models.TxModel, 0)
id := 1
pgStr := fmt.Sprintf(`SELECT transaction_cids.id, transaction_cids.header_id,
transaction_cids.tx_hash, transaction_cids.cid, transaction_cids.mh_key,
@@ -212,7 +211,7 @@ func (ecr *CIDRetriever) RetrieveTxCIDs(tx *sqlx.Tx, txFilter TxFilter, headerID
// RetrieveRctCIDsByHeaderID retrieves and returns all of the rct cids at the provided header ID that conform to the provided
// filter parameters and correspond to the provided tx ids
func (ecr *CIDRetriever) RetrieveRctCIDsByHeaderID(tx *sqlx.Tx, rctFilter ReceiptFilter, headerID int64, trxIds []int64) ([]eth.ReceiptModel, error) {
func (ecr *CIDRetriever) RetrieveRctCIDsByHeaderID(tx *sqlx.Tx, rctFilter ReceiptFilter, headerID int64, trxIds []int64) ([]models.ReceiptModel, error) {
log.Debug("retrieving receipt cids for header id ", headerID)
args := make([]interface{}, 0, 4)
pgStr := `SELECT receipt_cids.id, receipt_cids.tx_id, receipt_cids.cid, receipt_cids.mh_key,
@@ -286,13 +285,13 @@ func (ecr *CIDRetriever) RetrieveRctCIDsByHeaderID(tx *sqlx.Tx, rctFilter Receip
}
}
pgStr += ` ORDER BY transaction_cids.index`
receiptCids := make([]eth.ReceiptModel, 0)
receiptCids := make([]models.ReceiptModel, 0)
return receiptCids, tx.Select(&receiptCids, pgStr, args...)
}
// RetrieveRctCIDs retrieves and returns all of the rct cids at the provided blockheight or block hash that conform to the provided
// filter parameters and correspond to the provided tx ids
func (ecr *CIDRetriever) RetrieveRctCIDs(tx *sqlx.Tx, rctFilter ReceiptFilter, blockNumber int64, blockHash *common.Hash, trxIds []int64) ([]eth.ReceiptModel, error) {
func (ecr *CIDRetriever) RetrieveRctCIDs(tx *sqlx.Tx, rctFilter ReceiptFilter, blockNumber int64, blockHash *common.Hash, trxIds []int64) ([]models.ReceiptModel, error) {
log.Debug("retrieving receipt cids for block ", blockNumber)
args := make([]interface{}, 0, 5)
pgStr := `SELECT receipt_cids.id, receipt_cids.tx_id, receipt_cids.cid, receipt_cids.mh_key,
@@ -316,7 +315,7 @@ func (ecr *CIDRetriever) RetrieveRctCIDs(tx *sqlx.Tx, rctFilter ReceiptFilter, b
// TODO: Add the below filters when we have log index in DB.
if true {
pgStr += ` ORDER BY transaction_cids.index`
receiptCids := make([]eth.ReceiptModel, 0)
receiptCids := make([]models.ReceiptModel, 0)
return receiptCids, tx.Select(&receiptCids, pgStr, args...)
}
@@ -383,7 +382,7 @@ func (ecr *CIDRetriever) RetrieveRctCIDs(tx *sqlx.Tx, rctFilter ReceiptFilter, b
}
pgStr += ` ORDER BY transaction_cids.index`
receiptCids := make([]eth.ReceiptModel, 0)
receiptCids := make([]models.ReceiptModel, 0)
return receiptCids, tx.Select(&receiptCids, pgStr, args...)
}
@@ -397,7 +396,7 @@ func hasTopics(topics [][]string) bool {
}
// RetrieveStateCIDs retrieves and returns all of the state node cids at the provided header ID that conform to the provided filter parameters
func (ecr *CIDRetriever) RetrieveStateCIDs(tx *sqlx.Tx, stateFilter StateFilter, headerID int64) ([]eth.StateNodeModel, error) {
func (ecr *CIDRetriever) RetrieveStateCIDs(tx *sqlx.Tx, stateFilter StateFilter, headerID int64) ([]models.StateNodeModel, error) {
log.Debug("retrieving state cids for header id ", headerID)
args := make([]interface{}, 0, 2)
pgStr := `SELECT state_cids.id, state_cids.header_id,
@@ -417,12 +416,12 @@ func (ecr *CIDRetriever) RetrieveStateCIDs(tx *sqlx.Tx, stateFilter StateFilter,
if !stateFilter.IntermediateNodes {
pgStr += ` AND state_cids.node_type = 2`
}
stateNodeCIDs := make([]eth.StateNodeModel, 0)
stateNodeCIDs := make([]models.StateNodeModel, 0)
return stateNodeCIDs, tx.Select(&stateNodeCIDs, pgStr, args...)
}
// RetrieveStorageCIDs retrieves and returns all of the storage node cids at the provided header id that conform to the provided filter parameters
func (ecr *CIDRetriever) RetrieveStorageCIDs(tx *sqlx.Tx, storageFilter StorageFilter, headerID int64) ([]eth.StorageNodeWithStateKeyModel, error) {
func (ecr *CIDRetriever) RetrieveStorageCIDs(tx *sqlx.Tx, storageFilter StorageFilter, headerID int64) ([]models.StorageNodeWithStateKeyModel, error) {
log.Debug("retrieving storage cids for header id ", headerID)
args := make([]interface{}, 0, 3)
pgStr := `SELECT storage_cids.id, storage_cids.state_id, storage_cids.storage_leaf_key, storage_cids.node_type,
@@ -450,18 +449,18 @@ func (ecr *CIDRetriever) RetrieveStorageCIDs(tx *sqlx.Tx, storageFilter StorageF
if !storageFilter.IntermediateNodes {
pgStr += ` AND storage_cids.node_type = 2`
}
storageNodeCIDs := make([]eth.StorageNodeWithStateKeyModel, 0)
storageNodeCIDs := make([]models.StorageNodeWithStateKeyModel, 0)
return storageNodeCIDs, tx.Select(&storageNodeCIDs, pgStr, args...)
}
// RetrieveBlockByHash returns all of the CIDs needed to compose an entire block, for a given block hash
func (ecr *CIDRetriever) RetrieveBlockByHash(blockHash common.Hash) (eth.HeaderModel, []eth.UncleModel, []eth.TxModel, []eth.ReceiptModel, error) {
func (ecr *CIDRetriever) RetrieveBlockByHash(blockHash common.Hash) (models.HeaderModel, []models.UncleModel, []models.TxModel, []models.ReceiptModel, error) {
log.Debug("retrieving block cids for block hash ", blockHash.String())
// Begin new db tx
tx, err := ecr.db.Beginx()
if err != nil {
return eth.HeaderModel{}, nil, nil, nil, err
return models.HeaderModel{}, nil, nil, nil, err
}
defer func() {
if p := recover(); p != nil {
@@ -474,29 +473,29 @@ func (ecr *CIDRetriever) RetrieveBlockByHash(blockHash common.Hash) (eth.HeaderM
}
}()
var headerCID eth.HeaderModel
var headerCID models.HeaderModel
headerCID, err = ecr.RetrieveHeaderCIDByHash(tx, blockHash)
if err != nil {
log.Error("header cid retrieval error")
return eth.HeaderModel{}, nil, nil, nil, err
return models.HeaderModel{}, nil, nil, nil, err
}
var uncleCIDs []eth.UncleModel
var uncleCIDs []models.UncleModel
uncleCIDs, err = ecr.RetrieveUncleCIDsByHeaderID(tx, headerCID.ID)
if err != nil {
log.Error("uncle cid retrieval error")
return eth.HeaderModel{}, nil, nil, nil, err
return models.HeaderModel{}, nil, nil, nil, err
}
var txCIDs []eth.TxModel
var txCIDs []models.TxModel
txCIDs, err = ecr.RetrieveTxCIDsByHeaderID(tx, headerCID.ID)
if err != nil {
log.Error("tx cid retrieval error")
return eth.HeaderModel{}, nil, nil, nil, err
return models.HeaderModel{}, nil, nil, nil, err
}
txIDs := make([]int64, len(txCIDs))
for i, txCID := range txCIDs {
txIDs[i] = txCID.ID
}
var rctCIDs []eth.ReceiptModel
var rctCIDs []models.ReceiptModel
rctCIDs, err = ecr.RetrieveReceiptCIDsByTxIDs(tx, txIDs)
if err != nil {
log.Error("rct cid retrieval error")
@@ -505,13 +504,13 @@ func (ecr *CIDRetriever) RetrieveBlockByHash(blockHash common.Hash) (eth.HeaderM
}
// RetrieveBlockByNumber returns all of the CIDs needed to compose an entire block, for a given block number
func (ecr *CIDRetriever) RetrieveBlockByNumber(blockNumber int64) (eth.HeaderModel, []eth.UncleModel, []eth.TxModel, []eth.ReceiptModel, error) {
func (ecr *CIDRetriever) RetrieveBlockByNumber(blockNumber int64) (models.HeaderModel, []models.UncleModel, []models.TxModel, []models.ReceiptModel, error) {
log.Debug("retrieving block cids for block number ", blockNumber)
// Begin new db tx
tx, err := ecr.db.Beginx()
if err != nil {
return eth.HeaderModel{}, nil, nil, nil, err
return models.HeaderModel{}, nil, nil, nil, err
}
defer func() {
if p := recover(); p != nil {
@@ -524,32 +523,32 @@ func (ecr *CIDRetriever) RetrieveBlockByNumber(blockNumber int64) (eth.HeaderMod
}
}()
var headerCID []eth.HeaderModel
var headerCID []models.HeaderModel
headerCID, err = ecr.RetrieveHeaderCIDs(tx, blockNumber)
if err != nil {
log.Error("header cid retrieval error")
return eth.HeaderModel{}, nil, nil, nil, err
return models.HeaderModel{}, nil, nil, nil, err
}
if len(headerCID) < 1 {
return eth.HeaderModel{}, nil, nil, nil, fmt.Errorf("header cid retrieval error, no header CIDs found at block %d", blockNumber)
return models.HeaderModel{}, nil, nil, nil, fmt.Errorf("header cid retrieval error, no header CIDs found at block %d", blockNumber)
}
var uncleCIDs []eth.UncleModel
var uncleCIDs []models.UncleModel
uncleCIDs, err = ecr.RetrieveUncleCIDsByHeaderID(tx, headerCID[0].ID)
if err != nil {
log.Error("uncle cid retrieval error")
return eth.HeaderModel{}, nil, nil, nil, err
return models.HeaderModel{}, nil, nil, nil, err
}
var txCIDs []eth.TxModel
var txCIDs []models.TxModel
txCIDs, err = ecr.RetrieveTxCIDsByHeaderID(tx, headerCID[0].ID)
if err != nil {
log.Error("tx cid retrieval error")
return eth.HeaderModel{}, nil, nil, nil, err
return models.HeaderModel{}, nil, nil, nil, err
}
txIDs := make([]int64, len(txCIDs))
for i, txCID := range txCIDs {
txIDs[i] = txCID.ID
}
var rctCIDs []eth.ReceiptModel
var rctCIDs []models.ReceiptModel
rctCIDs, err = ecr.RetrieveReceiptCIDsByTxIDs(tx, txIDs)
if err != nil {
log.Error("rct cid retrieval error")
@@ -558,26 +557,26 @@ func (ecr *CIDRetriever) RetrieveBlockByNumber(blockNumber int64) (eth.HeaderMod
}
// RetrieveHeaderCIDByHash returns the header for the given block hash
func (ecr *CIDRetriever) RetrieveHeaderCIDByHash(tx *sqlx.Tx, blockHash common.Hash) (eth.HeaderModel, error) {
func (ecr *CIDRetriever) RetrieveHeaderCIDByHash(tx *sqlx.Tx, blockHash common.Hash) (models.HeaderModel, error) {
log.Debug("retrieving header cids for block hash ", blockHash.String())
pgStr := `SELECT * FROM eth.header_cids
WHERE block_hash = $1`
var headerCID eth.HeaderModel
var headerCID models.HeaderModel
return headerCID, tx.Get(&headerCID, pgStr, blockHash.String())
}
// RetrieveTxCIDsByHeaderID retrieves all tx CIDs for the given header id
func (ecr *CIDRetriever) RetrieveTxCIDsByHeaderID(tx *sqlx.Tx, headerID int64) ([]eth.TxModel, error) {
func (ecr *CIDRetriever) RetrieveTxCIDsByHeaderID(tx *sqlx.Tx, headerID int64) ([]models.TxModel, error) {
log.Debug("retrieving tx cids for block id ", headerID)
pgStr := `SELECT * FROM eth.transaction_cids
WHERE header_id = $1
ORDER BY index`
var txCIDs []eth.TxModel
var txCIDs []models.TxModel
return txCIDs, tx.Select(&txCIDs, pgStr, headerID)
}
// RetrieveReceiptCIDsByTxIDs retrieves receipt CIDs by their associated tx IDs
func (ecr *CIDRetriever) RetrieveReceiptCIDsByTxIDs(tx *sqlx.Tx, txIDs []int64) ([]eth.ReceiptModel, error) {
func (ecr *CIDRetriever) RetrieveReceiptCIDsByTxIDs(tx *sqlx.Tx, txIDs []int64) ([]models.ReceiptModel, error) {
log.Debugf("retrieving receipt cids for tx ids %v", txIDs)
pgStr := `SELECT receipt_cids.id, receipt_cids.tx_id, receipt_cids.cid, receipt_cids.mh_key,
receipt_cids.contract, receipt_cids.contract_hash, receipt_cids.topic0s, receipt_cids.topic1s,
@@ -586,6 +585,6 @@ func (ecr *CIDRetriever) RetrieveReceiptCIDsByTxIDs(tx *sqlx.Tx, txIDs []int64)
WHERE tx_id = ANY($1::INTEGER[])
AND receipt_cids.tx_id = transaction_cids.id
ORDER BY transaction_cids.index`
var rctCIDs []eth.ReceiptModel
var rctCIDs []models.ReceiptModel
return rctCIDs, tx.Select(&rctCIDs, pgStr, pq.Array(txIDs))
}
pkg/eth/cid_retriever_test.go
+60 -26
View File
@@ -19,17 +19,17 @@ package eth_test
import (
"math/big"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/statediff/indexer"
"github.com/ethereum/go-ethereum/statediff/indexer/models"
"github.com/ethereum/go-ethereum/statediff/indexer/postgres"
"github.com/ethereum/go-ethereum/trie"
"github.com/vulcanize/ipld-eth-indexer/pkg/shared"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
eth2 "github.com/vulcanize/ipld-eth-indexer/pkg/eth"
"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
"github.com/vulcanize/ipld-eth-server/pkg/eth"
"github.com/vulcanize/ipld-eth-server/pkg/eth/test_helpers"
)
@@ -211,15 +211,15 @@ var (
var _ = Describe("Retriever", func() {
var (
db *postgres.DB
repo *eth2.IPLDPublisher
retriever *eth.CIDRetriever
db *postgres.DB
diffIndexer *indexer.StateDiffIndexer
retriever *eth.CIDRetriever
)
BeforeEach(func() {
var err error
db, err = shared.SetupDB()
db, err = SetupDB()
Expect(err).ToNot(HaveOccurred())
repo = eth2.NewIPLDPublisher(db)
diffIndexer = indexer.NewStateDiffIndexer(params.TestChainConfig, db)
retriever = eth.NewCIDRetriever(db)
})
AfterEach(func() {
@@ -228,7 +228,14 @@ var _ = Describe("Retriever", func() {
Describe("Retrieve", func() {
BeforeEach(func() {
err := repo.Publish(test_helpers.MockConvertedPayload)
tx, err := diffIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
Expect(err).ToNot(HaveOccurred())
for _, node := range test_helpers.MockStateNodes {
err = diffIndexer.PushStateNode(tx, node)
Expect(err).ToNot(HaveOccurred())
}
err = tx.Close(err)
Expect(err).ToNot(HaveOccurred())
})
It("Retrieves all CIDs for the given blocknumber when provided an open filter", func() {
@@ -277,7 +284,7 @@ var _ = Describe("Retriever", func() {
Expect(empty).ToNot(BeTrue())
Expect(len(cids1)).To(Equal(1))
Expect(cids1[0].BlockNumber).To(Equal(test_helpers.MockCIDWrapper.BlockNumber))
Expect(cids1[0].Header).To(Equal(eth2.HeaderModel{}))
Expect(cids1[0].Header).To(Equal(models.HeaderModel{}))
Expect(len(cids1[0].Transactions)).To(Equal(0))
Expect(len(cids1[0].StateNodes)).To(Equal(0))
Expect(len(cids1[0].StorageNodes)).To(Equal(0))
@@ -292,7 +299,7 @@ var _ = Describe("Retriever", func() {
Expect(empty).ToNot(BeTrue())
Expect(len(cids2)).To(Equal(1))
Expect(cids2[0].BlockNumber).To(Equal(test_helpers.MockCIDWrapper.BlockNumber))
Expect(cids2[0].Header).To(Equal(eth2.HeaderModel{}))
Expect(cids2[0].Header).To(Equal(models.HeaderModel{}))
Expect(len(cids2[0].Transactions)).To(Equal(0))
Expect(len(cids2[0].StateNodes)).To(Equal(0))
Expect(len(cids2[0].StorageNodes)).To(Equal(0))
@@ -307,7 +314,7 @@ var _ = Describe("Retriever", func() {
Expect(empty).ToNot(BeTrue())
Expect(len(cids3)).To(Equal(1))
Expect(cids3[0].BlockNumber).To(Equal(test_helpers.MockCIDWrapper.BlockNumber))
Expect(cids3[0].Header).To(Equal(eth2.HeaderModel{}))
Expect(cids3[0].Header).To(Equal(models.HeaderModel{}))
Expect(len(cids3[0].Transactions)).To(Equal(0))
Expect(len(cids3[0].StateNodes)).To(Equal(0))
Expect(len(cids3[0].StorageNodes)).To(Equal(0))
@@ -322,7 +329,7 @@ var _ = Describe("Retriever", func() {
Expect(empty).ToNot(BeTrue())
Expect(len(cids4)).To(Equal(1))
Expect(cids4[0].BlockNumber).To(Equal(test_helpers.MockCIDWrapper.BlockNumber))
Expect(cids4[0].Header).To(Equal(eth2.HeaderModel{}))
Expect(cids4[0].Header).To(Equal(models.HeaderModel{}))
Expect(len(cids4[0].Transactions)).To(Equal(0))
Expect(len(cids4[0].StateNodes)).To(Equal(0))
Expect(len(cids4[0].StorageNodes)).To(Equal(0))
@@ -337,7 +344,7 @@ var _ = Describe("Retriever", func() {
Expect(empty).ToNot(BeTrue())
Expect(len(cids5)).To(Equal(1))
Expect(cids5[0].BlockNumber).To(Equal(test_helpers.MockCIDWrapper.BlockNumber))
Expect(cids5[0].Header).To(Equal(eth2.HeaderModel{}))
Expect(cids5[0].Header).To(Equal(models.HeaderModel{}))
Expect(len(cids5[0].Transactions)).To(Equal(3))
Expect(eth.TxModelsContainsCID(cids5[0].Transactions, test_helpers.Trx1CID.String())).To(BeTrue())
Expect(eth.TxModelsContainsCID(cids5[0].Transactions, test_helpers.Trx2CID.String())).To(BeTrue())
@@ -354,7 +361,7 @@ var _ = Describe("Retriever", func() {
Expect(empty).ToNot(BeTrue())
Expect(len(cids6)).To(Equal(1))
Expect(cids6[0].BlockNumber).To(Equal(test_helpers.MockCIDWrapper.BlockNumber))
Expect(cids6[0].Header).To(Equal(eth2.HeaderModel{}))
Expect(cids6[0].Header).To(Equal(models.HeaderModel{}))
Expect(len(cids6[0].Transactions)).To(Equal(1))
expectedTxCID := test_helpers.MockCIDWrapper.Transactions[1]
expectedTxCID.ID = cids6[0].Transactions[0].ID
@@ -373,12 +380,12 @@ var _ = Describe("Retriever", func() {
Expect(empty).ToNot(BeTrue())
Expect(len(cids7)).To(Equal(1))
Expect(cids7[0].BlockNumber).To(Equal(test_helpers.MockCIDWrapper.BlockNumber))
Expect(cids7[0].Header).To(Equal(eth2.HeaderModel{}))
Expect(cids7[0].Header).To(Equal(models.HeaderModel{}))
Expect(len(cids7[0].Transactions)).To(Equal(0))
Expect(len(cids7[0].Receipts)).To(Equal(0))
Expect(len(cids7[0].StorageNodes)).To(Equal(0))
Expect(len(cids7[0].StateNodes)).To(Equal(1))
Expect(cids7[0].StateNodes[0]).To(Equal(eth2.StateNodeModel{
Expect(cids7[0].StateNodes[0]).To(Equal(models.StateNodeModel{
ID: cids7[0].StateNodes[0].ID,
HeaderID: cids7[0].StateNodes[0].HeaderID,
NodeType: 2,
@@ -400,8 +407,12 @@ var _ = Describe("Retriever", func() {
Expect(err).To(HaveOccurred())
})
It("Gets the number of the first block that has data in the database", func() {
err := repo.Publish(test_helpers.MockConvertedPayload)
tx, err := diffIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
Expect(err).ToNot(HaveOccurred())
err = tx.Close(err)
Expect(err).ToNot(HaveOccurred())
num, err := retriever.RetrieveFirstBlockNumber()
Expect(err).ToNot(HaveOccurred())
Expect(num).To(Equal(int64(1)))
@@ -410,8 +421,12 @@ var _ = Describe("Retriever", func() {
It("Gets the number of the first block that has data in the database", func() {
payload := test_helpers.MockConvertedPayload
payload.Block = newMockBlock(1010101)
err := repo.Publish(payload)
tx, err := diffIndexer.PushBlock(payload.Block, payload.Receipts, payload.Block.Difficulty())
Expect(err).ToNot(HaveOccurred())
err = tx.Close(err)
Expect(err).ToNot(HaveOccurred())
num, err := retriever.RetrieveFirstBlockNumber()
Expect(err).ToNot(HaveOccurred())
Expect(num).To(Equal(int64(1010101)))
@@ -422,10 +437,16 @@ var _ = Describe("Retriever", func() {
payload1.Block = newMockBlock(1010101)
payload2 := payload1
payload2.Block = newMockBlock(5)
err := repo.Publish(payload1)
tx, err := diffIndexer.PushBlock(payload1.Block, payload1.Receipts, payload1.Block.Difficulty())
Expect(err).ToNot(HaveOccurred())
err = repo.Publish(payload2)
err = tx.Close(err)
Expect(err).ToNot(HaveOccurred())
tx, err = diffIndexer.PushBlock(payload2.Block, payload2.Receipts, payload2.Block.Difficulty())
Expect(err).ToNot(HaveOccurred())
err = tx.Close(err)
Expect(err).ToNot(HaveOccurred())
num, err := retriever.RetrieveFirstBlockNumber()
Expect(err).ToNot(HaveOccurred())
Expect(num).To(Equal(int64(5)))
@@ -438,8 +459,11 @@ var _ = Describe("Retriever", func() {
Expect(err).To(HaveOccurred())
})
It("Gets the number of the latest block that has data in the database", func() {
err := repo.Publish(test_helpers.MockConvertedPayload)
tx, err := diffIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
Expect(err).ToNot(HaveOccurred())
err = tx.Close(err)
Expect(err).ToNot(HaveOccurred())
num, err := retriever.RetrieveLastBlockNumber()
Expect(err).ToNot(HaveOccurred())
Expect(num).To(Equal(int64(1)))
@@ -448,8 +472,12 @@ var _ = Describe("Retriever", func() {
It("Gets the number of the latest block that has data in the database", func() {
payload := test_helpers.MockConvertedPayload
payload.Block = newMockBlock(1010101)
err := repo.Publish(payload)
tx, err := diffIndexer.PushBlock(payload.Block, payload.Receipts, payload.Block.Difficulty())
Expect(err).ToNot(HaveOccurred())
err = tx.Close(err)
Expect(err).ToNot(HaveOccurred())
num, err := retriever.RetrieveLastBlockNumber()
Expect(err).ToNot(HaveOccurred())
Expect(num).To(Equal(int64(1010101)))
@@ -460,10 +488,16 @@ var _ = Describe("Retriever", func() {
payload1.Block = newMockBlock(1010101)
payload2 := payload1
payload2.Block = newMockBlock(5)
err := repo.Publish(payload1)
tx, err := diffIndexer.PushBlock(payload1.Block, payload1.Receipts, payload1.Block.Difficulty())
Expect(err).ToNot(HaveOccurred())
err = repo.Publish(payload2)
err = tx.Close(err)
Expect(err).ToNot(HaveOccurred())
tx, err = diffIndexer.PushBlock(payload2.Block, payload2.Receipts, payload2.Block.Difficulty())
Expect(err).ToNot(HaveOccurred())
err = tx.Close(err)
Expect(err).ToNot(HaveOccurred())
num, err := retriever.RetrieveLastBlockNumber()
Expect(err).ToNot(HaveOccurred())
Expect(num).To(Equal(int64(1010101)))
pkg/eth/eth_state_test.go
+29 -22
View File
@@ -29,15 +29,14 @@ import (
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/core/vm"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/rpc"
"github.com/ethereum/go-ethereum/statediff"
"github.com/ethereum/go-ethereum/statediff/indexer"
"github.com/ethereum/go-ethereum/statediff/indexer/postgres"
sdtypes "github.com/ethereum/go-ethereum/statediff/types"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
eth2 "github.com/vulcanize/ipld-eth-indexer/pkg/eth"
"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
"github.com/vulcanize/ipld-eth-indexer/pkg/shared"
"github.com/vulcanize/ipld-eth-server/pkg/eth"
"github.com/vulcanize/ipld-eth-server/pkg/eth/test_helpers"
)
@@ -74,9 +73,9 @@ var _ = Describe("eth state reading tests", func() {
It("test init", func() {
// db and type initializations
var err error
db, err = shared.SetupDB()
db, err = SetupDB()
Expect(err).ToNot(HaveOccurred())
transformer := eth2.NewStateDiffTransformer(chainConfig, db)
transformer := indexer.NewStateDiffIndexer(chainConfig, db)
backend, err = eth.NewEthBackend(db, &eth.Config{
ChainConfig: chainConfig,
VmConfig: vm.Config{},
@@ -135,31 +134,39 @@ var _ = Describe("eth state reading tests", func() {
}
diff, err := builder.BuildStateDiffObject(args, params)
Expect(err).ToNot(HaveOccurred())
diffRlp, err := rlp.EncodeToBytes(diff)
tx, err := transformer.PushBlock(block, rcts, mockTD)
Expect(err).ToNot(HaveOccurred())
blockRlp, err := rlp.EncodeToBytes(block)
Expect(err).ToNot(HaveOccurred())
receiptsRlp, err := rlp.EncodeToBytes(rcts)
Expect(err).ToNot(HaveOccurred())
payload := statediff.Payload{
StateObjectRlp: diffRlp,
BlockRlp: blockRlp,
ReceiptsRlp: receiptsRlp,
TotalDifficulty: mockTD,
for _, node := range diff.Nodes {
err = transformer.PushStateNode(tx, node)
Expect(err).ToNot(HaveOccurred())
}
_, err = transformer.Transform(0, payload)
err = tx.Close(err)
Expect(err).ToNot(HaveOccurred())
}
// Insert some non-canonical data into the database so that we test our ability to discern canonicity
indexAndPublisher := eth2.NewIPLDPublisher(db)
api = eth.NewPublicEthAPI(backend, nil, false)
err = indexAndPublisher.Publish(test_helpers.MockConvertedPayload)
indexAndPublisher := indexer.NewStateDiffIndexer(chainConfig, db)
tx, err := indexAndPublisher.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
Expect(err).ToNot(HaveOccurred())
err = tx.Close(err)
Expect(err).ToNot(HaveOccurred())
// The non-canonical header has a child
err = indexAndPublisher.Publish(test_helpers.MockConvertedPayloadForChild)
tx, err = indexAndPublisher.PushBlock(test_helpers.MockChild, test_helpers.MockReceipts, test_helpers.MockChild.Difficulty())
Expect(err).ToNot(HaveOccurred())
err = publishCode(db, test_helpers.ContractCodeHash, test_helpers.ContractCode)
hash := sdtypes.CodeAndCodeHash{
Hash: test_helpers.CodeHash,
Code: test_helpers.ContractCode,
}
err = indexAndPublisher.PushCodeAndCodeHash(tx, hash)
Expect(err).ToNot(HaveOccurred())
err = tx.Close(err)
Expect(err).ToNot(HaveOccurred())
})
defer It("test teardown", func() {
pkg/eth/filterer.go
+20 -21
View File
@@ -19,6 +19,7 @@ package eth
import (
"bytes"
"github.com/ethereum/go-ethereum/statediff/indexer/ipfs/ipld"
sdtypes "github.com/ethereum/go-ethereum/statediff/types"
"github.com/ethereum/go-ethereum/common"
@@ -27,14 +28,12 @@ import (
"github.com/ethereum/go-ethereum/rlp"
"github.com/multiformats/go-multihash"
"github.com/vulcanize/ipld-eth-indexer/pkg/eth"
"github.com/vulcanize/ipld-eth-indexer/pkg/ipfs"
"github.com/vulcanize/ipld-eth-indexer/pkg/ipfs/ipld"
"github.com/ethereum/go-ethereum/statediff/indexer/ipfs"
)
// Filterer interface for substituing mocks in tests
type Filterer interface {
Filter(filter SubscriptionSettings, payload eth.ConvertedPayload) (*IPLDs, error)
Filter(filter SubscriptionSettings, payload ConvertedPayload) (*IPLDs, error)
}
// ResponseFilterer satisfies the ResponseFilterer interface for ethereum
@@ -46,7 +45,7 @@ func NewResponseFilterer() *ResponseFilterer {
}
// Filter is used to filter through eth data to extract and package requested data into a Payload
func (s *ResponseFilterer) Filter(filter SubscriptionSettings, payload eth.ConvertedPayload) (*IPLDs, error) {
func (s *ResponseFilterer) Filter(filter SubscriptionSettings, payload ConvertedPayload) (*IPLDs, error) {
if checkRange(filter.Start.Int64(), filter.End.Int64(), payload.Block.Number().Int64()) {
response := new(IPLDs)
response.TotalDifficulty = payload.TotalDifficulty
@@ -73,7 +72,7 @@ func (s *ResponseFilterer) Filter(filter SubscriptionSettings, payload eth.Conve
return nil, nil
}
func (s *ResponseFilterer) filterHeaders(headerFilter HeaderFilter, response *IPLDs, payload eth.ConvertedPayload) error {
func (s *ResponseFilterer) filterHeaders(headerFilter HeaderFilter, response *IPLDs, payload ConvertedPayload) error {
if !headerFilter.Off {
headerRLP, err := rlp.EncodeToBytes(payload.Block.Header())
if err != nil {
@@ -115,7 +114,7 @@ func checkRange(start, end, actual int64) bool {
return false
}
func (s *ResponseFilterer) filterTransactions(trxFilter TxFilter, response *IPLDs, payload eth.ConvertedPayload) ([]common.Hash, error) {
func (s *ResponseFilterer) filterTransactions(trxFilter TxFilter, response *IPLDs, payload ConvertedPayload) ([]common.Hash, error) {
var trxHashes []common.Hash
if !trxFilter.Off {
trxLen := len(payload.Block.Body().Transactions)
@@ -163,7 +162,7 @@ func checkTransactionAddrs(wantedSrc, wantedDst []string, actualSrc, actualDst s
return false
}
func (s *ResponseFilterer) filerReceipts(receiptFilter ReceiptFilter, response *IPLDs, payload eth.ConvertedPayload, trxHashes []common.Hash) error {
func (s *ResponseFilterer) filerReceipts(receiptFilter ReceiptFilter, response *IPLDs, payload ConvertedPayload, trxHashes []common.Hash) error {
if !receiptFilter.Off {
response.Receipts = make([]ipfs.BlockModel, 0, len(payload.Receipts))
for i, receipt := range payload.Receipts {
@@ -253,7 +252,7 @@ func slicesShareString(slice1, slice2 []string) int {
}
// filterStateAndStorage filters state and storage nodes into the response according to the provided filters
func (s *ResponseFilterer) filterStateAndStorage(stateFilter StateFilter, storageFilter StorageFilter, response *IPLDs, payload eth.ConvertedPayload) error {
func (s *ResponseFilterer) filterStateAndStorage(stateFilter StateFilter, storageFilter StorageFilter, response *IPLDs, payload ConvertedPayload) error {
response.StateNodes = make([]StateNode, 0, len(payload.StateNodes))
response.StorageNodes = make([]StorageNode, 0)
stateAddressFilters := make([]common.Hash, len(stateFilter.Addresses))
@@ -270,37 +269,37 @@ func (s *ResponseFilterer) filterStateAndStorage(stateFilter StateFilter, storag
}
for _, stateNode := range payload.StateNodes {
if !stateFilter.Off && checkNodeKeys(stateAddressFilters, stateNode.LeafKey) {
if stateNode.Type == sdtypes.Leaf || stateFilter.IntermediateNodes {
cid, err := ipld.RawdataToCid(ipld.MEthStateTrie, stateNode.Value, multihash.KECCAK_256)
if stateNode.NodeType == sdtypes.Leaf || stateFilter.IntermediateNodes {
cid, err := ipld.RawdataToCid(ipld.MEthStateTrie, stateNode.NodeValue, multihash.KECCAK_256)
if err != nil {
return err
}
response.StateNodes = append(response.StateNodes, StateNode{
StateLeafKey: stateNode.LeafKey,
StateLeafKey: common.BytesToHash(stateNode.LeafKey),
Path: stateNode.Path,
IPLD: ipfs.BlockModel{
Data: stateNode.Value,
Data: stateNode.NodeValue,
CID: cid.String(),
},
Type: stateNode.Type,
Type: stateNode.NodeType,
})
}
}
if !storageFilter.Off && checkNodeKeys(storageAddressFilters, stateNode.LeafKey) {
for _, storageNode := range payload.StorageNodes[common.Bytes2Hex(stateNode.Path)] {
if checkNodeKeys(storageKeyFilters, storageNode.LeafKey) {
cid, err := ipld.RawdataToCid(ipld.MEthStorageTrie, storageNode.Value, multihash.KECCAK_256)
cid, err := ipld.RawdataToCid(ipld.MEthStorageTrie, storageNode.NodeValue, multihash.KECCAK_256)
if err != nil {
return err
}
response.StorageNodes = append(response.StorageNodes, StorageNode{
StateLeafKey: stateNode.LeafKey,
StorageLeafKey: storageNode.LeafKey,
StateLeafKey: common.BytesToHash(stateNode.LeafKey),
StorageLeafKey: common.BytesToHash(storageNode.LeafKey),
IPLD: ipfs.BlockModel{
Data: storageNode.Value,
Data: storageNode.NodeValue,
CID: cid.String(),
},
Type: storageNode.Type,
Type: storageNode.NodeType,
Path: storageNode.Path,
})
}
@@ -310,13 +309,13 @@ func (s *ResponseFilterer) filterStateAndStorage(stateFilter StateFilter, storag
return nil
}
func checkNodeKeys(wantedKeys []common.Hash, actualKey common.Hash) bool {
func checkNodeKeys(wantedKeys []common.Hash, actualKey []byte) bool {
// If we aren't filtering for any specific keys, all nodes are a go
if len(wantedKeys) == 0 {
return true
}
for _, key := range wantedKeys {
if bytes.Equal(key.Bytes(), actualKey.Bytes()) {
if bytes.Equal(key.Bytes(), actualKey) {
return true
}
}
pkg/eth/filterer_test.go
+1 -2
View File
@@ -19,13 +19,12 @@ package eth_test
import (
"bytes"
"github.com/ethereum/go-ethereum/statediff/indexer/ipfs"
sdtypes "github.com/ethereum/go-ethereum/statediff/types"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/ipld-eth-indexer/pkg/ipfs"
"github.com/vulcanize/ipld-eth-server/pkg/eth"
"github.com/vulcanize/ipld-eth-server/pkg/eth/test_helpers"
"github.com/vulcanize/ipld-eth-server/pkg/shared"
pkg/eth/ipld_fetcher.go
+9 -10
View File
@@ -22,13 +22,12 @@ import (
"math/big"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/statediff/indexer/models"
"github.com/ethereum/go-ethereum/statediff/indexer/postgres"
"github.com/jmoiron/sqlx"
log "github.com/sirupsen/logrus"
"github.com/vulcanize/ipld-eth-indexer/pkg/eth"
"github.com/vulcanize/ipld-eth-indexer/pkg/ipfs"
"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
"github.com/ethereum/go-ethereum/statediff/indexer/ipfs"
"github.com/vulcanize/ipld-eth-server/pkg/shared"
)
@@ -104,7 +103,7 @@ func (f *IPLDFetcher) Fetch(cids CIDWrapper) (*IPLDs, error) {
}
// FetchHeaders fetches headers
func (f *IPLDFetcher) FetchHeader(tx *sqlx.Tx, c eth.HeaderModel) (ipfs.BlockModel, error) {
func (f *IPLDFetcher) FetchHeader(tx *sqlx.Tx, c models.HeaderModel) (ipfs.BlockModel, error) {
log.Debug("fetching header ipld")
headerBytes, err := shared.FetchIPLDByMhKey(tx, c.MhKey)
if err != nil {
@@ -117,7 +116,7 @@ func (f *IPLDFetcher) FetchHeader(tx *sqlx.Tx, c eth.HeaderModel) (ipfs.BlockMod
}
// FetchUncles fetches uncles
func (f *IPLDFetcher) FetchUncles(tx *sqlx.Tx, cids []eth.UncleModel) ([]ipfs.BlockModel, error) {
func (f *IPLDFetcher) FetchUncles(tx *sqlx.Tx, cids []models.UncleModel) ([]ipfs.BlockModel, error) {
log.Debug("fetching uncle iplds")
uncleIPLDs := make([]ipfs.BlockModel, len(cids))
for i, c := range cids {
@@ -134,7 +133,7 @@ func (f *IPLDFetcher) FetchUncles(tx *sqlx.Tx, cids []eth.UncleModel) ([]ipfs.Bl
}
// FetchTrxs fetches transactions
func (f *IPLDFetcher) FetchTrxs(tx *sqlx.Tx, cids []eth.TxModel) ([]ipfs.BlockModel, error) {
func (f *IPLDFetcher) FetchTrxs(tx *sqlx.Tx, cids []models.TxModel) ([]ipfs.BlockModel, error) {
log.Debug("fetching transaction iplds")
trxIPLDs := make([]ipfs.BlockModel, len(cids))
for i, c := range cids {
@@ -151,7 +150,7 @@ func (f *IPLDFetcher) FetchTrxs(tx *sqlx.Tx, cids []eth.TxModel) ([]ipfs.BlockMo
}
// FetchRcts fetches receipts
func (f *IPLDFetcher) FetchRcts(tx *sqlx.Tx, cids []eth.ReceiptModel) ([]ipfs.BlockModel, error) {
func (f *IPLDFetcher) FetchRcts(tx *sqlx.Tx, cids []models.ReceiptModel) ([]ipfs.BlockModel, error) {
log.Debug("fetching receipt iplds")
rctIPLDs := make([]ipfs.BlockModel, len(cids))
for i, c := range cids {
@@ -168,7 +167,7 @@ func (f *IPLDFetcher) FetchRcts(tx *sqlx.Tx, cids []eth.ReceiptModel) ([]ipfs.Bl
}
// FetchState fetches state nodes
func (f *IPLDFetcher) FetchState(tx *sqlx.Tx, cids []eth.StateNodeModel) ([]StateNode, error) {
func (f *IPLDFetcher) FetchState(tx *sqlx.Tx, cids []models.StateNodeModel) ([]StateNode, error) {
log.Debug("fetching state iplds")
stateNodes := make([]StateNode, 0, len(cids))
for _, stateNode := range cids {
@@ -193,7 +192,7 @@ func (f *IPLDFetcher) FetchState(tx *sqlx.Tx, cids []eth.StateNodeModel) ([]Stat
}
// FetchStorage fetches storage nodes
func (f *IPLDFetcher) FetchStorage(tx *sqlx.Tx, cids []eth.StorageNodeWithStateKeyModel) ([]StorageNode, error) {
func (f *IPLDFetcher) FetchStorage(tx *sqlx.Tx, cids []models.StorageNodeWithStateKeyModel) ([]StorageNode, error) {
log.Debug("fetching storage iplds")
storageNodes := make([]StorageNode, 0, len(cids))
for _, storageNode := range cids {
pkg/eth/ipld_fetcher_test.go
+18 -9
View File
@@ -17,12 +17,11 @@
package eth_test
import (
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/statediff/indexer"
"github.com/ethereum/go-ethereum/statediff/indexer/postgres"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/ipld-eth-indexer/pkg/shared"
eth2 "github.com/vulcanize/ipld-eth-indexer/pkg/eth"
"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
"github.com/vulcanize/ipld-eth-server/pkg/eth"
"github.com/vulcanize/ipld-eth-server/pkg/eth/test_helpers"
@@ -31,18 +30,28 @@ import (
var _ = Describe("IPLDFetcher", func() {
var (
db *postgres.DB
pubAndIndexer *eth2.IPLDPublisher
pubAndIndexer *indexer.StateDiffIndexer
fetcher *eth.IPLDFetcher
)
Describe("Fetch", func() {
BeforeEach(func() {
var err error
db, err = shared.SetupDB()
var (
err error
tx *indexer.BlockTx
)
db, err = SetupDB()
Expect(err).ToNot(HaveOccurred())
pubAndIndexer = eth2.NewIPLDPublisher(db)
err = pubAndIndexer.Publish(test_helpers.MockConvertedPayload)
pubAndIndexer = indexer.NewStateDiffIndexer(params.TestChainConfig, db)
tx, err = pubAndIndexer.PushBlock(test_helpers.MockBlock, test_helpers.MockReceipts, test_helpers.MockBlock.Difficulty())
for _, node := range test_helpers.MockStateNodes {
err = pubAndIndexer.PushStateNode(tx, node)
Expect(err).ToNot(HaveOccurred())
}
err = tx.Close(err)
Expect(err).ToNot(HaveOccurred())
fetcher = eth.NewIPLDFetcher(db)
})
AfterEach(func() {
eth.TearDownDB(db)
pkg/eth/ipld_retriever.go
+1 -2
View File
@@ -22,9 +22,8 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/statediff/indexer/postgres"
"github.com/lib/pq"
"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
)
const (
pkg/eth/test_helpers.go
+4 -5
View File
@@ -17,10 +17,9 @@
package eth
import (
"github.com/ethereum/go-ethereum/statediff/indexer/models"
"github.com/ethereum/go-ethereum/statediff/indexer/postgres"
. "github.com/onsi/gomega"
"github.com/vulcanize/ipld-eth-indexer/pkg/eth"
"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
)
// TearDownDB is used to tear down the watcher dbs after tests
@@ -46,7 +45,7 @@ func TearDownDB(db *postgres.DB) {
}
// TxModelsContainsCID used to check if a list of TxModels contains a specific cid string
func TxModelsContainsCID(txs []eth.TxModel, cid string) bool {
func TxModelsContainsCID(txs []models.TxModel, cid string) bool {
for _, tx := range txs {
if tx.CID == cid {
return true
@@ -56,7 +55,7 @@ func TxModelsContainsCID(txs []eth.TxModel, cid string) bool {
}
// ListContainsBytes used to check if a list of byte arrays contains a particular byte array
func ReceiptModelsContainsCID(rcts []eth.ReceiptModel, cid string) bool {
func ReceiptModelsContainsCID(rcts []models.ReceiptModel, cid string) bool {
for _, rct := range rcts {
if rct.CID == cid {
return true
pkg/eth/test_helpers/test_data.go
+45 -37
View File
@@ -23,11 +23,14 @@ import (
"crypto/rand"
"math/big"
"github.com/ethereum/go-ethereum/statediff/indexer"
"github.com/ethereum/go-ethereum/statediff/indexer/ipfs"
"github.com/ethereum/go-ethereum/statediff/indexer/ipfs/ipld"
"github.com/ethereum/go-ethereum/statediff/indexer/models"
"github.com/ethereum/go-ethereum/statediff/indexer/shared"
sdtypes "github.com/ethereum/go-ethereum/statediff/types"
"github.com/ethereum/go-ethereum/trie"
"github.com/vulcanize/ipld-eth-indexer/pkg/shared"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/core/types"
@@ -39,11 +42,7 @@ import (
"github.com/multiformats/go-multihash"
log "github.com/sirupsen/logrus"
"github.com/vulcanize/ipld-eth-indexer/pkg/eth"
"github.com/vulcanize/ipld-eth-indexer/pkg/ipfs"
"github.com/vulcanize/ipld-eth-indexer/pkg/ipfs/ipld"
eth2 "github.com/vulcanize/ipld-eth-server/pkg/eth"
"github.com/vulcanize/ipld-eth-server/pkg/eth"
)
// Test variables
@@ -149,7 +148,7 @@ var (
State2MhKey = shared.MultihashKeyFromCID(State2CID)
StorageCID, _ = ipld.RawdataToCid(ipld.MEthStorageTrie, StorageLeafNode, multihash.KECCAK_256)
StorageMhKey = shared.MultihashKeyFromCID(StorageCID)
MockTrxMeta = []eth.TxModel{
MockTrxMeta = []models.TxModel{
{
CID: "", // This is empty until we go to publish to ipfs
MhKey: "",
@@ -178,7 +177,7 @@ var (
Data: MockContractByteCode,
},
}
MockTrxMetaPostPublsh = []eth.TxModel{
MockTrxMetaPostPublsh = []models.TxModel{
{
CID: Trx1CID.String(), // This is empty until we go to publish to ipfs
MhKey: Trx1MhKey,
@@ -207,7 +206,7 @@ var (
Data: MockContractByteCode,
},
}
MockRctMeta = []eth.ReceiptModel{
MockRctMeta = []models.ReceiptModel{
{
CID: "",
MhKey: "",
@@ -246,7 +245,7 @@ var (
LogContracts: []string{},
},
}
MockRctMetaPostPublish = []eth.ReceiptModel{
MockRctMetaPostPublish = []models.ReceiptModel{
{
CID: Rct1CID.String(),
MhKey: Rct1MhKey,
@@ -331,21 +330,30 @@ var (
Account,
})
MockStateNodes = []eth.TrieNode{
MockStateNodes = []sdtypes.StateNode{
{
LeafKey: common.BytesToHash(ContractLeafKey),
Path: []byte{'\x06'},
Value: ContractLeafNode,
Type: sdtypes.Leaf,
LeafKey: ContractLeafKey,
Path: []byte{'\x06'},
NodeValue: ContractLeafNode,
NodeType: sdtypes.Leaf,
StorageNodes: []sdtypes.StorageNode{
{
Path: []byte{},
NodeType: sdtypes.Leaf,
LeafKey: StorageLeafKey,
NodeValue: StorageLeafNode,
},
},
},
{
LeafKey: common.BytesToHash(AccountLeafKey),
Path: []byte{'\x0c'},
Value: AccountLeafNode,
Type: sdtypes.Leaf,
LeafKey: AccountLeafKey,
Path: []byte{'\x0c'},
NodeValue: AccountLeafNode,
NodeType: sdtypes.Leaf,
StorageNodes: []sdtypes.StorageNode{},
},
}
MockStateMetaPostPublish = []eth.StateNodeModel{
MockStateMetaPostPublish = []models.StateNodeModel{
{
CID: State1CID.String(),
MhKey: State1MhKey,
@@ -361,13 +369,13 @@ var (
StateKey: common.BytesToHash(AccountLeafKey).Hex(),
},
}
MockStorageNodes = map[string][]eth.TrieNode{
MockStorageNodes = map[string][]sdtypes.StorageNode{
contractPath: {
{
LeafKey: common.BytesToHash(StorageLeafKey),
Value: StorageLeafNode,
Type: sdtypes.Leaf,
Path: []byte{},
LeafKey: StorageLeafKey,
NodeValue: StorageLeafNode,
NodeType: sdtypes.Leaf,
Path: []byte{},
},
},
}
@@ -391,11 +399,11 @@ var (
StateNodes: MockStateNodes,
}
Reward = eth.CalcEthBlockReward(MockBlock.Header(), MockBlock.Uncles(), MockBlock.Transactions(), MockReceipts)
Reward = indexer.CalcEthBlockReward(MockBlock.Header(), MockBlock.Uncles(), MockBlock.Transactions(), MockReceipts)
MockCIDWrapper = &eth2.CIDWrapper{
MockCIDWrapper = &eth.CIDWrapper{
BlockNumber: new(big.Int).Set(BlockNumber),
Header: eth.HeaderModel{
Header: models.HeaderModel{
BlockNumber: "1",
BlockHash: MockBlock.Hash().String(),
ParentHash: "0x0000000000000000000000000000000000000000000000000000000000000000",
@@ -413,9 +421,9 @@ var (
},
Transactions: MockTrxMetaPostPublsh,
Receipts: MockRctMetaPostPublish,
Uncles: []eth.UncleModel{},
Uncles: []models.UncleModel{},
StateNodes: MockStateMetaPostPublish,
StorageNodes: []eth.StorageNodeWithStateKeyModel{
StorageNodes: []models.StorageNodeWithStateKeyModel{
{
Path: []byte{},
CID: StorageCID.String(),
@@ -438,7 +446,7 @@ var (
State2IPLD, _ = blocks.NewBlockWithCid(AccountLeafNode, State2CID)
StorageIPLD, _ = blocks.NewBlockWithCid(StorageLeafNode, StorageCID)
MockIPLDs = eth2.IPLDs{
MockIPLDs = eth.IPLDs{
BlockNumber: new(big.Int).Set(BlockNumber),
Header: ipfs.BlockModel{
Data: HeaderIPLD.RawData(),
@@ -472,7 +480,7 @@ var (
CID: Rct3IPLD.Cid().String(),
},
},
StateNodes: []eth2.StateNode{
StateNodes: []eth.StateNode{
{
StateLeafKey: common.BytesToHash(ContractLeafKey),
Type: sdtypes.Leaf,
@@ -492,7 +500,7 @@ var (
Path: []byte{'\x0c'},
},
},
StorageNodes: []eth2.StorageNode{
StorageNodes: []eth.StorageNode{
{
StateLeafKey: common.BytesToHash(ContractLeafKey),
StorageLeafKey: common.BytesToHash(StorageLeafKey),
@@ -518,7 +526,7 @@ var (
MockLondonTransactions, MockLondonReceipts, SenderAdd = createDynamicTransactionsAndReceipts(LondonBlockNum)
MockLondonBlock = createNewBlock(&MockLondonHeader, MockLondonTransactions, nil, MockLondonReceipts, new(trie.Trie))
MockLondonTrxMeta = []eth.TxModel{
MockLondonTrxMeta = []models.TxModel{
{
CID: "", // This is empty until we go to publish to ipfs
MhKey: "",
@@ -529,7 +537,7 @@ var (
Data: []byte{},
},
}
MockLondonRctMeta = []eth.ReceiptModel{
MockLondonRctMeta = []models.ReceiptModel{
{
CID: "",
MhKey: "",
@@ -605,7 +613,7 @@ func createDynamicTransactionsAndReceipts(blockNumber *big.Int) (types.Transacti
// TODO: Change the receipt type to DynamicFeeTxType once this PR is merged.
// https://github.com/ethereum/go-ethereum/pull/22806
mockReceipt1 := &types.Receipt{
Type: types.AccessListTxType,
Type: types.DynamicFeeTxType,
PostState: common.HexToHash("0x1").Bytes(),
Status: types.ReceiptStatusSuccessful,
CumulativeGasUsed: 50,
pkg/eth/types.go
+22 -8
View File
@@ -22,9 +22,10 @@ import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/statediff/indexer/ipfs"
"github.com/ethereum/go-ethereum/statediff/indexer/models"
sdtypes "github.com/ethereum/go-ethereum/statediff/types"
"github.com/vulcanize/ipld-eth-indexer/pkg/eth"
"github.com/vulcanize/ipld-eth-indexer/pkg/ipfs"
)
// RPCTransaction represents a transaction that will serialize to the RPC representation of a transaction
@@ -146,10 +147,23 @@ type StorageNode struct {
// Passed to IPLDFetcher
type CIDWrapper struct {
BlockNumber *big.Int
Header eth.HeaderModel
Uncles []eth.UncleModel
Transactions []eth.TxModel
Receipts []eth.ReceiptModel
StateNodes []eth.StateNodeModel
StorageNodes []eth.StorageNodeWithStateKeyModel
Header models.HeaderModel
Uncles []models.UncleModel
Transactions []models.TxModel
Receipts []models.ReceiptModel
StateNodes []models.StateNodeModel
StorageNodes []models.StorageNodeWithStateKeyModel
}
// ConvertedPayload is a custom type which packages raw ETH data for publishing to IPFS and filtering to subscribers
// Returned by PayloadConverter
// Passed to IPLDPublisher and ResponseFilterer
type ConvertedPayload struct {
TotalDifficulty *big.Int
Block *types.Block
TxMetaData []models.TxModel
Receipts types.Receipts
ReceiptMetaData []models.ReceiptModel
StateNodes []sdtypes.StateNode
StorageNodes map[string][]sdtypes.StorageNode
}