cerc-io/ipld-eth-server
12
0
Fork 1
Code Issues 10 Pull Requests 2 Actions Packages Projects Releases 37 Wiki Activity

api test; return full ipld models to subscribers

Browse Source
This commit is contained in:
Ian Norden
2020-02-23 17:15:26 -06:00
parent fb360d8562
commit e3e8700d34
32 changed files with 1105 additions and 774 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/super_node/eth/api.go
+14 -11
View File
@@ -20,6 +20,8 @@ import (
"context"
"math/big"
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
"github.com/ethereum/go-ethereum"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
@@ -63,6 +65,7 @@ func (pea *PublicEthAPI) GetLogs(ctx context.Context, crit ethereum.FilterQuery)
topicStrSets := make([][]string, 4)
for i, topicSet := range crit.Topics {
if i > 3 {
// don't allow more than 4 topics
break
}
for _, topic := range topicSet {
@@ -173,19 +176,19 @@ func (pea *PublicEthAPI) GetTransactionByHash(ctx context.Context, hash common.H
return nil, err
}
if tx != nil {
return newRPCTransaction(tx, blockHash, blockNumber, index), nil
return NewRPCTransaction(tx, blockHash, blockNumber, index), nil
}
// Transaction unknown, return as such
return nil, nil
}
// extractLogsOfInterest returns logs from the receipt IPLD
func extractLogsOfInterest(rctIPLDs [][]byte, wantedTopics [][]string) ([]*types.Log, error) {
func extractLogsOfInterest(rctIPLDs []ipfs.BlockModel, wantedTopics [][]string) ([]*types.Log, error) {
var logs []*types.Log
for _, rctIPLD := range rctIPLDs {
rctRLP := rctIPLD
var rct types.Receipt
if err := rlp.DecodeBytes(rctRLP, &rct); err != nil {
if err := rlp.DecodeBytes(rctRLP.Data, &rct); err != nil {
return nil, err
}
for _, log := range rct.Logs {
@@ -200,7 +203,7 @@ func extractLogsOfInterest(rctIPLDs [][]byte, wantedTopics [][]string) ([]*types
// returns true if the log matches on the filter
func wantedLog(wantedTopics [][]string, actualTopics []common.Hash) bool {
// actualTopics will always have length <= 4
// wantedTopics will always have length == 4
// wantedTopics will always have length 4
matches := 0
for i, actualTopic := range actualTopics {
// If we have topics in this filter slot, count as a match if the actualTopic matches one of the ones in this filter slot
@@ -291,7 +294,7 @@ func RPCMarshalBlock(block *types.Block, inclTx bool, fullTx bool) (map[string]i
}
if fullTx {
formatTx = func(tx *types.Transaction) (interface{}, error) {
return newRPCTransactionFromBlockHash(block, tx.Hash()), nil
return NewRPCTransactionFromBlockHash(block, tx.Hash()), nil
}
}
txs := block.Transactions()
@@ -314,8 +317,8 @@ func RPCMarshalBlock(block *types.Block, inclTx bool, fullTx bool) (map[string]i
return fields, nil
}
// newRPCTransactionFromBlockHash returns a transaction that will serialize to the RPC representation.
func newRPCTransactionFromBlockHash(b *types.Block, hash common.Hash) *RPCTransaction {
// NewRPCTransactionFromBlockHash returns a transaction that will serialize to the RPC representation.
func NewRPCTransactionFromBlockHash(b *types.Block, hash common.Hash) *RPCTransaction {
for idx, tx := range b.Transactions() {
if tx.Hash() == hash {
return newRPCTransactionFromBlockIndex(b, uint64(idx))
@@ -330,7 +333,7 @@ func newRPCTransactionFromBlockIndex(b *types.Block, index uint64) *RPCTransacti
if index >= uint64(len(txs)) {
return nil
}
return newRPCTransaction(txs[index], b.Hash(), b.NumberU64(), index)
return NewRPCTransaction(txs[index], b.Hash(), b.NumberU64(), index)
}
// RPCTransaction represents a transaction that will serialize to the RPC representation of a transaction
@@ -351,9 +354,9 @@ type RPCTransaction struct {
S *hexutil.Big `json:"s"`
}
// newRPCTransaction returns a transaction that will serialize to the RPC
// NewRPCTransaction returns a transaction that will serialize to the RPC
// representation, with the given location metadata set (if available).
func newRPCTransaction(tx *types.Transaction, blockHash common.Hash, blockNumber uint64, index uint64) *RPCTransaction {
func NewRPCTransaction(tx *types.Transaction, blockHash common.Hash, blockNumber uint64, index uint64) *RPCTransaction {
var signer types.Signer = types.FrontierSigner{}
if tx.Protected() {
signer = types.NewEIP155Signer(tx.ChainId())
@@ -366,7 +369,7 @@ func newRPCTransaction(tx *types.Transaction, blockHash common.Hash, blockNumber
Gas: hexutil.Uint64(tx.Gas()),
GasPrice: (*hexutil.Big)(tx.GasPrice()),
Hash: tx.Hash(),
Input: hexutil.Bytes(tx.Data()),
Input: hexutil.Bytes(tx.Data()), // somehow this is ending up `nil`
Nonce: hexutil.Uint64(tx.Nonce()),
To: tx.To(),
Value: (*hexutil.Big)(tx.Value()),
pkg/super_node/eth/api_test.go
+588
View File
@@ -0,0 +1,588 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package eth_test
import (
"context"
"strconv"
"github.com/ethereum/go-ethereum"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/common/hexutil"
"github.com/ethereum/go-ethereum/rpc"
"github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
mocks3 "github.com/vulcanize/vulcanizedb/pkg/ipfs/mocks"
"github.com/vulcanize/vulcanizedb/pkg/postgres"
"github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
"github.com/vulcanize/vulcanizedb/pkg/super_node/eth/mocks"
"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
)
var (
expectedBlock = map[string]interface{}{
"number": (*hexutil.Big)(mocks.MockBlock.Number()),
"hash": mocks.MockBlock.Hash(),
"parentHash": mocks.MockBlock.ParentHash(),
"nonce": mocks.MockBlock.Header().Nonce,
"mixHash": mocks.MockBlock.MixDigest(),
"sha3Uncles": mocks.MockBlock.UncleHash(),
"logsBloom": mocks.MockBlock.Bloom(),
"stateRoot": mocks.MockBlock.Root(),
"miner": mocks.MockBlock.Coinbase(),
"difficulty": (*hexutil.Big)(mocks.MockBlock.Difficulty()),
"extraData": hexutil.Bytes(mocks.MockBlock.Header().Extra),
"gasLimit": hexutil.Uint64(mocks.MockBlock.GasLimit()),
"gasUsed": hexutil.Uint64(mocks.MockBlock.GasUsed()),
"timestamp": hexutil.Uint64(mocks.MockBlock.Time()),
"transactionsRoot": mocks.MockBlock.TxHash(),
"receiptsRoot": mocks.MockBlock.ReceiptHash(),
"totalDifficulty": (*hexutil.Big)(mocks.MockBlock.Difficulty()),
"size": hexutil.Uint64(mocks.MockBlock.Size()),
}
expectedHeader = map[string]interface{}{
"number": (*hexutil.Big)(mocks.MockBlock.Header().Number),
"hash": mocks.MockBlock.Header().Hash(),
"parentHash": mocks.MockBlock.Header().ParentHash,
"nonce": mocks.MockBlock.Header().Nonce,
"mixHash": mocks.MockBlock.Header().MixDigest,
"sha3Uncles": mocks.MockBlock.Header().UncleHash,
"logsBloom": mocks.MockBlock.Header().Bloom,
"stateRoot": mocks.MockBlock.Header().Root,
"miner": mocks.MockBlock.Header().Coinbase,
"difficulty": (*hexutil.Big)(mocks.MockBlock.Header().Difficulty),
"extraData": hexutil.Bytes(mocks.MockBlock.Header().Extra),
"size": hexutil.Uint64(mocks.MockBlock.Header().Size()),
"gasLimit": hexutil.Uint64(mocks.MockBlock.Header().GasLimit),
"gasUsed": hexutil.Uint64(mocks.MockBlock.Header().GasUsed),
"timestamp": hexutil.Uint64(mocks.MockBlock.Header().Time),
"transactionsRoot": mocks.MockBlock.Header().TxHash,
"receiptsRoot": mocks.MockBlock.Header().ReceiptHash,
"totalDifficulty": (*hexutil.Big)(mocks.MockBlock.Header().Difficulty),
}
expectedTransaction = eth.NewRPCTransaction(mocks.MockTransactions[0], mocks.MockBlock.Hash(), mocks.MockBlock.NumberU64(), 0)
)
var _ = Describe("API", func() {
var (
db *postgres.DB
retriever *eth.CIDRetriever
fetcher *eth.IPLDFetcher
indexer *eth.CIDIndexer
backend *eth.Backend
api *eth.PublicEthAPI
)
BeforeEach(func() {
var err error
db, err = shared.SetupDB()
Expect(err).ToNot(HaveOccurred())
retriever = eth.NewCIDRetriever(db)
blocksToReturn := map[cid.Cid]blocks.Block{
mocks.HeaderCID: mocks.HeaderIPLD,
mocks.Trx1CID: mocks.Trx1IPLD,
mocks.Trx2CID: mocks.Trx2IPLD,
mocks.Rct1CID: mocks.Rct1IPLD,
mocks.Rct2CID: mocks.Rct2IPLD,
mocks.State1CID: mocks.State1IPLD,
mocks.State2CID: mocks.State2IPLD,
mocks.StorageCID: mocks.StorageIPLD,
}
mockBlockService := &mocks3.MockIPFSBlockService{
Blocks: blocksToReturn,
}
fetcher = &eth.IPLDFetcher{
BlockService: mockBlockService,
}
indexer = eth.NewCIDIndexer(db)
backend = &eth.Backend{
Retriever: retriever,
Fetcher: fetcher,
DB: db,
}
api = eth.NewPublicEthAPI(backend)
err = indexer.Index(mocks.MockCIDPayload)
Expect(err).ToNot(HaveOccurred())
uncles := mocks.MockBlock.Uncles()
uncleHashes := make([]common.Hash, len(uncles))
for i, uncle := range uncles {
uncleHashes[i] = uncle.Hash()
}
expectedBlock["uncles"] = uncleHashes
})
AfterEach(func() {
eth.TearDownDB(db)
})
Describe("BlockNumber", func() {
It("Retrieves the head block number", func() {
bn := api.BlockNumber()
ubn := (uint64)(bn)
subn := strconv.FormatUint(ubn, 10)
Expect(subn).To(Equal(mocks.MockCIDPayload.HeaderCID.BlockNumber))
})
})
Describe("GetTransactionByHash", func() {
It("Retrieves the head block number", func() {
hash := mocks.MockTransactions[0].Hash()
tx, err := api.GetTransactionByHash(context.Background(), hash)
Expect(err).ToNot(HaveOccurred())
Expect(tx).To(Equal(expectedTransaction))
})
})
Describe("GetBlockByNumber", func() {
It("Retrieves a block by number", func() {
// without full txs
number, err := strconv.ParseInt(mocks.MockCIDPayload.HeaderCID.BlockNumber, 10, 64)
Expect(err).ToNot(HaveOccurred())
block, err := api.GetBlockByNumber(context.Background(), rpc.BlockNumber(number), false)
Expect(err).ToNot(HaveOccurred())
transactionHashes := make([]interface{}, len(mocks.MockBlock.Transactions()))
for i, trx := range mocks.MockBlock.Transactions() {
transactionHashes[i] = trx.Hash()
}
expectedBlock["transactions"] = transactionHashes
for key, val := range expectedBlock {
Expect(val).To(Equal(block[key]))
}
// with full txs
block, err = api.GetBlockByNumber(context.Background(), rpc.BlockNumber(number), true)
Expect(err).ToNot(HaveOccurred())
transactions := make([]interface{}, len(mocks.MockBlock.Transactions()))
for i, trx := range mocks.MockBlock.Transactions() {
transactions[i] = eth.NewRPCTransactionFromBlockHash(mocks.MockBlock, trx.Hash())
}
expectedBlock["transactions"] = transactions
for key, val := range expectedBlock {
Expect(val).To(Equal(block[key]))
}
})
})
Describe("GetHeaderByNumber", func() {
It("Retrieves a header by number", func() {
number, err := strconv.ParseInt(mocks.MockCIDPayload.HeaderCID.BlockNumber, 10, 64)
Expect(err).ToNot(HaveOccurred())
header, err := api.GetHeaderByNumber(context.Background(), rpc.BlockNumber(number))
Expect(header).To(Equal(expectedHeader))
})
})
Describe("GetBlockByHash", func() {
It("Retrieves a block by hash", func() {
// without full txs
block, err := api.GetBlockByHash(context.Background(), mocks.MockBlock.Hash(), false)
Expect(err).ToNot(HaveOccurred())
transactionHashes := make([]interface{}, len(mocks.MockBlock.Transactions()))
for i, trx := range mocks.MockBlock.Transactions() {
transactionHashes[i] = trx.Hash()
}
expectedBlock["transactions"] = transactionHashes
for key, val := range expectedBlock {
Expect(val).To(Equal(block[key]))
}
// with full txs
block, err = api.GetBlockByHash(context.Background(), mocks.MockBlock.Hash(), true)
Expect(err).ToNot(HaveOccurred())
transactions := make([]interface{}, len(mocks.MockBlock.Transactions()))
for i, trx := range mocks.MockBlock.Transactions() {
transactions[i] = eth.NewRPCTransactionFromBlockHash(mocks.MockBlock, trx.Hash())
}
expectedBlock["transactions"] = transactions
for key, val := range expectedBlock {
Expect(val).To(Equal(block[key]))
}
})
})
Describe("GetLogs", func() {
It("Retrieves receipt logs that match the provided topcis within the provided range", func() {
crit := ethereum.FilterQuery{
Topics: [][]common.Hash{
{
common.HexToHash("0x04"),
},
},
FromBlock: mocks.MockBlock.Number(),
ToBlock: mocks.MockBlock.Number(),
}
logs, err := api.GetLogs(context.Background(), crit)
Expect(err).ToNot(HaveOccurred())
Expect(len(logs)).To(Equal(1))
Expect(logs).To(Equal([]*types.Log{mocks.MockLog1}))
crit = ethereum.FilterQuery{
Topics: [][]common.Hash{
{
common.HexToHash("0x04"),
common.HexToHash("0x05"),
},
},
FromBlock: mocks.MockBlock.Number(),
ToBlock: mocks.MockBlock.Number(),
}
logs, err = api.GetLogs(context.Background(), crit)
Expect(err).ToNot(HaveOccurred())
Expect(len(logs)).To(Equal(2))
Expect(logs).To(Equal([]*types.Log{mocks.MockLog1, mocks.MockLog2}))
crit = ethereum.FilterQuery{
Topics: [][]common.Hash{
{
common.HexToHash("0x04"),
common.HexToHash("0x06"),
},
},
FromBlock: mocks.MockBlock.Number(),
ToBlock: mocks.MockBlock.Number(),
}
logs, err = api.GetLogs(context.Background(), crit)
Expect(err).ToNot(HaveOccurred())
Expect(len(logs)).To(Equal(1))
Expect(logs).To(Equal([]*types.Log{mocks.MockLog1}))
crit = ethereum.FilterQuery{
Topics: [][]common.Hash{
{
common.HexToHash("0x04"),
},
{
common.HexToHash("0x07"),
},
},
FromBlock: mocks.MockBlock.Number(),
ToBlock: mocks.MockBlock.Number(),
}
logs, err = api.GetLogs(context.Background(), crit)
Expect(err).ToNot(HaveOccurred())
Expect(len(logs)).To(Equal(0))
crit = ethereum.FilterQuery{
Topics: [][]common.Hash{
{
common.HexToHash("0x04"),
},
{
common.HexToHash("0x06"),
},
},
FromBlock: mocks.MockBlock.Number(),
ToBlock: mocks.MockBlock.Number(),
}
logs, err = api.GetLogs(context.Background(), crit)
Expect(err).ToNot(HaveOccurred())
Expect(len(logs)).To(Equal(1))
Expect(logs).To(Equal([]*types.Log{mocks.MockLog1}))
crit = ethereum.FilterQuery{
Topics: [][]common.Hash{
{
common.HexToHash("0x05"),
},
{
common.HexToHash("0x07"),
},
},
FromBlock: mocks.MockBlock.Number(),
ToBlock: mocks.MockBlock.Number(),
}
logs, err = api.GetLogs(context.Background(), crit)
Expect(err).ToNot(HaveOccurred())
Expect(len(logs)).To(Equal(1))
Expect(logs).To(Equal([]*types.Log{mocks.MockLog2}))
crit = ethereum.FilterQuery{
Topics: [][]common.Hash{
{
common.HexToHash("0x05"),
},
{
common.HexToHash("0x06"),
common.HexToHash("0x07"),
},
},
FromBlock: mocks.MockBlock.Number(),
ToBlock: mocks.MockBlock.Number(),
}
logs, err = api.GetLogs(context.Background(), crit)
Expect(err).ToNot(HaveOccurred())
Expect(len(logs)).To(Equal(1))
Expect(logs).To(Equal([]*types.Log{mocks.MockLog2}))
crit = ethereum.FilterQuery{
Topics: [][]common.Hash{
{
common.HexToHash("0x04"),
common.HexToHash("0x05"),
},
{
common.HexToHash("0x06"),
common.HexToHash("0x07"),
},
},
FromBlock: mocks.MockBlock.Number(),
ToBlock: mocks.MockBlock.Number(),
}
logs, err = api.GetLogs(context.Background(), crit)
Expect(err).ToNot(HaveOccurred())
Expect(len(logs)).To(Equal(2))
Expect(logs).To(Equal([]*types.Log{mocks.MockLog1, mocks.MockLog2}))
crit = ethereum.FilterQuery{
Topics: [][]common.Hash{
{},
{
common.HexToHash("0x07"),
},
},
FromBlock: mocks.MockBlock.Number(),
ToBlock: mocks.MockBlock.Number(),
}
logs, err = api.GetLogs(context.Background(), crit)
Expect(err).ToNot(HaveOccurred())
Expect(len(logs)).To(Equal(1))
Expect(logs).To(Equal([]*types.Log{mocks.MockLog2}))
crit = ethereum.FilterQuery{
Topics: [][]common.Hash{
{},
{
common.HexToHash("0x06"),
},
},
FromBlock: mocks.MockBlock.Number(),
ToBlock: mocks.MockBlock.Number(),
}
logs, err = api.GetLogs(context.Background(), crit)
Expect(err).ToNot(HaveOccurred())
Expect(len(logs)).To(Equal(1))
Expect(logs).To(Equal([]*types.Log{mocks.MockLog1}))
crit = ethereum.FilterQuery{
Topics: [][]common.Hash{},
FromBlock: mocks.MockBlock.Number(),
ToBlock: mocks.MockBlock.Number(),
}
logs, err = api.GetLogs(context.Background(), crit)
Expect(err).ToNot(HaveOccurred())
Expect(len(logs)).To(Equal(2))
Expect(logs).To(Equal([]*types.Log{mocks.MockLog1, mocks.MockLog2}))
})
It("Uses the provided blockhash if one is provided", func() {
hash := mocks.MockBlock.Hash()
crit := ethereum.FilterQuery{
BlockHash: &hash,
Topics: [][]common.Hash{
{},
{
common.HexToHash("0x06"),
},
},
}
logs, err := api.GetLogs(context.Background(), crit)
Expect(err).ToNot(HaveOccurred())
Expect(len(logs)).To(Equal(1))
Expect(logs).To(Equal([]*types.Log{mocks.MockLog1}))
crit = ethereum.FilterQuery{
BlockHash: &hash,
Topics: [][]common.Hash{
{
common.HexToHash("0x04"),
},
{
common.HexToHash("0x06"),
},
},
}
logs, err = api.GetLogs(context.Background(), crit)
Expect(err).ToNot(HaveOccurred())
Expect(len(logs)).To(Equal(1))
Expect(logs).To(Equal([]*types.Log{mocks.MockLog1}))
crit = ethereum.FilterQuery{
BlockHash: &hash,
Topics: [][]common.Hash{
{},
{
common.HexToHash("0x07"),
},
},
}
logs, err = api.GetLogs(context.Background(), crit)
Expect(err).ToNot(HaveOccurred())
Expect(len(logs)).To(Equal(1))
Expect(logs).To(Equal([]*types.Log{mocks.MockLog2}))
crit = ethereum.FilterQuery{
BlockHash: &hash,
Topics: [][]common.Hash{
{
common.HexToHash("0x05"),
},
{
common.HexToHash("0x07"),
},
},
}
logs, err = api.GetLogs(context.Background(), crit)
Expect(err).ToNot(HaveOccurred())
Expect(len(logs)).To(Equal(1))
Expect(logs).To(Equal([]*types.Log{mocks.MockLog2}))
crit = ethereum.FilterQuery{
BlockHash: &hash,
Topics: [][]common.Hash{
{
common.HexToHash("0x04"),
},
{
common.HexToHash("0x07"),
},
},
}
logs, err = api.GetLogs(context.Background(), crit)
Expect(err).ToNot(HaveOccurred())
Expect(len(logs)).To(Equal(0))
crit = ethereum.FilterQuery{
BlockHash: &hash,
Topics: [][]common.Hash{
{
common.HexToHash("0x04"),
common.HexToHash("0x05"),
},
{
common.HexToHash("0x07"),
},
},
}
logs, err = api.GetLogs(context.Background(), crit)
Expect(err).ToNot(HaveOccurred())
Expect(len(logs)).To(Equal(1))
Expect(logs).To(Equal([]*types.Log{mocks.MockLog2}))
crit = ethereum.FilterQuery{
BlockHash: &hash,
Topics: [][]common.Hash{
{
common.HexToHash("0x04"),
common.HexToHash("0x05"),
},
},
}
logs, err = api.GetLogs(context.Background(), crit)
Expect(err).ToNot(HaveOccurred())
Expect(len(logs)).To(Equal(2))
Expect(logs).To(Equal([]*types.Log{mocks.MockLog1, mocks.MockLog2}))
crit = ethereum.FilterQuery{
BlockHash: &hash,
Topics: [][]common.Hash{
{
common.HexToHash("0x04"),
common.HexToHash("0x05"),
},
{
common.HexToHash("0x06"),
common.HexToHash("0x07"),
},
},
}
logs, err = api.GetLogs(context.Background(), crit)
Expect(err).ToNot(HaveOccurred())
Expect(len(logs)).To(Equal(2))
Expect(logs).To(Equal([]*types.Log{mocks.MockLog1, mocks.MockLog2}))
crit = ethereum.FilterQuery{
BlockHash: &hash,
Topics: [][]common.Hash{},
}
logs, err = api.GetLogs(context.Background(), crit)
Expect(err).ToNot(HaveOccurred())
Expect(len(logs)).To(Equal(2))
Expect(logs).To(Equal([]*types.Log{mocks.MockLog1, mocks.MockLog2}))
})
It("Filters on contract address if any are provided", func() {
hash := mocks.MockBlock.Hash()
crit := ethereum.FilterQuery{
BlockHash: &hash,
Addresses: []common.Address{
mocks.Address,
},
Topics: [][]common.Hash{
{
common.HexToHash("0x04"),
common.HexToHash("0x05"),
},
{
common.HexToHash("0x06"),
common.HexToHash("0x07"),
},
},
}
logs, err := api.GetLogs(context.Background(), crit)
Expect(err).ToNot(HaveOccurred())
Expect(len(logs)).To(Equal(1))
Expect(logs).To(Equal([]*types.Log{mocks.MockLog1}))
hash = mocks.MockBlock.Hash()
crit = ethereum.FilterQuery{
BlockHash: &hash,
Addresses: []common.Address{
mocks.Address,
mocks.AnotherAddress,
},
Topics: [][]common.Hash{
{
common.HexToHash("0x04"),
common.HexToHash("0x05"),
},
{
common.HexToHash("0x06"),
common.HexToHash("0x07"),
},
},
}
logs, err = api.GetLogs(context.Background(), crit)
Expect(err).ToNot(HaveOccurred())
Expect(len(logs)).To(Equal(2))
Expect(logs).To(Equal([]*types.Log{mocks.MockLog1, mocks.MockLog2}))
hash = mocks.MockBlock.Hash()
crit = ethereum.FilterQuery{
BlockHash: &hash,
Addresses: []common.Address{
mocks.Address,
mocks.AnotherAddress,
},
}
logs, err = api.GetLogs(context.Background(), crit)
Expect(err).ToNot(HaveOccurred())
Expect(len(logs)).To(Equal(2))
Expect(logs).To(Equal([]*types.Log{mocks.MockLog1, mocks.MockLog2}))
})
})
})
pkg/super_node/eth/backend.go
+35 -34
View File
@@ -93,19 +93,19 @@ func (b *Backend) HeaderByNumber(ctx context.Context, blockNumber rpc.BlockNumbe
// Decode the first header at this block height and return it
// We throw an error in FetchHeaders() if the number of headers does not match the number of CIDs and we already
// confirmed the number of CIDs is greater than 0 so there is no need to bound check the slice before accessing
header := new(types.Header)
if err := rlp.DecodeBytes(headerIPLDs[0], header); err != nil {
var header types.Header
if err := rlp.DecodeBytes(headerIPLDs[0].Data, &header); err != nil {
return nil, err
}
return header, nil
return &header, nil
}
// GetTd retrieves and returns the total difficulty at the given block hash
func (b *Backend) GetTd(blockHash common.Hash) (*big.Int, error) {
pgStr := `SELECT header_cids.td FROM header_cids
pgStr := `SELECT td FROM eth.header_cids
WHERE header_cids.block_hash = $1`
var tdStr string
err := b.DB.Select(&tdStr, pgStr, blockHash.String())
err := b.DB.Get(&tdStr, pgStr, blockHash.String())
if err != nil {
return nil, err
}
@@ -142,7 +142,7 @@ func (b *Backend) GetLogs(ctx context.Context, hash common.Hash) ([][]*types.Log
logs := make([][]*types.Log, len(receiptIPLDs))
for i, rctIPLD := range receiptIPLDs {
var rct types.Receipt
if err := rlp.DecodeBytes(rctIPLD, &rct); err != nil {
if err := rlp.DecodeBytes(rctIPLD.Data, &rct); err != nil {
return nil, err
}
logs[i] = rct.Logs
@@ -170,13 +170,14 @@ func (b *Backend) BlockByNumber(ctx context.Context, blockNumber rpc.BlockNumber
if err != nil {
return nil, err
}
// Fetch and decode the header IPLD
headerIPLDs, err := b.Fetcher.FetchHeaders([]HeaderModel{headerCID})
if err != nil {
return nil, err
}
var header *types.Header
if err := rlp.DecodeBytes(headerIPLDs[0], header); err != nil {
var header types.Header
if err := rlp.DecodeBytes(headerIPLDs[0].Data, &header); err != nil {
return nil, err
}
// Fetch and decode the uncle IPLDs
@@ -186,11 +187,11 @@ func (b *Backend) BlockByNumber(ctx context.Context, blockNumber rpc.BlockNumber
}
var uncles []*types.Header
for _, uncleIPLD := range uncleIPLDs {
var uncle *types.Header
if err := rlp.DecodeBytes(uncleIPLD, uncle); err != nil {
var uncle types.Header
if err := rlp.DecodeBytes(uncleIPLD.Data, &uncle); err != nil {
return nil, err
}
uncles = append(uncles, uncle)
uncles = append(uncles, &uncle)
}
// Fetch and decode the transaction IPLDs
txIPLDs, err := b.Fetcher.FetchTrxs(txCIDs)
@@ -199,11 +200,11 @@ func (b *Backend) BlockByNumber(ctx context.Context, blockNumber rpc.BlockNumber
}
var transactions []*types.Transaction
for _, txIPLD := range txIPLDs {
var tx *types.Transaction
if err := rlp.DecodeBytes(txIPLD, tx); err != nil {
var tx types.Transaction
if err := rlp.DecodeBytes(txIPLD.Data, &tx); err != nil {
return nil, err
}
transactions = append(transactions, tx)
transactions = append(transactions, &tx)
}
// Fetch and decode the receipt IPLDs
rctIPLDs, err := b.Fetcher.FetchRcts(rctCIDs)
@@ -212,14 +213,14 @@ func (b *Backend) BlockByNumber(ctx context.Context, blockNumber rpc.BlockNumber
}
var receipts []*types.Receipt
for _, rctIPLD := range rctIPLDs {
var receipt *types.Receipt
if err := rlp.DecodeBytes(rctIPLD, receipt); err != nil {
var receipt types.Receipt
if err := rlp.DecodeBytes(rctIPLD.Data, &receipt); err != nil {
return nil, err
}
receipts = append(receipts, receipt)
receipts = append(receipts, &receipt)
}
// Compose everything together into a complete block
return types.NewBlock(header, transactions, uncles, receipts), nil
return types.NewBlock(&header, transactions, uncles, receipts), nil
}
// BlockByHash returns the requested block. When fullTx is true all transactions in the block are returned in full
@@ -235,8 +236,8 @@ func (b *Backend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Blo
if err != nil {
return nil, err
}
var header *types.Header
if err := rlp.DecodeBytes(headerIPLDs[0], header); err != nil {
var header types.Header
if err := rlp.DecodeBytes(headerIPLDs[0].Data, &header); err != nil {
return nil, err
}
// Fetch and decode the uncle IPLDs
@@ -246,11 +247,11 @@ func (b *Backend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Blo
}
var uncles []*types.Header
for _, uncleIPLD := range uncleIPLDs {
var uncle *types.Header
if err := rlp.DecodeBytes(uncleIPLD, uncle); err != nil {
var uncle types.Header
if err := rlp.DecodeBytes(uncleIPLD.Data, &uncle); err != nil {
return nil, err
}
uncles = append(uncles, uncle)
uncles = append(uncles, &uncle)
}
// Fetch and decode the transaction IPLDs
txIPLDs, err := b.Fetcher.FetchTrxs(txCIDs)
@@ -259,11 +260,11 @@ func (b *Backend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Blo
}
var transactions []*types.Transaction
for _, txIPLD := range txIPLDs {
var tx *types.Transaction
if err := rlp.DecodeBytes(txIPLD, tx); err != nil {
var tx types.Transaction
if err := rlp.DecodeBytes(txIPLD.Data, &tx); err != nil {
return nil, err
}
transactions = append(transactions, tx)
transactions = append(transactions, &tx)
}
// Fetch and decode the receipt IPLDs
rctIPLDs, err := b.Fetcher.FetchRcts(rctCIDs)
@@ -272,21 +273,21 @@ func (b *Backend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Blo
}
var receipts []*types.Receipt
for _, rctIPLD := range rctIPLDs {
var receipt *types.Receipt
if err := rlp.DecodeBytes(rctIPLD, receipt); err != nil {
var receipt types.Receipt
if err := rlp.DecodeBytes(rctIPLD.Data, &receipt); err != nil {
return nil, err
}
receipts = append(receipts, receipt)
receipts = append(receipts, &receipt)
}
// Compose everything together into a complete block
return types.NewBlock(header, transactions, uncles, receipts), nil
return types.NewBlock(&header, transactions, uncles, receipts), nil
}
// GetTransaction retrieves a tx by hash
// It also returns the blockhash, blocknumber, and tx index associated with the transaction
func (b *Backend) GetTransaction(ctx context.Context, txHash common.Hash) (*types.Transaction, common.Hash, uint64, uint64, error) {
pgStr := `SELECT transaction_cids.cid, transaction_cids.index, header_cids.block_hash, header_cids.block_number
FROM transaction_cids, header_cids
FROM eth.transaction_cids, eth.header_cids
WHERE transaction_cids.header_id = header_cids.id
AND transaction_cids.tx_hash = $1`
var txCIDWithHeaderInfo struct {
@@ -302,9 +303,9 @@ func (b *Backend) GetTransaction(ctx context.Context, txHash common.Hash) (*type
if err != nil {
return nil, common.Hash{}, 0, 0, err
}
var transaction *types.Transaction
if err := rlp.DecodeBytes(txIPLD[0], transaction); err != nil {
var transaction types.Transaction
if err := rlp.DecodeBytes(txIPLD[0].Data, &transaction); err != nil {
return nil, common.Hash{}, 0, 0, err
}
return transaction, common.HexToHash(txCIDWithHeaderInfo.BlockHash), uint64(txCIDWithHeaderInfo.BlockNumber), uint64(txCIDWithHeaderInfo.Index), nil
return &transaction, common.HexToHash(txCIDWithHeaderInfo.BlockHash), uint64(txCIDWithHeaderInfo.BlockNumber), uint64(txCIDWithHeaderInfo.Index), nil
}
pkg/super_node/eth/filterer.go
+59 -13
View File
@@ -24,7 +24,10 @@ import (
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/rlp"
"github.com/multiformats/go-multihash"
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
"github.com/vulcanize/vulcanizedb/pkg/ipfs/ipld"
"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
)
@@ -80,15 +83,29 @@ func (s *ResponseFilterer) filterHeaders(headerFilter HeaderFilter, response *IP
if err != nil {
return err
}
response.Headers = append(response.Headers, headerRLP)
cid, err := ipld.RawdataToCid(ipld.MEthHeader, headerRLP, multihash.KECCAK_256)
if err != nil {
return err
}
response.Headers = append(response.Headers, ipfs.BlockModel{
Data: headerRLP,
CID: cid.String(),
})
if headerFilter.Uncles {
response.Uncles = make([][]byte, len(payload.Block.Body().Uncles))
response.Uncles = make([]ipfs.BlockModel, len(payload.Block.Body().Uncles))
for i, uncle := range payload.Block.Body().Uncles {
uncleRlp, err := rlp.EncodeToBytes(uncle)
if err != nil {
return err
}
response.Uncles[i] = uncleRlp
cid, err := ipld.RawdataToCid(ipld.MEthHeader, uncleRlp, multihash.KECCAK_256)
if err != nil {
return err
}
response.Uncles[i] = ipfs.BlockModel{
Data: uncleRlp,
CID: cid.String(),
}
}
}
}
@@ -107,14 +124,22 @@ func (s *ResponseFilterer) filterTransactions(trxFilter TxFilter, response *IPLD
if !trxFilter.Off {
trxLen := len(payload.Block.Body().Transactions)
trxHashes = make([]common.Hash, 0, trxLen)
response.Transactions = make([][]byte, 0, trxLen)
response.Transactions = make([]ipfs.BlockModel, 0, trxLen)
for i, trx := range payload.Block.Body().Transactions {
if checkTransactionAddrs(trxFilter.Src, trxFilter.Dst, payload.TxMetaData[i].Src, payload.TxMetaData[i].Dst) {
trxBuffer := new(bytes.Buffer)
if err := trx.EncodeRLP(trxBuffer); err != nil {
return nil, err
}
response.Transactions = append(response.Transactions, trxBuffer.Bytes())
data := trxBuffer.Bytes()
cid, err := ipld.RawdataToCid(ipld.MEthTx, data, multihash.KECCAK_256)
if err != nil {
return nil, err
}
response.Transactions = append(response.Transactions, ipfs.BlockModel{
Data: data,
CID: cid.String(),
})
trxHashes = append(trxHashes, trx.Hash())
}
}
@@ -143,17 +168,24 @@ func checkTransactionAddrs(wantedSrc, wantedDst []string, actualSrc, actualDst s
func (s *ResponseFilterer) filerReceipts(receiptFilter ReceiptFilter, response *IPLDs, payload ConvertedPayload, trxHashes []common.Hash) error {
if !receiptFilter.Off {
response.Receipts = make([][]byte, 0, len(payload.Receipts))
response.Receipts = make([]ipfs.BlockModel, 0, len(payload.Receipts))
for i, receipt := range payload.Receipts {
// topics is always length 4
topics := [][]string{payload.ReceiptMetaData[i].Topic0s, payload.ReceiptMetaData[i].Topic1s, payload.ReceiptMetaData[i].Topic2s, payload.ReceiptMetaData[i].Topic3s}
if checkReceipts(receipt, receiptFilter.Topics, topics, receiptFilter.Contracts, payload.ReceiptMetaData[i].Contract, trxHashes) {
receiptForStorage := (*types.ReceiptForStorage)(receipt)
receiptBuffer := new(bytes.Buffer)
if err := receiptForStorage.EncodeRLP(receiptBuffer); err != nil {
if err := receipt.EncodeRLP(receiptBuffer); err != nil {
return err
}
response.Receipts = append(response.Receipts, receiptBuffer.Bytes())
data := receiptBuffer.Bytes()
cid, err := ipld.RawdataToCid(ipld.MEthTxReceipt, data, multihash.KECCAK_256)
if err != nil {
return err
}
response.Receipts = append(response.Receipts, ipfs.BlockModel{
Data: data,
CID: cid.String(),
})
}
}
}
@@ -231,10 +263,17 @@ func (s *ResponseFilterer) filterState(stateFilter StateFilter, response *IPLDs,
for _, stateNode := range payload.StateNodes {
if checkNodeKeys(keyFilters, stateNode.Key) {
if stateNode.Leaf || stateFilter.IntermediateNodes {
cid, err := ipld.RawdataToCid(ipld.MEthStateTrie, stateNode.Value, multihash.KECCAK_256)
if err != nil {
return err
}
response.StateNodes = append(response.StateNodes, StateNode{
StateTrieKey: stateNode.Key,
IPLD: stateNode.Value,
Leaf: stateNode.Leaf,
IPLD: ipfs.BlockModel{
Data: stateNode.Value,
CID: cid.String(),
},
Leaf: stateNode.Leaf,
})
}
}
@@ -271,11 +310,18 @@ func (s *ResponseFilterer) filterStorage(storageFilter StorageFilter, response *
if checkNodeKeys(stateKeyFilters, stateKey) {
for _, storageNode := range storageNodes {
if checkNodeKeys(storageKeyFilters, storageNode.Key) {
cid, err := ipld.RawdataToCid(ipld.MEthStorageTrie, storageNode.Value, multihash.KECCAK_256)
if err != nil {
return err
}
response.StorageNodes = append(response.StorageNodes, StorageNode{
StateTrieKey: stateKey,
StorageTrieKey: storageNode.Key,
IPLD: storageNode.Value,
Leaf: storageNode.Leaf,
IPLD: ipfs.BlockModel{
Data: storageNode.Value,
CID: cid.String(),
},
Leaf: storageNode.Leaf,
})
}
}
pkg/super_node/eth/filterer_test.go
+45 -34
View File
@@ -17,9 +17,8 @@
package eth_test
import (
"bytes"
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
"github.com/ethereum/go-ethereum/core/types"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
@@ -29,17 +28,13 @@ import (
)
var (
filterer *eth.ResponseFilterer
expectedRctForStorageRLP1 []byte
expectedRctForStorageRLP2 []byte
filterer *eth.ResponseFilterer
)
var _ = Describe("Filterer", func() {
Describe("FilterResponse", func() {
BeforeEach(func() {
filterer = eth.NewResponseFilterer()
expectedRctForStorageRLP1 = getReceiptForStorageRLP(mocks.MockReceipts, 0)
expectedRctForStorageRLP2 = getReceiptForStorageRLP(mocks.MockReceipts, 1)
})
It("Transcribes all the data from the IPLDPayload into the StreamPayload if given an open filter", func() {
@@ -49,22 +44,28 @@ var _ = Describe("Filterer", func() {
Expect(ok).To(BeTrue())
Expect(iplds.BlockNumber.Int64()).To(Equal(mocks.MockIPLDs.BlockNumber.Int64()))
Expect(iplds.Headers).To(Equal(mocks.MockIPLDs.Headers))
var unclesRlp [][]byte
Expect(iplds.Uncles).To(Equal(unclesRlp))
var expectedEmptyUncles []ipfs.BlockModel
Expect(iplds.Uncles).To(Equal(expectedEmptyUncles))
Expect(len(iplds.Transactions)).To(Equal(2))
Expect(shared.ListContainsBytes(iplds.Transactions, mocks.MockTransactions.GetRlp(0))).To(BeTrue())
Expect(shared.ListContainsBytes(iplds.Transactions, mocks.MockTransactions.GetRlp(1))).To(BeTrue())
Expect(shared.IPLDsContainBytes(iplds.Transactions, mocks.MockTransactions.GetRlp(0))).To(BeTrue())
Expect(shared.IPLDsContainBytes(iplds.Transactions, mocks.MockTransactions.GetRlp(1))).To(BeTrue())
Expect(len(iplds.Receipts)).To(Equal(2))
Expect(shared.ListContainsBytes(iplds.Receipts, expectedRctForStorageRLP1)).To(BeTrue())
Expect(shared.ListContainsBytes(iplds.Receipts, expectedRctForStorageRLP2)).To(BeTrue())
Expect(shared.IPLDsContainBytes(iplds.Receipts, mocks.MockReceipts.GetRlp(0))).To(BeTrue())
Expect(shared.IPLDsContainBytes(iplds.Receipts, mocks.MockReceipts.GetRlp(1))).To(BeTrue())
Expect(len(iplds.StateNodes)).To(Equal(2))
for _, stateNode := range iplds.StateNodes {
Expect(stateNode.Leaf).To(BeTrue())
if stateNode.StateTrieKey == mocks.ContractLeafKey {
Expect(stateNode.IPLD).To(Equal(mocks.ValueBytes))
Expect(stateNode.IPLD).To(Equal(ipfs.BlockModel{
Data: mocks.State1IPLD.RawData(),
CID: mocks.State1IPLD.Cid().String(),
}))
}
if stateNode.StateTrieKey == mocks.AnotherContractLeafKey {
Expect(stateNode.IPLD).To(Equal(mocks.AnotherValueBytes))
Expect(stateNode.IPLD).To(Equal(ipfs.BlockModel{
Data: mocks.State2IPLD.RawData(),
CID: mocks.State2IPLD.Cid().String(),
}))
}
}
Expect(iplds.StorageNodes).To(Equal(mocks.MockIPLDs.StorageNodes))
@@ -82,7 +83,10 @@ var _ = Describe("Filterer", func() {
Expect(len(iplds1.StorageNodes)).To(Equal(0))
Expect(len(iplds1.StateNodes)).To(Equal(0))
Expect(len(iplds1.Receipts)).To(Equal(1))
Expect(iplds1.Receipts[0]).To(Equal(expectedRctForStorageRLP2))
Expect(iplds1.Receipts[0]).To(Equal(ipfs.BlockModel{
Data: mocks.Rct2IPLD.RawData(),
CID: mocks.Rct2IPLD.Cid().String(),
}))
payload2, err := filterer.Filter(rctTopicsFilter, mocks.MockConvertedPayload)
Expect(err).ToNot(HaveOccurred())
@@ -95,7 +99,10 @@ var _ = Describe("Filterer", func() {
Expect(len(iplds2.StorageNodes)).To(Equal(0))
Expect(len(iplds2.StateNodes)).To(Equal(0))
Expect(len(iplds2.Receipts)).To(Equal(1))
Expect(iplds2.Receipts[0]).To(Equal(expectedRctForStorageRLP1))
Expect(iplds2.Receipts[0]).To(Equal(ipfs.BlockModel{
Data: mocks.Rct1IPLD.RawData(),
CID: mocks.Rct1IPLD.Cid().String(),
}))
payload3, err := filterer.Filter(rctTopicsAndContractFilter, mocks.MockConvertedPayload)
Expect(err).ToNot(HaveOccurred())
@@ -108,7 +115,10 @@ var _ = Describe("Filterer", func() {
Expect(len(iplds3.StorageNodes)).To(Equal(0))
Expect(len(iplds3.StateNodes)).To(Equal(0))
Expect(len(iplds3.Receipts)).To(Equal(1))
Expect(iplds3.Receipts[0]).To(Equal(expectedRctForStorageRLP1))
Expect(iplds3.Receipts[0]).To(Equal(ipfs.BlockModel{
Data: mocks.Rct1IPLD.RawData(),
CID: mocks.Rct1IPLD.Cid().String(),
}))
payload4, err := filterer.Filter(rctContractsAndTopicFilter, mocks.MockConvertedPayload)
Expect(err).ToNot(HaveOccurred())
@@ -121,7 +131,10 @@ var _ = Describe("Filterer", func() {
Expect(len(iplds4.StorageNodes)).To(Equal(0))
Expect(len(iplds4.StateNodes)).To(Equal(0))
Expect(len(iplds4.Receipts)).To(Equal(1))
Expect(iplds4.Receipts[0]).To(Equal(expectedRctForStorageRLP2))
Expect(iplds4.Receipts[0]).To(Equal(ipfs.BlockModel{
Data: mocks.Rct2IPLD.RawData(),
CID: mocks.Rct2IPLD.Cid().String(),
}))
payload5, err := filterer.Filter(rctsForAllCollectedTrxs, mocks.MockConvertedPayload)
Expect(err).ToNot(HaveOccurred())
@@ -131,13 +144,13 @@ var _ = Describe("Filterer", func() {
Expect(len(iplds5.Headers)).To(Equal(0))
Expect(len(iplds5.Uncles)).To(Equal(0))
Expect(len(iplds5.Transactions)).To(Equal(2))
Expect(shared.ListContainsBytes(iplds5.Transactions, mocks.MockTransactions.GetRlp(0))).To(BeTrue())
Expect(shared.ListContainsBytes(iplds5.Transactions, mocks.MockTransactions.GetRlp(1))).To(BeTrue())
Expect(shared.IPLDsContainBytes(iplds5.Transactions, mocks.MockTransactions.GetRlp(0))).To(BeTrue())
Expect(shared.IPLDsContainBytes(iplds5.Transactions, mocks.MockTransactions.GetRlp(1))).To(BeTrue())
Expect(len(iplds5.StorageNodes)).To(Equal(0))
Expect(len(iplds5.StateNodes)).To(Equal(0))
Expect(len(iplds5.Receipts)).To(Equal(2))
Expect(shared.ListContainsBytes(iplds5.Receipts, expectedRctForStorageRLP1)).To(BeTrue())
Expect(shared.ListContainsBytes(iplds5.Receipts, expectedRctForStorageRLP2)).To(BeTrue())
Expect(shared.IPLDsContainBytes(iplds5.Receipts, mocks.MockReceipts.GetRlp(0))).To(BeTrue())
Expect(shared.IPLDsContainBytes(iplds5.Receipts, mocks.MockReceipts.GetRlp(1))).To(BeTrue())
payload6, err := filterer.Filter(rctsForSelectCollectedTrxs, mocks.MockConvertedPayload)
Expect(err).ToNot(HaveOccurred())
@@ -147,11 +160,14 @@ var _ = Describe("Filterer", func() {
Expect(len(iplds6.Headers)).To(Equal(0))
Expect(len(iplds6.Uncles)).To(Equal(0))
Expect(len(iplds6.Transactions)).To(Equal(1))
Expect(shared.ListContainsBytes(iplds5.Transactions, mocks.MockTransactions.GetRlp(1))).To(BeTrue())
Expect(shared.IPLDsContainBytes(iplds5.Transactions, mocks.MockTransactions.GetRlp(1))).To(BeTrue())
Expect(len(iplds6.StorageNodes)).To(Equal(0))
Expect(len(iplds6.StateNodes)).To(Equal(0))
Expect(len(iplds6.Receipts)).To(Equal(1))
Expect(iplds4.Receipts[0]).To(Equal(expectedRctForStorageRLP2))
Expect(iplds4.Receipts[0]).To(Equal(ipfs.BlockModel{
Data: mocks.Rct2IPLD.RawData(),
CID: mocks.Rct2IPLD.Cid().String(),
}))
payload7, err := filterer.Filter(stateFilter, mocks.MockConvertedPayload)
Expect(err).ToNot(HaveOccurred())
@@ -165,7 +181,10 @@ var _ = Describe("Filterer", func() {
Expect(len(iplds7.Receipts)).To(Equal(0))
Expect(len(iplds7.StateNodes)).To(Equal(1))
Expect(iplds7.StateNodes[0].StateTrieKey).To(Equal(mocks.ContractLeafKey))
Expect(iplds7.StateNodes[0].IPLD).To(Equal(mocks.ValueBytes))
Expect(iplds7.StateNodes[0].IPLD).To(Equal(ipfs.BlockModel{
Data: mocks.State1IPLD.RawData(),
CID: mocks.State1IPLD.Cid().String(),
}))
payload8, err := filterer.Filter(rctTopicsAndContractFilterFail, mocks.MockConvertedPayload)
Expect(err).ToNot(HaveOccurred())
@@ -181,11 +200,3 @@ var _ = Describe("Filterer", func() {
})
})
})
func getReceiptForStorageRLP(receipts types.Receipts, i int) []byte {
receiptForStorage := (*types.ReceiptForStorage)(receipts[i])
receiptBuffer := new(bytes.Buffer)
err := receiptForStorage.EncodeRLP(receiptBuffer)
Expect(err).ToNot(HaveOccurred())
return receiptBuffer.Bytes()
}
pkg/super_node/eth/indexer.go
+5 -5
View File
@@ -82,16 +82,16 @@ func (in *CIDIndexer) Index(cids shared.CIDsForIndexing) error {
func (in *CIDIndexer) indexHeaderCID(tx *sqlx.Tx, header HeaderModel, nodeID int64) (int64, error) {
var headerID int64
err := tx.QueryRowx(`INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_id) VALUES ($1, $2, $3, $4, $5, $6)
ON CONFLICT (block_number, block_hash) DO UPDATE SET (parent_hash, cid, td, node_id) = ($3, $4, $5, $6)
err := tx.QueryRowx(`INSERT INTO eth.header_cids (block_number, block_hash, parent_hash, cid, td, node_id, reward) VALUES ($1, $2, $3, $4, $5, $6, $7)
ON CONFLICT (block_number, block_hash) DO UPDATE SET (parent_hash, cid, td, node_id, reward) = ($3, $4, $5, $6, $7)
RETURNING id`,
header.BlockNumber, header.BlockHash, header.ParentHash, header.CID, header.TotalDifficulty, nodeID).Scan(&headerID)
header.BlockNumber, header.BlockHash, header.ParentHash, header.CID, header.TotalDifficulty, nodeID, header.Reward).Scan(&headerID)
return headerID, err
}
func (in *CIDIndexer) indexUncleCID(tx *sqlx.Tx, uncle UncleModel, headerID int64) error {
_, err := tx.Exec(`INSERT INTO eth.uncle_cids (block_hash, header_id, parent_hash, cid) VALUES ($1, $2, $3, $4)
ON CONFLICT (header_id, block_hash) DO UPDATE SET (parent_hash, cid) = ($3, $4)`,
_, err := tx.Exec(`INSERT INTO eth.uncle_cids (block_hash, header_id, parent_hash, cid, reward) VALUES ($1, $2, $3, $4, $5)
ON CONFLICT (header_id, block_hash) DO UPDATE SET (parent_hash, cid, reward) = ($3, $4, $5)`,
uncle.BlockHash, headerID, uncle.ParentHash, uncle.CID)
return err
}
pkg/super_node/eth/indexer_test.go
+14 -12
View File
@@ -45,18 +45,20 @@ var _ = Describe("Indexer", func() {
It("Indexes CIDs and related metadata into vulcanizedb", func() {
err = repo.Index(mocks.MockCIDPayload)
Expect(err).ToNot(HaveOccurred())
pgStr := `SELECT cid, td FROM eth.header_cids
pgStr := `SELECT cid, td, reward FROM eth.header_cids
WHERE block_number = $1`
// check header was properly indexed
type res struct {
CID string
TD string
CID string
TD string
Reward string
}
headers := new(res)
err = db.QueryRowx(pgStr, 1).StructScan(headers)
Expect(err).ToNot(HaveOccurred())
Expect(headers.CID).To(Equal("mockHeaderCID"))
Expect(headers.TD).To(Equal("1337"))
Expect(headers.CID).To(Equal(mocks.HeaderCID.String()))
Expect(headers.TD).To(Equal(mocks.MockBlock.Difficulty().String()))
Expect(headers.Reward).To(Equal("5000000000000000000"))
// check trxs were properly indexed
trxs := make([]string, 0)
pgStr = `SELECT transaction_cids.cid FROM eth.transaction_cids INNER JOIN eth.header_cids ON (transaction_cids.header_id = header_cids.id)
@@ -64,8 +66,8 @@ var _ = Describe("Indexer", func() {
err = db.Select(&trxs, pgStr, 1)
Expect(err).ToNot(HaveOccurred())
Expect(len(trxs)).To(Equal(2))
Expect(shared.ListContainsString(trxs, "mockTrxCID1")).To(BeTrue())
Expect(shared.ListContainsString(trxs, "mockTrxCID2")).To(BeTrue())
Expect(shared.ListContainsString(trxs, mocks.Trx1CID.String())).To(BeTrue())
Expect(shared.ListContainsString(trxs, mocks.Trx2CID.String())).To(BeTrue())
// check receipts were properly indexed
rcts := make([]string, 0)
pgStr = `SELECT receipt_cids.cid FROM eth.receipt_cids, eth.transaction_cids, eth.header_cids
@@ -75,8 +77,8 @@ var _ = Describe("Indexer", func() {
err = db.Select(&rcts, pgStr, 1)
Expect(err).ToNot(HaveOccurred())
Expect(len(rcts)).To(Equal(2))
Expect(shared.ListContainsString(rcts, "mockRctCID1")).To(BeTrue())
Expect(shared.ListContainsString(rcts, "mockRctCID2")).To(BeTrue())
Expect(shared.ListContainsString(rcts, mocks.Rct1CID.String())).To(BeTrue())
Expect(shared.ListContainsString(rcts, mocks.Rct2CID.String())).To(BeTrue())
// check that state nodes were properly indexed
stateNodes := make([]eth.StateNodeModel, 0)
pgStr = `SELECT state_cids.cid, state_cids.state_key, state_cids.leaf FROM eth.state_cids INNER JOIN eth.header_cids ON (state_cids.header_id = header_cids.id)
@@ -85,11 +87,11 @@ var _ = Describe("Indexer", func() {
Expect(err).ToNot(HaveOccurred())
Expect(len(stateNodes)).To(Equal(2))
for _, stateNode := range stateNodes {
if stateNode.CID == "mockStateCID1" {
if stateNode.CID == mocks.State1CID.String() {
Expect(stateNode.Leaf).To(Equal(true))
Expect(stateNode.StateKey).To(Equal(mocks.ContractLeafKey.Hex()))
}
if stateNode.CID == "mockStateCID2" {
if stateNode.CID == mocks.State2CID.String() {
Expect(stateNode.Leaf).To(Equal(true))
Expect(stateNode.StateKey).To(Equal(mocks.AnotherContractLeafKey.Hex()))
}
@@ -104,7 +106,7 @@ var _ = Describe("Indexer", func() {
Expect(err).ToNot(HaveOccurred())
Expect(len(storageNodes)).To(Equal(1))
Expect(storageNodes[0]).To(Equal(eth.StorageNodeWithStateKeyModel{
CID: "mockStorageCID",
CID: mocks.StorageCID.String(),
Leaf: true,
StorageKey: "0x0000000000000000000000000000000000000000000000000000000000000001",
StateKey: mocks.ContractLeafKey.Hex(),
pkg/super_node/eth/ipld_fetcher.go
+45 -26
View File
@@ -90,7 +90,7 @@ func (f *IPLDFetcher) Fetch(cids shared.CIDsForFetching) (shared.IPLDs, error) {
// FetchHeaders fetches headers
// It uses the f.fetchBatch method
func (f *IPLDFetcher) FetchHeaders(cids []HeaderModel) ([][]byte, error) {
func (f *IPLDFetcher) FetchHeaders(cids []HeaderModel) ([]ipfs.BlockModel, error) {
log.Debug("fetching header iplds")
headerCids := make([]cid.Cid, len(cids))
for i, c := range cids {
@@ -101,20 +101,23 @@ func (f *IPLDFetcher) FetchHeaders(cids []HeaderModel) ([][]byte, error) {
headerCids[i] = dc
}
headers := f.fetchBatch(headerCids)
headersRLP := make([][]byte, len(headers))
headerIPLDs := make([]ipfs.BlockModel, len(headers))
for i, header := range headers {
headersRLP[i] = header.RawData()
headerIPLDs[i] = ipfs.BlockModel{
Data: header.RawData(),
CID: header.Cid().String(),
}
}
if len(headersRLP) != len(headerCids) {
if len(headerIPLDs) != len(headerCids) {
log.Errorf("ipfs fetcher: number of header blocks returned (%d) does not match number expected (%d)", len(headers), len(headerCids))
return headersRLP, errUnexpectedNumberOfIPLDs
return headerIPLDs, errUnexpectedNumberOfIPLDs
}
return headersRLP, nil
return headerIPLDs, nil
}
// FetchUncles fetches uncles
// It uses the f.fetchBatch method
func (f *IPLDFetcher) FetchUncles(cids []UncleModel) ([][]byte, error) {
func (f *IPLDFetcher) FetchUncles(cids []UncleModel) ([]ipfs.BlockModel, error) {
log.Debug("fetching uncle iplds")
uncleCids := make([]cid.Cid, len(cids))
for i, c := range cids {
@@ -125,20 +128,23 @@ func (f *IPLDFetcher) FetchUncles(cids []UncleModel) ([][]byte, error) {
uncleCids[i] = dc
}
uncles := f.fetchBatch(uncleCids)
unclesRLP := make([][]byte, len(uncles))
uncleIPLDs := make([]ipfs.BlockModel, len(uncles))
for i, uncle := range uncles {
unclesRLP[i] = uncle.RawData()
uncleIPLDs[i] = ipfs.BlockModel{
Data: uncle.RawData(),
CID: uncle.Cid().String(),
}
}
if len(unclesRLP) != len(uncleCids) {
if len(uncleIPLDs) != len(uncleCids) {
log.Errorf("ipfs fetcher: number of uncle blocks returned (%d) does not match number expected (%d)", len(uncles), len(uncleCids))
return unclesRLP, errUnexpectedNumberOfIPLDs
return uncleIPLDs, errUnexpectedNumberOfIPLDs
}
return unclesRLP, nil
return uncleIPLDs, nil
}
// FetchTrxs fetches transactions
// It uses the f.fetchBatch method
func (f *IPLDFetcher) FetchTrxs(cids []TxModel) ([][]byte, error) {
func (f *IPLDFetcher) FetchTrxs(cids []TxModel) ([]ipfs.BlockModel, error) {
log.Debug("fetching transaction iplds")
trxCids := make([]cid.Cid, len(cids))
for i, c := range cids {
@@ -149,20 +155,24 @@ func (f *IPLDFetcher) FetchTrxs(cids []TxModel) ([][]byte, error) {
trxCids[i] = dc
}
trxs := f.fetchBatch(trxCids)
trxsRLP := make([][]byte, len(trxs))
trxIPLDs := make([]ipfs.BlockModel, len(trxs))
for i, trx := range trxs {
trxsRLP[i] = trx.RawData()
trxIPLDs[i] = ipfs.BlockModel{
Data: trx.RawData(),
CID: trx.Cid().String(),
}
}
if len(trxsRLP) != len(trxCids) {
if len(trxIPLDs) != len(trxCids) {
log.Errorf("ipfs fetcher: number of transaction blocks returned (%d) does not match number expected (%d)", len(trxs), len(trxCids))
return trxsRLP, errUnexpectedNumberOfIPLDs
return trxIPLDs, errUnexpectedNumberOfIPLDs
}
return trxsRLP, nil
return trxIPLDs, nil
}
// FetchRcts fetches receipts
// It uses the f.fetchBatch method
func (f *IPLDFetcher) FetchRcts(cids []ReceiptModel) ([][]byte, error) {
// batch fetch preserves order?
func (f *IPLDFetcher) FetchRcts(cids []ReceiptModel) ([]ipfs.BlockModel, error) {
log.Debug("fetching receipt iplds")
rctCids := make([]cid.Cid, len(cids))
for i, c := range cids {
@@ -173,15 +183,18 @@ func (f *IPLDFetcher) FetchRcts(cids []ReceiptModel) ([][]byte, error) {
rctCids[i] = dc
}
rcts := f.fetchBatch(rctCids)
rctsRLP := make([][]byte, len(rcts))
rctIPLDs := make([]ipfs.BlockModel, len(rcts))
for i, rct := range rcts {
rctsRLP[i] = rct.RawData()
rctIPLDs[i] = ipfs.BlockModel{
Data: rct.RawData(),
CID: rct.Cid().String(),
}
}
if len(rctsRLP) != len(rctCids) {
if len(rctIPLDs) != len(rctCids) {
log.Errorf("ipfs fetcher: number of receipt blocks returned (%d) does not match number expected (%d)", len(rcts), len(rctCids))
return rctsRLP, errUnexpectedNumberOfIPLDs
return rctIPLDs, errUnexpectedNumberOfIPLDs
}
return rctsRLP, nil
return rctIPLDs, nil
}
// FetchState fetches state nodes
@@ -203,7 +216,10 @@ func (f *IPLDFetcher) FetchState(cids []StateNodeModel) ([]StateNode, error) {
return nil, err
}
stateNodes[i] = StateNode{
IPLD: state.RawData(),
IPLD: ipfs.BlockModel{
Data: state.RawData(),
CID: state.Cid().String(),
},
StateTrieKey: common.HexToHash(stateNode.StateKey),
Leaf: stateNode.Leaf,
}
@@ -230,7 +246,10 @@ func (f *IPLDFetcher) FetchStorage(cids []StorageNodeWithStateKeyModel) ([]Stora
return nil, err
}
storageNodes[i] = StorageNode{
IPLD: storage.RawData(),
IPLD: ipfs.BlockModel{
Data: storage.RawData(),
CID: storage.Cid().String(),
},
StateTrieKey: common.HexToHash(storageNode.StateKey),
StorageTrieKey: common.HexToHash(storageNode.StorageKey),
Leaf: storageNode.Leaf,
pkg/super_node/eth/ipld_fetcher_test.go
+30 -7
View File
@@ -20,6 +20,8 @@ import (
"bytes"
"math/big"
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
"github.com/ethereum/go-ethereum/common"
"github.com/ipfs/go-block-format"
. "github.com/onsi/ginkgo"
@@ -106,26 +108,47 @@ var _ = Describe("Fetcher", func() {
Expect(ok).To(BeTrue())
Expect(iplds.BlockNumber).To(Equal(mockCIDWrapper.BlockNumber))
Expect(len(iplds.Headers)).To(Equal(1))
Expect(iplds.Headers[0]).To(Equal(mockHeaderBlock.RawData()))
Expect(iplds.Headers[0]).To(Equal(ipfs.BlockModel{
Data: mockHeaderBlock.RawData(),
CID: mockHeaderBlock.Cid().String(),
}))
Expect(len(iplds.Uncles)).To(Equal(1))
Expect(iplds.Uncles[0]).To(Equal(mockUncleBlock.RawData()))
Expect(iplds.Uncles[0]).To(Equal(ipfs.BlockModel{
Data: mockUncleBlock.RawData(),
CID: mockUncleBlock.Cid().String(),
}))
Expect(len(iplds.Transactions)).To(Equal(1))
Expect(iplds.Transactions[0]).To(Equal(mockTrxBlock.RawData()))
Expect(iplds.Transactions[0]).To(Equal(ipfs.BlockModel{
Data: mockTrxBlock.RawData(),
CID: mockTrxBlock.Cid().String(),
}))
Expect(len(iplds.Receipts)).To(Equal(1))
Expect(iplds.Receipts[0]).To(Equal(mockReceiptBlock.RawData()))
Expect(iplds.Receipts[0]).To(Equal(ipfs.BlockModel{
Data: mockReceiptBlock.RawData(),
CID: mockReceiptBlock.Cid().String(),
}))
Expect(len(iplds.StateNodes)).To(Equal(1))
Expect(iplds.StateNodes[0].StateTrieKey).To(Equal(common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")))
Expect(iplds.StateNodes[0].Leaf).To(BeTrue())
Expect(iplds.StateNodes[0].IPLD).To(Equal(mockStateBlock.RawData()))
Expect(iplds.StateNodes[0].IPLD).To(Equal(ipfs.BlockModel{
Data: mockStateBlock.RawData(),
CID: mockStateBlock.Cid().String(),
}))
Expect(len(iplds.StorageNodes)).To(Equal(2))
for _, storage := range iplds.StorageNodes {
Expect(storage.Leaf).To(BeTrue())
Expect(storage.StateTrieKey).To(Equal(common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")))
if bytes.Equal(storage.StorageTrieKey.Bytes(), common.HexToHash("0000000000000000000000000000000000000000000000000000000000000001").Bytes()) {
Expect(storage.IPLD).To(Equal(mockStorageBlock1.RawData()))
Expect(storage.IPLD).To(Equal(ipfs.BlockModel{
Data: mockStorageBlock1.RawData(),
CID: mockStorageBlock1.Cid().String(),
}))
}
if bytes.Equal(storage.StorageTrieKey.Bytes(), common.HexToHash("0000000000000000000000000000000000000000000000000000000000000002").Bytes()) {
Expect(storage.IPLD).To(Equal(mockStorageBlock2.RawData()))
Expect(storage.IPLD).To(Equal(ipfs.BlockModel{
Data: mockStorageBlock2.RawData(),
CID: mockStorageBlock2.Cid().String(),
}))
}
}
})
pkg/super_node/eth/mocks/test_data.go
+85 -52
View File
@@ -23,6 +23,8 @@ import (
"math/big"
rand2 "math/rand"
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
"github.com/multiformats/go-multihash"
"github.com/vulcanize/vulcanizedb/pkg/ipfs/ipld"
@@ -50,6 +52,8 @@ var (
Root: common.HexToHash("0x0"),
TxHash: common.HexToHash("0x0"),
ReceiptHash: common.HexToHash("0x0"),
Difficulty: big.NewInt(5000000),
Extra: []byte{},
}
MockTransactions, MockReceipts, senderAddr = createTransactionsAndReceipts()
ReceiptsRlp, _ = rlp.EncodeToBytes(MockReceipts)
@@ -62,7 +66,24 @@ var (
mockTopic12 = common.HexToHash("0x06")
mockTopic21 = common.HexToHash("0x05")
mockTopic22 = common.HexToHash("0x07")
MockTrxMeta = []eth.TxModel{
MockLog1 = &types.Log{
Topics: []common.Hash{mockTopic11, mockTopic12},
Data: []byte{},
}
MockLog2 = &types.Log{
Topics: []common.Hash{mockTopic21, mockTopic22},
Data: []byte{},
}
HeaderCID, _ = ipld.RawdataToCid(ipld.MEthHeader, MockHeaderRlp, multihash.KECCAK_256)
Trx1CID, _ = ipld.RawdataToCid(ipld.MEthTx, MockTransactions.GetRlp(0), multihash.KECCAK_256)
Trx2CID, _ = ipld.RawdataToCid(ipld.MEthTx, MockTransactions.GetRlp(1), multihash.KECCAK_256)
Rct1CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, MockReceipts.GetRlp(0), multihash.KECCAK_256)
Rct2CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, MockReceipts.GetRlp(1), multihash.KECCAK_256)
State1CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, ValueBytes, multihash.KECCAK_256)
State2CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, AnotherValueBytes, multihash.KECCAK_256)
StorageCID, _ = ipld.RawdataToCid(ipld.MEthStorageTrie, StorageValue, multihash.KECCAK_256)
MockTrxMeta = []eth.TxModel{
{
CID: "", // This is empty until we go to publish to ipfs
Src: senderAddr.Hex(),
@@ -80,14 +101,14 @@ var (
}
MockTrxMetaPostPublsh = []eth.TxModel{
{
CID: "mockTrxCID1", // This is empty until we go to publish to ipfs
CID: Trx1CID.String(), // This is empty until we go to publish to ipfs
Src: senderAddr.Hex(),
Dst: Address.String(),
Index: 0,
TxHash: MockTransactions[0].Hash().String(),
},
{
CID: "mockTrxCID2",
CID: Trx2CID.String(),
Src: senderAddr.Hex(),
Dst: AnotherAddress.String(),
Index: 1,
@@ -118,7 +139,7 @@ var (
}
MockRctMetaPostPublish = []eth.ReceiptModel{
{
CID: "mockRctCID1",
CID: Rct1CID.String(),
Topic0s: []string{
mockTopic11.String(),
},
@@ -128,7 +149,7 @@ var (
Contract: Address.String(),
},
{
CID: "mockRctCID2",
CID: Rct2CID.String(),
Topic0s: []string{
mockTopic21.String(),
},
@@ -208,12 +229,12 @@ var (
}
MockStateMetaPostPublish = []eth.StateNodeModel{
{
CID: "mockStateCID1",
CID: State1CID.String(),
Leaf: true,
StateKey: ContractLeafKey.String(),
},
{
CID: "mockStateCID2",
CID: State2CID.String(),
Leaf: true,
StateKey: AnotherContractLeafKey.String(),
},
@@ -233,11 +254,11 @@ var (
BlockRlp: MockBlockRlp,
StateDiffRlp: MockStateDiffBytes,
ReceiptsRlp: ReceiptsRlp,
TotalDifficulty: big.NewInt(1337),
TotalDifficulty: MockBlock.Difficulty(),
}
MockConvertedPayload = eth.ConvertedPayload{
TotalDifficulty: big.NewInt(1337),
TotalDifficulty: MockBlock.Difficulty(),
Block: MockBlock,
Receipts: MockReceipts,
TxMetaData: MockTrxMeta,
@@ -250,9 +271,10 @@ var (
HeaderCID: eth2.HeaderModel{
BlockHash: MockBlock.Hash().String(),
BlockNumber: MockBlock.Number().String(),
CID: "mockHeaderCID",
CID: HeaderCID.String(),
ParentHash: MockBlock.ParentHash().String(),
TotalDifficulty: "1337",
TotalDifficulty: MockBlock.Difficulty().String(),
Reward: "5000000000000000000",
},
UncleCIDs: []eth2.UncleModel{},
TransactionCIDs: MockTrxMetaPostPublsh,
@@ -264,7 +286,7 @@ var (
StorageNodeCIDs: map[common.Hash][]eth.StorageNodeModel{
ContractLeafKey: {
{
CID: "mockStorageCID",
CID: StorageCID.String(),
StorageKey: "0x0000000000000000000000000000000000000000000000000000000000000001",
Leaf: true,
},
@@ -279,8 +301,9 @@ var (
BlockNumber: "1",
BlockHash: MockBlock.Hash().String(),
ParentHash: "0x0000000000000000000000000000000000000000000000000000000000000000",
CID: "mockHeaderCID",
TotalDifficulty: "1337",
CID: HeaderCID.String(),
TotalDifficulty: MockBlock.Difficulty().String(),
Reward: "5000000000000000000",
},
},
Transactions: MockTrxMetaPostPublsh,
@@ -289,54 +312,67 @@ var (
StateNodes: MockStateMetaPostPublish,
StorageNodes: []eth.StorageNodeWithStateKeyModel{
{
CID: "mockStorageCID",
CID: StorageCID.String(),
Leaf: true,
StateKey: ContractLeafKey.Hex(),
StorageKey: "0x0000000000000000000000000000000000000000000000000000000000000001",
},
},
}
headerCID, _ = ipld.RawdataToCid(ipld.MEthHeader, MockHeaderRlp, multihash.KECCAK_256)
trx1CID, _ = ipld.RawdataToCid(ipld.MEthTx, MockTransactions.GetRlp(0), multihash.KECCAK_256)
trx2CID, _ = ipld.RawdataToCid(ipld.MEthTx, MockTransactions.GetRlp(1), multihash.KECCAK_256)
rct1CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, MockReceipts.GetRlp(0), multihash.KECCAK_256)
rct2CID, _ = ipld.RawdataToCid(ipld.MEthTxReceipt, MockReceipts.GetRlp(1), multihash.KECCAK_256)
state1CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, ValueBytes, multihash.KECCAK_256)
state2CID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, AnotherValueBytes, multihash.KECCAK_256)
storageCID, _ = ipld.RawdataToCid(ipld.MEthStorageTrie, StorageValue, multihash.KECCAK_256)
HeaderIPLD, _ = blocks.NewBlockWithCid(MockHeaderRlp, headerCID)
Trx1IPLD, _ = blocks.NewBlockWithCid(MockTransactions.GetRlp(0), trx1CID)
Trx2IPLD, _ = blocks.NewBlockWithCid(MockTransactions.GetRlp(1), trx2CID)
Rct1IPLD, _ = blocks.NewBlockWithCid(MockReceipts.GetRlp(0), rct1CID)
Rct2IPLD, _ = blocks.NewBlockWithCid(MockReceipts.GetRlp(1), rct2CID)
State1IPLD, _ = blocks.NewBlockWithCid(ValueBytes, state1CID)
State2IPLD, _ = blocks.NewBlockWithCid(AnotherValueBytes, state2CID)
StorageIPLD, _ = blocks.NewBlockWithCid(StorageValue, storageCID)
HeaderIPLD, _ = blocks.NewBlockWithCid(MockHeaderRlp, HeaderCID)
Trx1IPLD, _ = blocks.NewBlockWithCid(MockTransactions.GetRlp(0), Trx1CID)
Trx2IPLD, _ = blocks.NewBlockWithCid(MockTransactions.GetRlp(1), Trx2CID)
Rct1IPLD, _ = blocks.NewBlockWithCid(MockReceipts.GetRlp(0), Rct1CID)
Rct2IPLD, _ = blocks.NewBlockWithCid(MockReceipts.GetRlp(1), Rct2CID)
State1IPLD, _ = blocks.NewBlockWithCid(ValueBytes, State1CID)
State2IPLD, _ = blocks.NewBlockWithCid(AnotherValueBytes, State2CID)
StorageIPLD, _ = blocks.NewBlockWithCid(StorageValue, StorageCID)
MockIPLDs = eth.IPLDs{
BlockNumber: big.NewInt(1),
Headers: [][]byte{
HeaderIPLD.RawData(),
Headers: []ipfs.BlockModel{
{
Data: HeaderIPLD.RawData(),
CID: HeaderIPLD.Cid().String(),
},
},
Transactions: [][]byte{
Trx1IPLD.RawData(),
Trx2IPLD.RawData(),
Transactions: []ipfs.BlockModel{
{
Data: Trx1IPLD.RawData(),
CID: Trx1IPLD.Cid().String(),
},
{
Data: Trx2IPLD.RawData(),
CID: Trx2IPLD.Cid().String(),
},
},
Receipts: [][]byte{
Rct1IPLD.RawData(),
Rct2IPLD.RawData(),
Receipts: []ipfs.BlockModel{
{
Data: Rct1IPLD.RawData(),
CID: Rct1IPLD.Cid().String(),
},
{
Data: Rct2IPLD.RawData(),
CID: Rct2IPLD.Cid().String(),
},
},
StateNodes: []eth2.StateNode{
{
StateTrieKey: ContractLeafKey,
Leaf: true,
IPLD: State1IPLD.RawData(),
IPLD: ipfs.BlockModel{
Data: State1IPLD.RawData(),
CID: State1IPLD.Cid().String(),
},
},
{
StateTrieKey: AnotherContractLeafKey,
Leaf: true,
IPLD: State2IPLD.RawData(),
IPLD: ipfs.BlockModel{
Data: State2IPLD.RawData(),
CID: State2IPLD.Cid().String(),
},
},
},
StorageNodes: []eth2.StorageNode{
@@ -344,7 +380,10 @@ var (
StateTrieKey: ContractLeafKey,
StorageTrieKey: common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000001"),
Leaf: true,
IPLD: StorageIPLD.RawData(),
IPLD: ipfs.BlockModel{
Data: StorageIPLD.RawData(),
CID: StorageIPLD.Cid().String(),
},
},
},
}
@@ -353,8 +392,8 @@ var (
// createTransactionsAndReceipts is a helper function to generate signed mock transactions and mock receipts with mock logs
func createTransactionsAndReceipts() (types.Transactions, types.Receipts, common.Address) {
// make transactions
trx1 := types.NewTransaction(0, Address, big.NewInt(1000), 50, big.NewInt(100), nil)
trx2 := types.NewTransaction(1, AnotherAddress, big.NewInt(2000), 100, big.NewInt(200), nil)
trx1 := types.NewTransaction(0, Address, big.NewInt(1000), 50, big.NewInt(100), []byte{})
trx2 := types.NewTransaction(1, AnotherAddress, big.NewInt(2000), 100, big.NewInt(200), []byte{})
transactionSigner := types.MakeSigner(params.MainnetChainConfig, BlockNumber)
mockCurve := elliptic.P256()
mockPrvKey, err := ecdsa.GenerateKey(mockCurve, rand.Reader)
@@ -375,16 +414,10 @@ func createTransactionsAndReceipts() (types.Transactions, types.Receipts, common
}
// make receipts
mockReceipt1 := types.NewReceipt(common.HexToHash("0x0").Bytes(), false, 50)
mockLog1 := &types.Log{
Topics: []common.Hash{mockTopic11, mockTopic12},
}
mockReceipt1.Logs = []*types.Log{mockLog1}
mockReceipt1.Logs = []*types.Log{MockLog1}
mockReceipt1.TxHash = signedTrx1.Hash()
mockReceipt2 := types.NewReceipt(common.HexToHash("0x1").Bytes(), false, 100)
mockLog2 := &types.Log{
Topics: []common.Hash{mockTopic21, mockTopic22},
}
mockReceipt2.Logs = []*types.Log{mockLog2}
mockReceipt2.Logs = []*types.Log{MockLog2}
mockReceipt2.TxHash = signedTrx2.Hash()
return types.Transactions{signedTrx1, signedTrx2}, types.Receipts{mockReceipt1, mockReceipt2}, senderAddr
}
pkg/super_node/eth/models.go
+2
View File
@@ -27,6 +27,7 @@ type HeaderModel struct {
CID string `db:"cid"`
TotalDifficulty string `db:"td"`
NodeID int64 `db:"node_id"`
Reward string `db:"reward"`
}
// UncleModel is the db model for eth.uncle_cids
@@ -36,6 +37,7 @@ type UncleModel struct {
BlockHash string `db:"block_hash"`
ParentHash string `db:"parent_hash"`
CID string `db:"cid"`
Reward string `db:"reward"`
}
// TxModel is the db model for eth.transaction_cids
pkg/super_node/eth/publisher.go
+7 -2
View File
@@ -20,12 +20,13 @@ import (
"errors"
"fmt"
"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
common2 "github.com/vulcanize/vulcanizedb/pkg/eth/converters/common"
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
"github.com/vulcanize/vulcanizedb/pkg/ipfs/dag_putters"
"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
)
// IPLDPublisher satisfies the IPLDPublisher for ethereum
@@ -63,12 +64,14 @@ func (pub *IPLDPublisher) Publish(payload shared.ConvertedData) (shared.CIDsForI
if err != nil {
return nil, err
}
reward := common2.CalcEthBlockReward(ipldPayload.Block, ipldPayload.Receipts)
header := HeaderModel{
CID: headerCid,
ParentHash: ipldPayload.Block.ParentHash().String(),
BlockNumber: ipldPayload.Block.Number().String(),
BlockHash: ipldPayload.Block.Hash().String(),
TotalDifficulty: ipldPayload.TotalDifficulty.String(),
Reward: reward.String(),
}
// Process and publish uncles
@@ -78,10 +81,12 @@ func (pub *IPLDPublisher) Publish(payload shared.ConvertedData) (shared.CIDsForI
if err != nil {
return nil, err
}
uncleReward := common2.CalcUncleMinerReward(ipldPayload.Block.Number().Int64(), uncle.Number.Int64())
uncleCids = append(uncleCids, UncleModel{
CID: uncleCid,
ParentHash: uncle.ParentHash.String(),
BlockHash: uncle.Hash().String(),
Reward: uncleReward.String(),
})
}
pkg/super_node/eth/publisher_test.go
+7 -6
View File
@@ -45,16 +45,16 @@ var _ = Describe("Publisher", func() {
Describe("Publish", func() {
It("Publishes the passed IPLDPayload objects to IPFS and returns a CIDPayload for indexing", func() {
mockHeaderDagPutter.CIDsToReturn = []string{"mockHeaderCID"}
mockTrxDagPutter.CIDsToReturn = []string{"mockTrxCID1", "mockTrxCID2"}
mockRctDagPutter.CIDsToReturn = []string{"mockRctCID1", "mockRctCID2"}
mockHeaderDagPutter.CIDsToReturn = []string{mocks.HeaderCID.String()}
mockTrxDagPutter.CIDsToReturn = []string{mocks.Trx1CID.String(), mocks.Trx2CID.String()}
mockRctDagPutter.CIDsToReturn = []string{mocks.Rct1CID.String(), mocks.Rct2CID.String()}
val1 := common.BytesToHash(mocks.MockConvertedPayload.StateNodes[0].Value)
val2 := common.BytesToHash(mocks.MockConvertedPayload.StateNodes[1].Value)
mockStateDagPutter.CIDsToReturn = map[common.Hash][]string{
val1: {"mockStateCID1"},
val2: {"mockStateCID2"},
val1: {mocks.State1CID.String()},
val2: {mocks.State2CID.String()},
}
mockStorageDagPutter.CIDsToReturn = []string{"mockStorageCID"}
mockStorageDagPutter.CIDsToReturn = []string{mocks.StorageCID.String()}
publisher := eth.IPLDPublisher{
HeaderPutter: mockHeaderDagPutter,
TransactionPutter: mockTrxDagPutter,
@@ -69,6 +69,7 @@ var _ = Describe("Publisher", func() {
Expect(cidPayload.HeaderCID.TotalDifficulty).To(Equal(mocks.MockConvertedPayload.TotalDifficulty.String()))
Expect(cidPayload.HeaderCID.BlockNumber).To(Equal(mocks.MockCIDPayload.HeaderCID.BlockNumber))
Expect(cidPayload.HeaderCID.BlockHash).To(Equal(mocks.MockCIDPayload.HeaderCID.BlockHash))
Expect(cidPayload.HeaderCID.Reward).To(Equal(mocks.MockCIDPayload.HeaderCID.Reward))
Expect(cidPayload.UncleCIDs).To(Equal(mocks.MockCIDPayload.UncleCIDs))
Expect(cidPayload.HeaderCID).To(Equal(mocks.MockCIDPayload.HeaderCID))
Expect(len(cidPayload.TransactionCIDs)).To(Equal(2))
pkg/super_node/eth/retriever.go
+22 -5
View File
@@ -193,6 +193,7 @@ func (ecr *CIDRetriever) RetrieveTxCIDs(tx *sqlx.Tx, txFilter TxFilter, blockNum
pgStr += fmt.Sprintf(` AND transaction_cids.src = ANY($%d::VARCHAR(66)[])`, id)
args = append(args, pq.Array(txFilter.Src))
}
pgStr += ` ORDER BY transaction_cids.index`
return results, tx.Select(&results, pgStr, args...)
}
@@ -224,7 +225,7 @@ func (ecr *CIDRetriever) RetrieveRctCIDs(tx *sqlx.Tx, rctFilter ReceiptFilter, b
args = append(args, pq.Array(rctFilter.Contracts))
id++
// Filter on topics if there are any
if len(rctFilter.Topics) > 0 {
if hasTopics(rctFilter.Topics) {
pgStr += " AND ("
first := true
for i, topicSet := range rctFilter.Topics {
@@ -250,7 +251,7 @@ func (ecr *CIDRetriever) RetrieveRctCIDs(tx *sqlx.Tx, rctFilter ReceiptFilter, b
pgStr += ")"
} else { // If there are no contract addresses to filter on
// Filter on topics if there are any
if len(rctFilter.Topics) > 0 {
if hasTopics(rctFilter.Topics) {
pgStr += " AND (("
first := true
for i, topicSet := range rctFilter.Topics {
@@ -279,10 +280,20 @@ func (ecr *CIDRetriever) RetrieveRctCIDs(tx *sqlx.Tx, rctFilter ReceiptFilter, b
args = append(args, pq.Array(trxIds))
}
}
pgStr += ` ORDER BY transaction_cids.index`
receiptCids := make([]ReceiptModel, 0)
return receiptCids, tx.Select(&receiptCids, pgStr, args...)
}
func hasTopics(topics [][]string) bool {
for _, topicSet := range topics {
if len(topicSet) > 0 {
return true
}
}
return false
}
// RetrieveStateCIDs retrieves and returns all of the state node cids at the provided blockheight that conform to the provided filter parameters
func (ecr *CIDRetriever) RetrieveStateCIDs(tx *sqlx.Tx, stateFilter StateFilter, blockNumber int64) ([]StateNodeModel, error) {
log.Debug("retrieving state cids for block ", blockNumber)
@@ -474,7 +485,8 @@ func (ecr *CIDRetriever) RetrieveHeaderCIDByHash(tx *sqlx.Tx, blockHash common.H
func (ecr *CIDRetriever) RetrieveTxCIDsByHeaderID(tx *sqlx.Tx, headerID int64) ([]TxModel, error) {
log.Debug("retrieving tx cids for block id ", headerID)
pgStr := `SELECT * FROM eth.transaction_cids
WHERE header_id = $1`
WHERE header_id = $1
ORDER BY index`
var txCIDs []TxModel
return txCIDs, tx.Select(&txCIDs, pgStr, headerID)
}
@@ -482,8 +494,13 @@ func (ecr *CIDRetriever) RetrieveTxCIDsByHeaderID(tx *sqlx.Tx, headerID int64) (
// RetrieveReceiptCIDsByTxIDs retrieves receipt CIDs by their associated tx IDs
func (ecr *CIDRetriever) RetrieveReceiptCIDsByTxIDs(tx *sqlx.Tx, txIDs []int64) ([]ReceiptModel, error) {
log.Debugf("retrieving receipt cids for tx ids %v", txIDs)
pgStr := `SELECT * FROM eth.receipt_cids
WHERE tx_id = ANY($1::INTEGER[])`
pgStr := `SELECT receipt_cids.id, receipt_cids.tx_id, receipt_cids.cid,
receipt_cids.contract, receipt_cids.topic0s, receipt_cids.topic1s,
receipt_cids.topic2s, receipt_cids.topic3s
FROM eth.receipt_cids, eth.transaction_cids
WHERE tx_id = ANY($1::INTEGER[])
AND receipt_cids.tx_id = transaction_cids.id
ORDER BY transaction_cids.index`
var rctCIDs []ReceiptModel
return rctCIDs, tx.Select(&rctCIDs, pgStr, pq.Array(txIDs))
}
pkg/super_node/eth/retriever_test.go
+7 -7
View File
@@ -246,11 +246,11 @@ var _ = Describe("Retriever", func() {
Expect(eth.ReceiptModelsContainsCID(cidWrapper.Receipts, mocks.MockCIDWrapper.Receipts[1].CID)).To(BeTrue())
Expect(len(cidWrapper.StateNodes)).To(Equal(2))
for _, stateNode := range cidWrapper.StateNodes {
if stateNode.CID == "mockStateCID1" {
if stateNode.CID == mocks.State1CID.String() {
Expect(stateNode.StateKey).To(Equal(mocks.ContractLeafKey.Hex()))
Expect(stateNode.Leaf).To(Equal(true))
}
if stateNode.CID == "mockStateCID2" {
if stateNode.CID == mocks.State2CID.String() {
Expect(stateNode.StateKey).To(Equal(mocks.AnotherContractLeafKey.Hex()))
Expect(stateNode.Leaf).To(Equal(true))
}
@@ -335,13 +335,13 @@ var _ = Describe("Retriever", func() {
Expect(cidWrapper5.BlockNumber).To(Equal(mocks.MockCIDWrapper.BlockNumber))
Expect(len(cidWrapper5.Headers)).To(Equal(0))
Expect(len(cidWrapper5.Transactions)).To(Equal(2))
Expect(eth.TxModelsContainsCID(cidWrapper5.Transactions, "mockTrxCID1")).To(BeTrue())
Expect(eth.TxModelsContainsCID(cidWrapper5.Transactions, "mockTrxCID2")).To(BeTrue())
Expect(eth.TxModelsContainsCID(cidWrapper5.Transactions, mocks.Trx1CID.String())).To(BeTrue())
Expect(eth.TxModelsContainsCID(cidWrapper5.Transactions, mocks.Trx2CID.String())).To(BeTrue())
Expect(len(cidWrapper5.StateNodes)).To(Equal(0))
Expect(len(cidWrapper5.StorageNodes)).To(Equal(0))
Expect(len(cidWrapper5.Receipts)).To(Equal(2))
Expect(eth.ReceiptModelsContainsCID(cidWrapper5.Receipts, "mockRctCID1")).To(BeTrue())
Expect(eth.ReceiptModelsContainsCID(cidWrapper5.Receipts, "mockRctCID2")).To(BeTrue())
Expect(eth.ReceiptModelsContainsCID(cidWrapper5.Receipts, mocks.Rct1CID.String())).To(BeTrue())
Expect(eth.ReceiptModelsContainsCID(cidWrapper5.Receipts, mocks.Rct2CID.String())).To(BeTrue())
cids6, empty, err := retriever.Retrieve(rctsForSelectCollectedTrxs, 1)
Expect(err).ToNot(HaveOccurred())
@@ -379,7 +379,7 @@ var _ = Describe("Retriever", func() {
HeaderID: cidWrapper7.StateNodes[0].HeaderID,
Leaf: true,
StateKey: mocks.ContractLeafKey.Hex(),
CID: "mockStateCID1",
CID: mocks.State1CID.String(),
}))
_, empty, err = retriever.Retrieve(rctTopicsAndContractFilterFail, 1)
pkg/super_node/eth/types.go
+8 -6
View File
@@ -19,6 +19,8 @@ package eth
import (
"math/big"
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
)
@@ -77,10 +79,10 @@ type CIDWrapper struct {
// Returned by IPLDFetcher and ResponseFilterer
type IPLDs struct {
BlockNumber *big.Int
Headers [][]byte
Uncles [][]byte
Transactions [][]byte
Receipts [][]byte
Headers []ipfs.BlockModel
Uncles []ipfs.BlockModel
Transactions []ipfs.BlockModel
Receipts []ipfs.BlockModel
StateNodes []StateNode
StorageNodes []StorageNode
}
@@ -92,13 +94,13 @@ func (i IPLDs) Height() int64 {
type StateNode struct {
StateTrieKey common.Hash
IPLD []byte
IPLD ipfs.BlockModel
Leaf bool
}
type StorageNode struct {
StateTrieKey common.Hash
StorageTrieKey common.Hash
IPLD []byte
IPLD ipfs.BlockModel
Leaf bool
}