refactor eth parser to not process rct, tx, and log tries
This commit is contained in:
i-norden
parent
b568bb49e9
commit
f05d6b6baa
@ -104,16 +104,13 @@ func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receip
|
||||
}
|
||||
|
||||
// Generate the block iplds
|
||||
headerNode, txNodes, txTrieNodes, rctNodes, rctTrieNodes, logTrieNodes, logLeafNodeCIDs, rctLeafNodeCIDs, err := ipld2.FromBlockAndReceipts(block, receipts)
|
||||
headerNode, txNodes, rctNodes, logNodes, err := ipld2.FromBlockAndReceipts(block, receipts)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error creating IPLD nodes from block and receipts: %v", err)
|
||||
}
|
||||
|
||||
if len(txNodes) != len(rctNodes) || len(rctNodes) != len(rctLeafNodeCIDs) {
|
||||
return nil, fmt.Errorf("expected number of transactions (%d), receipts (%d), and receipt trie leaf nodes (%d) to be equal", len(txNodes), len(rctNodes), len(rctLeafNodeCIDs))
|
||||
}
|
||||
if len(txTrieNodes) != len(rctTrieNodes) {
|
||||
return nil, fmt.Errorf("expected number of tx trie (%d) and rct trie (%d) nodes to be equal", len(txTrieNodes), len(rctTrieNodes))
|
||||
if len(txNodes) != len(rctNodes) {
|
||||
return nil, fmt.Errorf("expected number of transactions (%d), receipts (%d)", len(txNodes), len(rctNodes))
|
||||
}
|
||||
|
||||
// Calculate reward
|
||||
|
||||
@ -19,15 +19,11 @@ package ipld
|
||||
import (
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
|
||||
"github.com/ipfs/go-cid"
|
||||
node "github.com/ipfs/go-ipld-format"
|
||||
"github.com/multiformats/go-multihash"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/ethereum/go-ethereum/rlp"
|
||||
)
|
||||
@ -35,11 +31,11 @@ import (
|
||||
// FromBlockRLP takes an RLP message representing
|
||||
// an ethereum block header or body (header, ommers and txs)
|
||||
// to return it as a set of IPLD nodes for further processing.
|
||||
func FromBlockRLP(r io.Reader) (*EthHeader, []*EthTx, []*EthTxTrie, error) {
|
||||
func FromBlockRLP(r io.Reader) (*EthHeader, []*EthTx, error) {
|
||||
// We may want to use this stream several times
|
||||
rawdata, err := ioutil.ReadAll(r)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
// Let's try to decode the received element as a block body
|
||||
@ -47,26 +43,26 @@ func FromBlockRLP(r io.Reader) (*EthHeader, []*EthTx, []*EthTxTrie, error) {
|
||||
err = rlp.Decode(bytes.NewBuffer(rawdata), &decodedBlock)
|
||||
if err != nil {
|
||||
if err.Error()[:41] != "rlp: expected input list for types.Header" {
|
||||
return nil, nil, nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
// Maybe it is just a header... (body sans ommers and txs)
|
||||
var decodedHeader types.Header
|
||||
err := rlp.Decode(bytes.NewBuffer(rawdata), &decodedHeader)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
c, err := RawdataToCid(MEthHeader, rawdata, multihash.KECCAK_256)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
// It was a header
|
||||
return &EthHeader{
|
||||
Header: &decodedHeader,
|
||||
cid: c,
|
||||
rawdata: rawdata,
|
||||
}, nil, nil, nil
|
||||
}, nil, nil
|
||||
}
|
||||
|
||||
// This is a block body (header + ommers + txs)
|
||||
@ -74,7 +70,7 @@ func FromBlockRLP(r io.Reader) (*EthHeader, []*EthTx, []*EthTxTrie, error) {
|
||||
headerRawData := getRLP(decodedBlock.Header())
|
||||
c, err := RawdataToCid(MEthHeader, headerRawData, multihash.KECCAK_256)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
ethBlock := &EthHeader{
|
||||
Header: decodedBlock.Header(),
|
||||
@ -83,29 +79,28 @@ func FromBlockRLP(r io.Reader) (*EthHeader, []*EthTx, []*EthTxTrie, error) {
|
||||
}
|
||||
|
||||
// Process the found eth-tx objects
|
||||
ethTxNodes, ethTxTrieNodes, err := processTransactions(decodedBlock.Transactions(),
|
||||
decodedBlock.Header().TxHash[:])
|
||||
ethTxNodes, err := processTransactions(decodedBlock.Transactions())
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
return ethBlock, ethTxNodes, ethTxTrieNodes, nil
|
||||
return ethBlock, ethTxNodes, nil
|
||||
}
|
||||
|
||||
// FromBlockJSON takes the output of an ethereum client JSON API
|
||||
// (i.e. parity or geth) and returns a set of IPLD nodes.
|
||||
func FromBlockJSON(r io.Reader) (*EthHeader, []*EthTx, []*EthTxTrie, error) {
|
||||
func FromBlockJSON(r io.Reader) (*EthHeader, []*EthTx, error) {
|
||||
var obj objJSONHeader
|
||||
dec := json.NewDecoder(r)
|
||||
err := dec.Decode(&obj)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
headerRawData := getRLP(&obj.Result.Header)
|
||||
c, err := RawdataToCid(MEthHeader, headerRawData, multihash.KECCAK_256)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
ethBlock := &EthHeader{
|
||||
Header: &obj.Result.Header,
|
||||
@ -114,179 +109,83 @@ func FromBlockJSON(r io.Reader) (*EthHeader, []*EthTx, []*EthTxTrie, error) {
|
||||
}
|
||||
|
||||
// Process the found eth-tx objects
|
||||
ethTxNodes, ethTxTrieNodes, err := processTransactions(obj.Result.Transactions,
|
||||
obj.Result.Header.TxHash[:])
|
||||
ethTxNodes, err := processTransactions(obj.Result.Transactions)
|
||||
if err != nil {
|
||||
return nil, nil, nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
return ethBlock, ethTxNodes, ethTxTrieNodes, nil
|
||||
return ethBlock, ethTxNodes, nil
|
||||
}
|
||||
|
||||
// FromBlockAndReceipts takes a block and processes it
|
||||
// to return it a set of IPLD nodes for further processing.
|
||||
func FromBlockAndReceipts(block *types.Block, receipts []*types.Receipt) (*EthHeader, []*EthTx, []*EthTxTrie, []*EthReceipt, []*EthRctTrie, [][]node.Node, [][]cid.Cid, []cid.Cid, error) {
|
||||
func FromBlockAndReceipts(block *types.Block, receipts []*types.Receipt) (*EthHeader, []*EthTx, []*EthReceipt, [][]*EthLog, error) {
|
||||
// Process the header
|
||||
headerNode, err := NewEthHeader(block.Header())
|
||||
if err != nil {
|
||||
return nil, nil, nil, nil, nil, nil, nil, nil, err
|
||||
return nil, nil, nil, nil, err
|
||||
}
|
||||
|
||||
// Process the txs
|
||||
txNodes, txTrieNodes, err := processTransactions(block.Transactions(),
|
||||
block.Header().TxHash[:])
|
||||
txNodes, err := processTransactions(block.Transactions())
|
||||
if err != nil {
|
||||
return nil, nil, nil, nil, nil, nil, nil, nil, err
|
||||
return nil, nil, nil, nil, err
|
||||
}
|
||||
|
||||
// Process the receipts and logs
|
||||
rctNodes, tctTrieNodes, logTrieAndLogNodes, logLeafNodeCIDs, rctLeafNodeCIDs, err := processReceiptsAndLogs(receipts,
|
||||
block.Header().ReceiptHash[:])
|
||||
rctNodes, logNodes, err := processReceiptsAndLogs(receipts, block.Header().ReceiptHash[:])
|
||||
|
||||
return headerNode, txNodes, txTrieNodes, rctNodes, tctTrieNodes, logTrieAndLogNodes, logLeafNodeCIDs, rctLeafNodeCIDs, err
|
||||
return headerNode, txNodes, rctNodes, logNodes, err
|
||||
}
|
||||
|
||||
// processTransactions will take the found transactions in a parsed block body
|
||||
// to return IPLD node slices for eth-tx and eth-tx-trie
|
||||
func processTransactions(txs []*types.Transaction, expectedTxRoot []byte) ([]*EthTx, []*EthTxTrie, error) {
|
||||
// to return IPLD node slices for eth-tx
|
||||
func processTransactions(txs []*types.Transaction) ([]*EthTx, error) {
|
||||
var ethTxNodes []*EthTx
|
||||
transactionTrie := newTxTrie()
|
||||
|
||||
for idx, tx := range txs {
|
||||
for _, tx := range txs {
|
||||
ethTx, err := NewEthTx(tx)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
return nil, err
|
||||
}
|
||||
ethTxNodes = append(ethTxNodes, ethTx)
|
||||
if err := transactionTrie.Add(idx, ethTx.RawData()); err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
}
|
||||
|
||||
if !bytes.Equal(transactionTrie.rootHash(), expectedTxRoot) {
|
||||
return nil, nil, fmt.Errorf("wrong transaction hash computed")
|
||||
}
|
||||
txTrieNodes, err := transactionTrie.getNodes()
|
||||
return ethTxNodes, txTrieNodes, err
|
||||
return ethTxNodes, nil
|
||||
}
|
||||
|
||||
// processReceiptsAndLogs will take in receipts
|
||||
// to return IPLD node slices for eth-rct, eth-rct-trie, eth-log, eth-log-trie, eth-log-trie-CID, eth-rct-trie-CID
|
||||
func processReceiptsAndLogs(rcts []*types.Receipt, expectedRctRoot []byte) ([]*EthReceipt, []*EthRctTrie, [][]node.Node, [][]cid.Cid, []cid.Cid, error) {
|
||||
// to return IPLD node slices for eth-rct and eth-log
|
||||
func processReceiptsAndLogs(rcts []*types.Receipt, expectedRctRoot []byte) ([]*EthReceipt, [][]*EthLog, error) {
|
||||
// Pre allocating memory.
|
||||
ethRctNodes := make([]*EthReceipt, 0, len(rcts))
|
||||
ethLogleafNodeCids := make([][]cid.Cid, 0, len(rcts))
|
||||
ethLogTrieAndLogNodes := make([][]node.Node, 0, len(rcts))
|
||||
|
||||
receiptTrie := NewRctTrie()
|
||||
ethRctNodes := make([]*EthReceipt, len(rcts))
|
||||
ethLogNodes := make([][]*EthLog, len(rcts))
|
||||
|
||||
for idx, rct := range rcts {
|
||||
// Process logs for each receipt.
|
||||
logTrieNodes, leafNodeCids, logTrieHash, err := processLogs(rct.Logs)
|
||||
logNodes, err := processLogs(rct.Logs)
|
||||
if err != nil {
|
||||
return nil, nil, nil, nil, nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
rct.LogRoot = logTrieHash
|
||||
ethLogTrieAndLogNodes = append(ethLogTrieAndLogNodes, logTrieNodes)
|
||||
ethLogleafNodeCids = append(ethLogleafNodeCids, leafNodeCids)
|
||||
|
||||
ethRct, err := NewReceipt(rct)
|
||||
if err != nil {
|
||||
return nil, nil, nil, nil, nil, err
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
ethRctNodes = append(ethRctNodes, ethRct)
|
||||
if err = receiptTrie.Add(idx, ethRct.RawData()); err != nil {
|
||||
return nil, nil, nil, nil, nil, err
|
||||
}
|
||||
ethRctNodes[idx] = ethRct
|
||||
ethLogNodes[idx] = logNodes
|
||||
}
|
||||
|
||||
if !bytes.Equal(receiptTrie.rootHash(), expectedRctRoot) {
|
||||
return nil, nil, nil, nil, nil, fmt.Errorf("wrong receipt hash computed")
|
||||
}
|
||||
|
||||
rctTrieNodes, err := receiptTrie.GetNodes()
|
||||
if err != nil {
|
||||
return nil, nil, nil, nil, nil, err
|
||||
}
|
||||
|
||||
rctLeafNodes, keys, err := receiptTrie.GetLeafNodes()
|
||||
if err != nil {
|
||||
return nil, nil, nil, nil, nil, err
|
||||
}
|
||||
|
||||
ethRctleafNodeCids := make([]cid.Cid, len(rctLeafNodes))
|
||||
for i, rln := range rctLeafNodes {
|
||||
var idx uint
|
||||
|
||||
r := bytes.NewReader(keys[i].TrieKey)
|
||||
err = rlp.Decode(r, &idx)
|
||||
if err != nil {
|
||||
return nil, nil, nil, nil, nil, err
|
||||
}
|
||||
ethRctleafNodeCids[idx] = rln.Cid()
|
||||
}
|
||||
|
||||
return ethRctNodes, rctTrieNodes, ethLogTrieAndLogNodes, ethLogleafNodeCids, ethRctleafNodeCids, err
|
||||
return ethRctNodes, ethLogNodes, nil
|
||||
}
|
||||
|
||||
const keccak256Length = 32
|
||||
|
||||
func processLogs(logs []*types.Log) ([]node.Node, []cid.Cid, common.Hash, error) {
|
||||
logTr := newLogTrie()
|
||||
shortLog := make(map[uint64]*EthLog, len(logs))
|
||||
func processLogs(logs []*types.Log) ([]*EthLog, error) {
|
||||
logNodes := make([]*EthLog, len(logs))
|
||||
for idx, log := range logs {
|
||||
logRaw, err := rlp.EncodeToBytes(log)
|
||||
logNode, err := NewLog(log)
|
||||
if err != nil {
|
||||
return nil, nil, common.Hash{}, err
|
||||
}
|
||||
// if len(logRaw) <= keccak256Length it is possible this value's "leaf node"
|
||||
// will be stored in its parent branch but only if len(partialPathOfTheNode) + len(logRaw) <= keccak256Length
|
||||
// But we can't tell what the partial path will be until the trie is Commit()-ed
|
||||
// So wait until we collect all the leaf nodes, and if we are missing any at the indexes we note in shortLogCIDs
|
||||
// we know that these "leaf nodes" were internalized into their parent branch node and we move forward with
|
||||
// using the cid.Cid we cached in shortLogCIDs
|
||||
if len(logRaw) <= keccak256Length {
|
||||
logNode, err := NewLog(log)
|
||||
if err != nil {
|
||||
return nil, nil, common.Hash{}, err
|
||||
}
|
||||
shortLog[uint64(idx)] = logNode
|
||||
}
|
||||
if err = logTr.Add(idx, logRaw); err != nil {
|
||||
return nil, nil, common.Hash{}, err
|
||||
return nil, err
|
||||
}
|
||||
logNodes[idx] = logNode
|
||||
}
|
||||
|
||||
logTrieNodes, err := logTr.getNodes()
|
||||
if err != nil {
|
||||
return nil, nil, common.Hash{}, err
|
||||
}
|
||||
|
||||
leafNodes, keys, err := logTr.getLeafNodes()
|
||||
if err != nil {
|
||||
return nil, nil, common.Hash{}, err
|
||||
}
|
||||
leafNodeCids := make([]cid.Cid, len(logs))
|
||||
for i, ln := range leafNodes {
|
||||
var idx uint
|
||||
|
||||
r := bytes.NewReader(keys[i].TrieKey)
|
||||
err = rlp.Decode(r, &idx)
|
||||
if err != nil {
|
||||
return nil, nil, common.Hash{}, err
|
||||
}
|
||||
leafNodeCids[idx] = ln.Cid()
|
||||
}
|
||||
// this is where we check which logs <= keccak256Length were actually internalized into parent branch node
|
||||
// and replace those that were with the cid.Cid for the raw log IPLD
|
||||
for i, l := range shortLog {
|
||||
if !leafNodeCids[i].Defined() {
|
||||
leafNodeCids[i] = l.Cid()
|
||||
// if the leaf node was internalized, we append an IPLD for log itself to the list of IPLDs we need to publish
|
||||
logTrieNodes = append(logTrieNodes, l)
|
||||
}
|
||||
}
|
||||
|
||||
return logTrieNodes, leafNodeCids, common.BytesToHash(logTr.rootHash()), err
|
||||
return logNodes, nil
|
||||
}
|
||||
|
||||
@ -92,7 +92,7 @@ func loadBlockData(t *testing.T) []testCase {
|
||||
func TestFromBlockAndReceipts(t *testing.T) {
|
||||
testCases := loadBlockData(t)
|
||||
for _, tc := range testCases {
|
||||
_, _, _, _, _, _, _, _, _, err := FromBlockAndReceipts(tc.block, tc.receipts)
|
||||
_, _, _, _, err := FromBlockAndReceipts(tc.block, tc.receipts)
|
||||
if err != nil {
|
||||
t.Fatalf("error generating IPLDs from block and receipts, err %v, kind %s, block hash %s", err, tc.kind, tc.block.Hash())
|
||||
}
|
||||
@ -101,8 +101,7 @@ func TestFromBlockAndReceipts(t *testing.T) {
|
||||
|
||||
func TestProcessLogs(t *testing.T) {
|
||||
logs := []*types.Log{mocks.MockLog1, mocks.MockLog2}
|
||||
nodes, cids, _, err := processLogs(logs)
|
||||
nodes, err := processLogs(logs)
|
||||
require.NoError(t, err)
|
||||
require.GreaterOrEqual(t, len(nodes), len(logs))
|
||||
require.Equal(t, len(logs), len(cids))
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user