cerc-io/ipld-eth-server
12
0
Fork 1
Code Issues 10 Pull Requests 2 Actions Packages Projects Releases 37 Wiki Activity

adjust btc and eth publishers and publisher tests

Browse Source
This commit is contained in:
Ian Norden
2020-03-17 13:05:19 -05:00
parent 7eddf396a5
commit d47ba24373
19 changed files with 334 additions and 220 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/super_node/eth/publisher.go
+85 -61
View File
@@ -17,7 +17,6 @@
package eth
import (
"errors"
"fmt"
"github.com/ethereum/go-ethereum/common"
@@ -26,16 +25,19 @@ import (
common2 "github.com/vulcanize/vulcanizedb/pkg/eth/converters/common"
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
"github.com/vulcanize/vulcanizedb/pkg/ipfs/dag_putters"
"github.com/vulcanize/vulcanizedb/pkg/ipfs/ipld"
"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
)
// IPLDPublisher satisfies the IPLDPublisher for ethereum
type IPLDPublisher struct {
HeaderPutter shared.DagPutter
TransactionPutter shared.DagPutter
ReceiptPutter shared.DagPutter
StatePutter shared.DagPutter
StoragePutter shared.DagPutter
HeaderPutter shared.DagPutter
TransactionPutter shared.DagPutter
TransactionTriePutter shared.DagPutter
ReceiptPutter shared.DagPutter
ReceiptTriePutter shared.DagPutter
StatePutter shared.DagPutter
StoragePutter shared.DagPutter
}
// NewIPLDPublisher creates a pointer to a new Publisher which satisfies the IPLDPublisher interface
@@ -45,11 +47,13 @@ func NewIPLDPublisher(ipfsPath string) (*IPLDPublisher, error) {
return nil, err
}
return &IPLDPublisher{
HeaderPutter: dag_putters.NewEthBlockHeaderDagPutter(node),
TransactionPutter: dag_putters.NewEthTxsDagPutter(node),
ReceiptPutter: dag_putters.NewEthReceiptDagPutter(node),
StatePutter: dag_putters.NewEthStateDagPutter(node),
StoragePutter: dag_putters.NewEthStorageDagPutter(node),
HeaderPutter: dag_putters.NewEthBlockHeaderDagPutter(node),
TransactionPutter: dag_putters.NewEthTxsDagPutter(node),
TransactionTriePutter: dag_putters.NewEthTxTrieDagPutter(node),
ReceiptPutter: dag_putters.NewEthReceiptDagPutter(node),
ReceiptTriePutter: dag_putters.NewEthRctTrieDagPutter(node),
StatePutter: dag_putters.NewEthStateDagPutter(node),
StoragePutter: dag_putters.NewEthStorageDagPutter(node),
}, nil
}
@@ -59,8 +63,14 @@ func (pub *IPLDPublisher) Publish(payload shared.ConvertedData) (shared.CIDsForI
if !ok {
return nil, fmt.Errorf("eth publisher expected payload type %T got %T", ConvertedPayload{}, payload)
}
// Generate the nodes for publishing
headerNode, uncleNodes, txNodes, txTrieNodes, rctNodes, rctTrieNodes, err := ipld.FromBlockAndReceipts(ipldPayload.Block, ipldPayload.Receipts)
if err != nil {
return nil, err
}
// Process and publish headers
headerCid, err := pub.publishHeader(ipldPayload.Block.Header())
headerCid, err := pub.publishHeader(headerNode)
if err != nil {
return nil, err
}
@@ -75,29 +85,29 @@ func (pub *IPLDPublisher) Publish(payload shared.ConvertedData) (shared.CIDsForI
}
// Process and publish uncles
uncleCids := make([]UncleModel, 0, len(ipldPayload.Block.Uncles()))
for _, uncle := range ipldPayload.Block.Uncles() {
uncleCids := make([]UncleModel, len(uncleNodes))
for i, uncle := range uncleNodes {
uncleCid, err := pub.publishHeader(uncle)
if err != nil {
return nil, err
}
uncleReward := common2.CalcUncleMinerReward(ipldPayload.Block.Number().Int64(), uncle.Number.Int64())
uncleCids = append(uncleCids, UncleModel{
uncleCids[i] = UncleModel{
CID: uncleCid,
ParentHash: uncle.ParentHash.String(),
BlockHash: uncle.Hash().String(),
Reward: uncleReward.String(),
})
}
}
// Process and publish transactions
transactionCids, err := pub.publishTransactions(ipldPayload.Block.Body().Transactions, ipldPayload.TxMetaData)
transactionCids, err := pub.publishTransactions(txNodes, txTrieNodes, ipldPayload.TxMetaData)
if err != nil {
return nil, err
}
// Process and publish receipts
receiptsCids, err := pub.publishReceipts(ipldPayload.Receipts, ipldPayload.ReceiptMetaData)
receiptsCids, err := pub.publishReceipts(rctNodes, rctTrieNodes, ipldPayload.ReceiptMetaData)
if err != nil {
return nil, err
}
@@ -125,25 +135,23 @@ func (pub *IPLDPublisher) Publish(payload shared.ConvertedData) (shared.CIDsForI
}, nil
}
func (pub *IPLDPublisher) publishHeader(header *types.Header) (string, error) {
cids, err := pub.HeaderPutter.DagPut(header)
if err != nil {
return "", err
}
return cids[0], nil
func (pub *IPLDPublisher) generateBlockNodes(body *types.Block, receipts types.Receipts) (*ipld.EthHeader,
[]*ipld.EthHeader, []*ipld.EthTx, []*ipld.EthTxTrie, []*ipld.EthReceipt, []*ipld.EthRctTrie, error) {
return ipld.FromBlockAndReceipts(body, receipts)
}
func (pub *IPLDPublisher) publishTransactions(transactions types.Transactions, trxMeta []TxModel) ([]TxModel, error) {
transactionCids, err := pub.TransactionPutter.DagPut(transactions)
if err != nil {
return nil, err
}
if len(transactionCids) != len(trxMeta) {
return nil, errors.New("expected one CID for each transaction")
}
mappedTrxCids := make([]TxModel, len(transactionCids))
for i, cid := range transactionCids {
mappedTrxCids[i] = TxModel{
func (pub *IPLDPublisher) publishHeader(header *ipld.EthHeader) (string, error) {
return pub.HeaderPutter.DagPut(header)
}
func (pub *IPLDPublisher) publishTransactions(transactions []*ipld.EthTx, txTrie []*ipld.EthTxTrie, trxMeta []TxModel) ([]TxModel, error) {
trxCids := make([]TxModel, len(transactions))
for i, tx := range transactions {
cid, err := pub.TransactionPutter.DagPut(tx)
if err != nil {
return nil, err
}
trxCids[i] = TxModel{
CID: cid,
Index: trxMeta[i].Index,
TxHash: trxMeta[i].TxHash,
@@ -151,22 +159,24 @@ func (pub *IPLDPublisher) publishTransactions(transactions types.Transactions, t
Dst: trxMeta[i].Dst,
}
}
return mappedTrxCids, nil
for _, txNode := range txTrie {
// We don't do anything with the tx trie cids atm
if _, err := pub.TransactionTriePutter.DagPut(txNode); err != nil {
return nil, err
}
}
return trxCids, nil
}
func (pub *IPLDPublisher) publishReceipts(receipts types.Receipts, receiptMeta []ReceiptModel) (map[common.Hash]ReceiptModel, error) {
receiptsCids, err := pub.ReceiptPutter.DagPut(receipts)
if err != nil {
return nil, err
}
if len(receiptsCids) != len(receipts) {
return nil, errors.New("expected one CID for each receipt")
}
// Map receipt cids to their transaction hashes
mappedRctCids := make(map[common.Hash]ReceiptModel, len(receiptsCids))
func (pub *IPLDPublisher) publishReceipts(receipts []*ipld.EthReceipt, receiptTrie []*ipld.EthRctTrie, receiptMeta []ReceiptModel) (map[common.Hash]ReceiptModel, error) {
rctCids := make(map[common.Hash]ReceiptModel)
for i, rct := range receipts {
mappedRctCids[rct.TxHash] = ReceiptModel{
CID: receiptsCids[i],
cid, err := pub.ReceiptPutter.DagPut(rct)
if err != nil {
return nil, err
}
rctCids[rct.TxHash] = ReceiptModel{
CID: cid,
Contract: receiptMeta[i].Contract,
Topic0s: receiptMeta[i].Topic0s,
Topic1s: receiptMeta[i].Topic1s,
@@ -174,21 +184,31 @@ func (pub *IPLDPublisher) publishReceipts(receipts types.Receipts, receiptMeta [
Topic3s: receiptMeta[i].Topic3s,
}
}
return mappedRctCids, nil
for _, rctNode := range receiptTrie {
// We don't do anything with the rct trie cids atm
if _, err := pub.ReceiptTriePutter.DagPut(rctNode); err != nil {
return nil, err
}
}
return rctCids, nil
}
func (pub *IPLDPublisher) publishStateNodes(stateNodes []TrieNode) ([]StateNodeModel, error) {
stateNodeCids := make([]StateNodeModel, 0, len(stateNodes))
for _, node := range stateNodes {
cids, err := pub.StatePutter.DagPut(node.Value)
for _, stateNode := range stateNodes {
node, err := ipld.FromStateTrieRLP(stateNode.Value)
if err != nil {
return nil, err
}
cid, err := pub.StatePutter.DagPut(node)
if err != nil {
return nil, err
}
stateNodeCids = append(stateNodeCids, StateNodeModel{
Path: node.Path,
StateKey: node.LeafKey.String(),
CID: cids[0],
NodeType: ResolveFromNodeType(node.Type),
Path: stateNode.Path,
StateKey: stateNode.LeafKey.String(),
CID: cid,
NodeType: ResolveFromNodeType(stateNode.Type),
})
}
return stateNodeCids, nil
@@ -198,17 +218,21 @@ func (pub *IPLDPublisher) publishStorageNodes(storageNodes map[common.Hash][]Tri
storageLeafCids := make(map[common.Hash][]StorageNodeModel)
for pathHash, storageTrie := range storageNodes {
storageLeafCids[pathHash] = make([]StorageNodeModel, 0, len(storageTrie))
for _, node := range storageTrie {
cids, err := pub.StoragePutter.DagPut(node.Value)
for _, storageNode := range storageTrie {
node, err := ipld.FromStorageTrieRLP(storageNode.Value)
if err != nil {
return nil, err
}
cid, err := pub.StoragePutter.DagPut(node)
if err != nil {
return nil, err
}
// Map storage node cids to their path hashes
storageLeafCids[pathHash] = append(storageLeafCids[pathHash], StorageNodeModel{
Path: node.Path,
StorageKey: node.LeafKey.Hex(),
CID: cids[0],
NodeType: ResolveFromNodeType(node.Type),
Path: storageNode.Path,
StorageKey: storageNode.LeafKey.Hex(),
CID: cid,
NodeType: ResolveFromNodeType(storageNode.Type),
})
}
}
pkg/super_node/eth/publisher_test.go
+36 -22
View File
@@ -27,40 +27,54 @@ import (
)
var (
mockHeaderDagPutter *mocks2.DagPutter
mockTrxDagPutter *mocks2.DagPutter
mockRctDagPutter *mocks2.DagPutter
mockHeaderDagPutter *mocks2.MappedDagPutter
mockTrxDagPutter *mocks2.MappedDagPutter
mockTrxTrieDagPutter *mocks2.DagPutter
mockRctDagPutter *mocks2.MappedDagPutter
mockRctTrieDagPutter *mocks2.DagPutter
mockStateDagPutter *mocks2.MappedDagPutter
mockStorageDagPutter *mocks2.DagPutter
mockStorageDagPutter *mocks2.MappedDagPutter
)
var _ = Describe("Publisher", func() {
BeforeEach(func() {
mockHeaderDagPutter = new(mocks2.DagPutter)
mockTrxDagPutter = new(mocks2.DagPutter)
mockRctDagPutter = new(mocks2.DagPutter)
mockHeaderDagPutter = new(mocks2.MappedDagPutter)
mockTrxDagPutter = new(mocks2.MappedDagPutter)
mockTrxTrieDagPutter = new(mocks2.DagPutter)
mockRctDagPutter = new(mocks2.MappedDagPutter)
mockRctTrieDagPutter = new(mocks2.DagPutter)
mockStateDagPutter = new(mocks2.MappedDagPutter)
mockStorageDagPutter = new(mocks2.DagPutter)
mockStorageDagPutter = new(mocks2.MappedDagPutter)
})
Describe("Publish", func() {
It("Publishes the passed IPLDPayload objects to IPFS and returns a CIDPayload for indexing", func() {
mockHeaderDagPutter.CIDsToReturn = []string{mocks.HeaderCID.String()}
mockTrxDagPutter.CIDsToReturn = []string{mocks.Trx1CID.String(), mocks.Trx2CID.String()}
mockRctDagPutter.CIDsToReturn = []string{mocks.Rct1CID.String(), mocks.Rct2CID.String()}
val1 := common.BytesToHash(mocks.MockConvertedPayload.StateNodes[0].Value)
val2 := common.BytesToHash(mocks.MockConvertedPayload.StateNodes[1].Value)
mockStateDagPutter.CIDsToReturn = map[common.Hash][]string{
val1: {mocks.State1CID.String()},
val2: {mocks.State2CID.String()},
mockHeaderDagPutter.CIDsToReturn = map[common.Hash]string{
common.BytesToHash(mocks.HeaderIPLD.RawData()): mocks.HeaderCID.String(),
}
mockTrxDagPutter.CIDsToReturn = map[common.Hash]string{
common.BytesToHash(mocks.Trx1IPLD.RawData()): mocks.Trx1CID.String(),
common.BytesToHash(mocks.Trx2IPLD.RawData()): mocks.Trx2CID.String(),
}
mockRctDagPutter.CIDsToReturn = map[common.Hash]string{
common.BytesToHash(mocks.Rct1IPLD.RawData()): mocks.Rct1CID.String(),
common.BytesToHash(mocks.Rct2IPLD.RawData()): mocks.Rct2CID.String(),
}
mockStateDagPutter.CIDsToReturn = map[common.Hash]string{
common.BytesToHash(mocks.State1IPLD.RawData()): mocks.State1CID.String(),
common.BytesToHash(mocks.State2IPLD.RawData()): mocks.State2CID.String(),
}
mockStorageDagPutter.CIDsToReturn = map[common.Hash]string{
common.BytesToHash(mocks.StorageIPLD.RawData()): mocks.StorageCID.String(),
}
mockStorageDagPutter.CIDsToReturn = []string{mocks.StorageCID.String()}
publisher := eth.IPLDPublisher{
HeaderPutter: mockHeaderDagPutter,
TransactionPutter: mockTrxDagPutter,
ReceiptPutter: mockRctDagPutter,
StatePutter: mockStateDagPutter,
StoragePutter: mockStorageDagPutter,
HeaderPutter: mockHeaderDagPutter,
TransactionPutter: mockTrxDagPutter,
TransactionTriePutter: mockTrxTrieDagPutter,
ReceiptPutter: mockRctDagPutter,
ReceiptTriePutter: mockRctTrieDagPutter,
StatePutter: mockStateDagPutter,
StoragePutter: mockStorageDagPutter,
}
payload, err := publisher.Publish(mocks.MockConvertedPayload)
Expect(err).ToNot(HaveOccurred())