From 03dbe5dd22c19c8123d3e05de0bd55ba85d43c5d Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Wed, 28 Aug 2019 14:43:27 -0500 Subject: [PATCH] resolver unit test --- pkg/ipfs/mocks/test_data.go | 77 +++++++++++++++++++++----------- pkg/ipfs/publisher.go | 10 ++--- pkg/ipfs/resolver.go | 20 ++++----- pkg/ipfs/resolver_test.go | 51 +++++++++++++++++++++ pkg/seed_node/filterer.go | 20 ++++----- pkg/seed_node/repository_test.go | 8 ++-- pkg/seed_node/retriever_test.go | 12 ++--- pkg/seed_node/service.go | 4 +- pkg/seed_node/test_helpers.go | 13 +++++- 9 files changed, 149 insertions(+), 66 deletions(-) create mode 100644 pkg/ipfs/resolver_test.go diff --git a/pkg/ipfs/mocks/test_data.go b/pkg/ipfs/mocks/test_data.go index c1cd8b73..e95e9a5e 100644 --- a/pkg/ipfs/mocks/test_data.go +++ b/pkg/ipfs/mocks/test_data.go @@ -23,6 +23,9 @@ import ( "math/big" rand2 "math/rand" + "github.com/ipfs/go-block-format" + "github.com/vulcanize/vulcanizedb/libraries/shared/streamer" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" @@ -49,7 +52,7 @@ var ( ReceiptsRlp, _ = rlp.EncodeToBytes(MockReceipts) MockBlock = types.NewBlock(&MockHeader, MockTransactions, nil, MockReceipts) MockBlockRlp, _ = rlp.EncodeToBytes(MockBlock) - MockHeaderRlp, err = rlp.EncodeToBytes(MockBlock.Header()) + MockHeaderRlp, _ = rlp.EncodeToBytes(MockBlock.Header()) MockTrxMeta = []*ipfs.TrxMetaData{ { CID: "", // This is empty until we go to publish to ipfs @@ -113,37 +116,23 @@ var ( Root: common.HexToHash("0x"), CodeHash: nil, } - valueBytes, _ = rlp.EncodeToBytes(testAccount) - anotherValueBytes, _ = rlp.EncodeToBytes(anotherTestAccount) + ValueBytes, _ = rlp.EncodeToBytes(testAccount) + AnotherValueBytes, _ = rlp.EncodeToBytes(anotherTestAccount) CreatedAccountDiffs = []statediff.AccountDiff{ { Key: ContractLeafKey.Bytes(), - Value: valueBytes, + Value: ValueBytes, Storage: storage, Leaf: true, }, { Key: AnotherContractLeafKey.Bytes(), - Value: anotherValueBytes, + Value: AnotherValueBytes, Storage: emptyStorage, Leaf: true, }, } - UpdatedAccountDiffs = []statediff.AccountDiff{{ - Key: ContractLeafKey.Bytes(), - Value: valueBytes, - Storage: storage, - Leaf: true, - }} - - DeletedAccountDiffs = []statediff.AccountDiff{{ - Key: ContractLeafKey.Bytes(), - Value: valueBytes, - Storage: storage, - Leaf: true, - }} - MockStateDiff = statediff.StateDiff{ BlockNumber: BlockNumber, BlockHash: MockBlock.Hash(), @@ -152,11 +141,11 @@ var ( MockStateDiffBytes, _ = rlp.EncodeToBytes(MockStateDiff) MockStateNodes = map[common.Hash]ipfs.StateNode{ ContractLeafKey: { - Value: valueBytes, + Value: ValueBytes, Leaf: true, }, AnotherContractLeafKey: { - Value: anotherValueBytes, + Value: AnotherValueBytes, Leaf: true, }, } @@ -177,12 +166,6 @@ var ( ReceiptsRlp: ReceiptsRlp, } - EmptyStateDiffPayload = statediff.Payload{ - BlockRlp: []byte{}, - StateDiffRlp: []byte{}, - ReceiptsRlp: []byte{}, - } - MockIPLDPayload = &ipfs.IPLDPayload{ BlockNumber: big.NewInt(1), BlockHash: MockBlock.Hash(), @@ -301,6 +284,46 @@ var ( }, }, } + + MockIPLDWrapper = ipfs.IPLDWrapper{ + BlockNumber: big.NewInt(1), + Headers: []blocks.Block{ + blocks.NewBlock(MockHeaderRlp), + }, + Transactions: []blocks.Block{ + blocks.NewBlock(MockTransactions.GetRlp(0)), + blocks.NewBlock(MockTransactions.GetRlp(1)), + }, + Receipts: []blocks.Block{ + blocks.NewBlock(MockReceipts.GetRlp(0)), + blocks.NewBlock(MockReceipts.GetRlp(1)), + }, + StateNodes: map[common.Hash]blocks.Block{ + ContractLeafKey: blocks.NewBlock(ValueBytes), + AnotherContractLeafKey: blocks.NewBlock(AnotherValueBytes), + }, + StorageNodes: map[common.Hash]map[common.Hash]blocks.Block{ + ContractLeafKey: { + common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000001"): blocks.NewBlock(StorageValue), + }, + }, + } + + MockSeeNodePayload = streamer.SeedNodePayload{ + BlockNumber: big.NewInt(1), + HeadersRlp: [][]byte{MockHeaderRlp}, + TransactionsRlp: [][]byte{MockTransactions.GetRlp(0), MockTransactions.GetRlp(1)}, + ReceiptsRlp: [][]byte{MockTransactions.GetRlp(0), MockTransactions.GetRlp(1)}, + StateNodesRlp: map[common.Hash][]byte{ + ContractLeafKey: ValueBytes, + AnotherContractLeafKey: AnotherValueBytes, + }, + StorageNodesRlp: map[common.Hash]map[common.Hash][]byte{ + ContractLeafKey: { + common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000001"): StorageValue, + }, + }, + } ) // createTransactionsAndReceipts is a helper function to generate signed mock transactions and mock receipts with mock logs diff --git a/pkg/ipfs/publisher.go b/pkg/ipfs/publisher.go index 2746c2ea..14a0bf97 100644 --- a/pkg/ipfs/publisher.go +++ b/pkg/ipfs/publisher.go @@ -179,7 +179,7 @@ func (pub *Publisher) publishReceipts(receipts types.Receipts, receiptMeta []*Re func (pub *Publisher) publishStateNodes(stateNodes map[common.Hash]StateNode) (map[common.Hash]StateNodeCID, error) { stateNodeCids := make(map[common.Hash]StateNodeCID) - for addr, node := range stateNodes { + for addrKey, node := range stateNodes { stateNodeCid, err := pub.StatePutter.DagPut(node.Value) if err != nil { return nil, err @@ -187,7 +187,7 @@ func (pub *Publisher) publishStateNodes(stateNodes map[common.Hash]StateNode) (m if len(stateNodeCid) != 1 { return nil, errors.New("single CID expected to be returned for state leaf") } - stateNodeCids[addr] = StateNodeCID{ + stateNodeCids[addrKey] = StateNodeCID{ CID: stateNodeCid[0], Leaf: node.Leaf, } @@ -197,8 +197,8 @@ func (pub *Publisher) publishStateNodes(stateNodes map[common.Hash]StateNode) (m func (pub *Publisher) publishStorageNodes(storageNodes map[common.Hash][]StorageNode) (map[common.Hash][]StorageNodeCID, error) { storageLeafCids := make(map[common.Hash][]StorageNodeCID) - for addr, storageTrie := range storageNodes { - storageLeafCids[addr] = make([]StorageNodeCID, 0, len(storageTrie)) + for addrKey, storageTrie := range storageNodes { + storageLeafCids[addrKey] = make([]StorageNodeCID, 0, len(storageTrie)) for _, node := range storageTrie { storageNodeCid, err := pub.StoragePutter.DagPut(node.Value) if err != nil { @@ -207,7 +207,7 @@ func (pub *Publisher) publishStorageNodes(storageNodes map[common.Hash][]Storage if len(storageNodeCid) != 1 { return nil, errors.New("single CID expected to be returned for storage leaf") } - storageLeafCids[addr] = append(storageLeafCids[addr], StorageNodeCID{ + storageLeafCids[addrKey] = append(storageLeafCids[addrKey], StorageNodeCID{ Key: node.Key.Hex(), CID: storageNodeCid[0], Leaf: node.Leaf, diff --git a/pkg/ipfs/resolver.go b/pkg/ipfs/resolver.go index 7bc9e3dc..b5d9d37d 100644 --- a/pkg/ipfs/resolver.go +++ b/pkg/ipfs/resolver.go @@ -24,7 +24,7 @@ import ( // IPLDResolver is the interface to resolving IPLDs type IPLDResolver interface { - ResolveIPLDs(ipfsBlocks IPLDWrapper) (*streamer.SeedNodePayload, error) + ResolveIPLDs(ipfsBlocks IPLDWrapper) (streamer.SeedNodePayload, error) } // EthIPLDResolver is the underlying struct to support the IPLDResolver interface @@ -36,16 +36,19 @@ func NewIPLDResolver() *EthIPLDResolver { } // ResolveIPLDs is the exported method for resolving all of the ETH IPLDs packaged in an IpfsBlockWrapper -func (eir *EthIPLDResolver) ResolveIPLDs(ipfsBlocks IPLDWrapper) (*streamer.SeedNodePayload, error) { - response := new(streamer.SeedNodePayload) - response.BlockNumber = ipfsBlocks.BlockNumber +func (eir *EthIPLDResolver) ResolveIPLDs(ipfsBlocks IPLDWrapper) (streamer.SeedNodePayload, error) { + response := &streamer.SeedNodePayload{ + BlockNumber: ipfsBlocks.BlockNumber, + StateNodesRlp: make(map[common.Hash][]byte), + StorageNodesRlp: make(map[common.Hash]map[common.Hash][]byte), + } eir.resolveHeaders(ipfsBlocks.Headers, response) eir.resolveUncles(ipfsBlocks.Uncles, response) eir.resolveTransactions(ipfsBlocks.Transactions, response) eir.resolveReceipts(ipfsBlocks.Receipts, response) eir.resolveState(ipfsBlocks.StateNodes, response) eir.resolveStorage(ipfsBlocks.StorageNodes, response) - return response, nil + return *response, nil } func (eir *EthIPLDResolver) resolveHeaders(blocks []blocks.Block, response *streamer.SeedNodePayload) { @@ -77,9 +80,6 @@ func (eir *EthIPLDResolver) resolveReceipts(blocks []blocks.Block, response *str } func (eir *EthIPLDResolver) resolveState(blocks map[common.Hash]blocks.Block, response *streamer.SeedNodePayload) { - if response.StateNodesRlp == nil { - response.StateNodesRlp = make(map[common.Hash][]byte) - } for key, block := range blocks { raw := block.RawData() response.StateNodesRlp[key] = raw @@ -87,10 +87,8 @@ func (eir *EthIPLDResolver) resolveState(blocks map[common.Hash]blocks.Block, re } func (eir *EthIPLDResolver) resolveStorage(blocks map[common.Hash]map[common.Hash]blocks.Block, response *streamer.SeedNodePayload) { - if response.StateNodesRlp == nil { - response.StorageNodesRlp = make(map[common.Hash]map[common.Hash][]byte) - } for stateKey, storageBlocks := range blocks { + response.StorageNodesRlp[stateKey] = make(map[common.Hash][]byte) for storageKey, storageVal := range storageBlocks { raw := storageVal.RawData() response.StorageNodesRlp[stateKey][storageKey] = raw diff --git a/pkg/ipfs/resolver_test.go b/pkg/ipfs/resolver_test.go new file mode 100644 index 00000000..17e1a4fb --- /dev/null +++ b/pkg/ipfs/resolver_test.go @@ -0,0 +1,51 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ipfs_test + +import ( + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/vulcanize/vulcanizedb/pkg/seed_node" + + "github.com/vulcanize/vulcanizedb/pkg/ipfs" + "github.com/vulcanize/vulcanizedb/pkg/ipfs/mocks" +) + +var ( + resolver ipfs.IPLDResolver +) + +var _ = Describe("Resolver", func() { + Describe("ResolveIPLDs", func() { + It("Resolves IPLD data to their correct geth data types and packages them to send to requesting transformers", func() { + resolver = ipfs.NewIPLDResolver() + seedNodePayload, err := resolver.ResolveIPLDs(mocks.MockIPLDWrapper) + Expect(err).ToNot(HaveOccurred()) + Expect(seedNodePayload.BlockNumber.Int64()).To(Equal(int64(1))) + Expect(seedNodePayload.HeadersRlp).To(Equal(mocks.MockSeeNodePayload.HeadersRlp)) + Expect(seedNodePayload.UnclesRlp).To(Equal(mocks.MockSeeNodePayload.UnclesRlp)) + Expect(len(seedNodePayload.TransactionsRlp)).To(Equal(2)) + Expect(seed_node.ListContainsBytes(seedNodePayload.TransactionsRlp, mocks.MockTransactions.GetRlp(0))).To(BeTrue()) + Expect(seed_node.ListContainsBytes(seedNodePayload.TransactionsRlp, mocks.MockTransactions.GetRlp(1))).To(BeTrue()) + Expect(len(seedNodePayload.ReceiptsRlp)).To(Equal(2)) + Expect(seed_node.ListContainsBytes(seedNodePayload.ReceiptsRlp, mocks.MockReceipts.GetRlp(0))).To(BeTrue()) + Expect(seed_node.ListContainsBytes(seedNodePayload.ReceiptsRlp, mocks.MockReceipts.GetRlp(1))).To(BeTrue()) + Expect(len(seedNodePayload.StateNodesRlp)).To(Equal(2)) + Expect(seedNodePayload.StorageNodesRlp).To(Equal(mocks.MockSeeNodePayload.StorageNodesRlp)) + }) + }) +}) diff --git a/pkg/seed_node/filterer.go b/pkg/seed_node/filterer.go index ce8e006d..d31d6fce 100644 --- a/pkg/seed_node/filterer.go +++ b/pkg/seed_node/filterer.go @@ -30,42 +30,42 @@ import ( // ResponseFilterer is the inteface used to screen eth data and package appropriate data into a response payload type ResponseFilterer interface { - FilterResponse(streamFilters config.Subscription, payload ipfs.IPLDPayload) (*streamer.SeedNodePayload, error) + FilterResponse(streamFilters config.Subscription, payload ipfs.IPLDPayload) (streamer.SeedNodePayload, error) } -// Filterer is the underlying struct for the ReponseFilterer interface +// Filterer is the underlying struct for the ResponseFilterer interface type Filterer struct{} -// NewResponseFilterer creates a new Filterer satisfyign the ReponseFilterer interface +// NewResponseFilterer creates a new Filterer satisfying the ResponseFilterer interface func NewResponseFilterer() *Filterer { return &Filterer{} } // FilterResponse is used to filter through eth data to extract and package requested data into a Payload -func (s *Filterer) FilterResponse(streamFilters config.Subscription, payload ipfs.IPLDPayload) (*streamer.SeedNodePayload, error) { +func (s *Filterer) FilterResponse(streamFilters config.Subscription, payload ipfs.IPLDPayload) (streamer.SeedNodePayload, error) { response := new(streamer.SeedNodePayload) err := s.filterHeaders(streamFilters, response, payload) if err != nil { - return nil, err + return streamer.SeedNodePayload{}, err } txHashes, err := s.filterTransactions(streamFilters, response, payload) if err != nil { - return nil, err + return streamer.SeedNodePayload{}, err } err = s.filerReceipts(streamFilters, response, payload, txHashes) if err != nil { - return nil, err + return streamer.SeedNodePayload{}, err } err = s.filterState(streamFilters, response, payload) if err != nil { - return nil, err + return streamer.SeedNodePayload{}, err } err = s.filterStorage(streamFilters, response, payload) if err != nil { - return nil, err + return streamer.SeedNodePayload{}, err } response.BlockNumber = payload.BlockNumber - return response, nil + return *response, nil } func (s *Filterer) filterHeaders(streamFilters config.Subscription, response *streamer.SeedNodePayload, payload ipfs.IPLDPayload) error { diff --git a/pkg/seed_node/repository_test.go b/pkg/seed_node/repository_test.go index 0696ff6e..0e85404e 100644 --- a/pkg/seed_node/repository_test.go +++ b/pkg/seed_node/repository_test.go @@ -60,8 +60,8 @@ var _ = Describe("Repository", func() { err = db.Select(&trxs, pgStr, 1) Expect(err).ToNot(HaveOccurred()) Expect(len(trxs)).To(Equal(2)) - Expect(seed_node.ListContains(trxs, "mockTrxCID1")).To(BeTrue()) - Expect(seed_node.ListContains(trxs, "mockTrxCID2")).To(BeTrue()) + Expect(seed_node.ListContainsString(trxs, "mockTrxCID1")).To(BeTrue()) + Expect(seed_node.ListContainsString(trxs, "mockTrxCID2")).To(BeTrue()) // check receipts were properly indexed rcts := make([]string, 0) pgStr = `SELECT receipt_cids.cid FROM receipt_cids, transaction_cids, header_cids @@ -71,8 +71,8 @@ var _ = Describe("Repository", func() { err = db.Select(&rcts, pgStr, 1) Expect(err).ToNot(HaveOccurred()) Expect(len(rcts)).To(Equal(2)) - Expect(seed_node.ListContains(rcts, "mockRctCID1")).To(BeTrue()) - Expect(seed_node.ListContains(rcts, "mockRctCID2")).To(BeTrue()) + Expect(seed_node.ListContainsString(rcts, "mockRctCID1")).To(BeTrue()) + Expect(seed_node.ListContainsString(rcts, "mockRctCID2")).To(BeTrue()) // check that state nodes were properly indexed stateNodes := make([]ipfs.StateNodeCID, 0) pgStr = `SELECT state_cids.cid, state_cids.state_key, state_cids.leaf FROM state_cids INNER JOIN header_cids ON (state_cids.header_id = header_cids.id) diff --git a/pkg/seed_node/retriever_test.go b/pkg/seed_node/retriever_test.go index d5ffc174..7075d08b 100644 --- a/pkg/seed_node/retriever_test.go +++ b/pkg/seed_node/retriever_test.go @@ -60,11 +60,11 @@ var _ = Describe("Retriever", func() { Expect(len(cidWrapper.Headers)).To(Equal(1)) Expect(cidWrapper.Headers).To(Equal(mocks.MockCIDWrapper.Headers)) Expect(len(cidWrapper.Transactions)).To(Equal(2)) - Expect(seed_node.ListContains(cidWrapper.Transactions, mocks.MockCIDWrapper.Transactions[0])).To(BeTrue()) - Expect(seed_node.ListContains(cidWrapper.Transactions, mocks.MockCIDWrapper.Transactions[1])).To(BeTrue()) + Expect(seed_node.ListContainsString(cidWrapper.Transactions, mocks.MockCIDWrapper.Transactions[0])).To(BeTrue()) + Expect(seed_node.ListContainsString(cidWrapper.Transactions, mocks.MockCIDWrapper.Transactions[1])).To(BeTrue()) Expect(len(cidWrapper.Receipts)).To(Equal(2)) - Expect(seed_node.ListContains(cidWrapper.Receipts, mocks.MockCIDWrapper.Receipts[0])).To(BeTrue()) - Expect(seed_node.ListContains(cidWrapper.Receipts, mocks.MockCIDWrapper.Receipts[1])).To(BeTrue()) + Expect(seed_node.ListContainsString(cidWrapper.Receipts, mocks.MockCIDWrapper.Receipts[0])).To(BeTrue()) + Expect(seed_node.ListContainsString(cidWrapper.Receipts, mocks.MockCIDWrapper.Receipts[1])).To(BeTrue()) Expect(len(cidWrapper.StateNodes)).To(Equal(2)) for _, stateNode := range cidWrapper.StateNodes { if stateNode.CID == "mockStateCID1" { @@ -222,8 +222,8 @@ var _ = Describe("Retriever", func() { Expect(len(cidWrapper5.StateNodes)).To(Equal(0)) Expect(len(cidWrapper5.StorageNodes)).To(Equal(0)) Expect(len(cidWrapper5.Receipts)).To(Equal(2)) - Expect(seed_node.ListContains(cidWrapper5.Receipts, "mockRctCID1")).To(BeTrue()) - Expect(seed_node.ListContains(cidWrapper5.Receipts, "mockRctCID2")).To(BeTrue()) + Expect(seed_node.ListContainsString(cidWrapper5.Receipts, "mockRctCID1")).To(BeTrue()) + Expect(seed_node.ListContainsString(cidWrapper5.Receipts, "mockRctCID2")).To(BeTrue()) rctsForSelectCollectedTrxs := config.Subscription{ StartingBlock: big.NewInt(0), diff --git a/pkg/seed_node/service.go b/pkg/seed_node/service.go index 5b85c404..18d3b2e7 100644 --- a/pkg/seed_node/service.go +++ b/pkg/seed_node/service.go @@ -250,7 +250,7 @@ func (sap *Service) sendResponse(payload ipfs.IPLDPayload) error { } for id, sub := range subs { select { - case sub.PayloadChan <- *response: + case sub.PayloadChan <- response: log.Infof("sending seed node payload to subscription %s", id) default: log.Infof("unable to send payload to subscription %s; channel has no receiver", id) @@ -349,7 +349,7 @@ func (sap *Service) backFill(sub Subscription, id rpc.ID, con config.Subscriptio continue } select { - case sub.PayloadChan <- *backFillIplds: + case sub.PayloadChan <- backFillIplds: log.Infof("sending seed node back-fill payload to subscription %s", id) default: log.Infof("unable to send back-fill payload to subscription %s; channel has no receiver", id) diff --git a/pkg/seed_node/test_helpers.go b/pkg/seed_node/test_helpers.go index 5725ee76..8c03a1eb 100644 --- a/pkg/seed_node/test_helpers.go +++ b/pkg/seed_node/test_helpers.go @@ -17,6 +17,8 @@ package seed_node import ( + "bytes" + . "github.com/onsi/gomega" "github.com/vulcanize/vulcanizedb/pkg/config" @@ -53,7 +55,7 @@ func TearDownDB(db *postgres.DB) { Expect(err).NotTo(HaveOccurred()) } -func ListContains(sss []string, s string) bool { +func ListContainsString(sss []string, s string) bool { for _, str := range sss { if s == str { return true @@ -61,3 +63,12 @@ func ListContains(sss []string, s string) bool { } return false } + +func ListContainsBytes(bbb [][]byte, b []byte) bool { + for _, by := range bbb { + if bytes.Equal(by, b) { + return true + } + } + return false +}