resolver unit test

pkg/ipfs/mocks/test_data.go

@@ -23,6 +23,9 @@ import (
 	"math/big"
 	rand2 "math/rand"
 
+	"github.com/ipfs/go-block-format"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/streamer"
+
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
@@ -49,7 +52,7 @@ var (
 	ReceiptsRlp, _                             = rlp.EncodeToBytes(MockReceipts)
 	MockBlock                                  = types.NewBlock(&MockHeader, MockTransactions, nil, MockReceipts)
 	MockBlockRlp, _                            = rlp.EncodeToBytes(MockBlock)
-	MockHeaderRlp, err                         = rlp.EncodeToBytes(MockBlock.Header())
+	MockHeaderRlp, _                           = rlp.EncodeToBytes(MockBlock.Header())
 	MockTrxMeta                                = []*ipfs.TrxMetaData{
 		{
 			CID: "", // This is empty until we go to publish to ipfs
@@ -113,37 +116,23 @@ var (
 		Root:     common.HexToHash("0x"),
 		CodeHash: nil,
 	}
-	valueBytes, _        = rlp.EncodeToBytes(testAccount)
+	ValueBytes, _        = rlp.EncodeToBytes(testAccount)
-	anotherValueBytes, _ = rlp.EncodeToBytes(anotherTestAccount)
+	AnotherValueBytes, _ = rlp.EncodeToBytes(anotherTestAccount)
 	CreatedAccountDiffs  = []statediff.AccountDiff{
 		{
 			Key:     ContractLeafKey.Bytes(),
-			Value:   valueBytes,
+			Value:   ValueBytes,
 			Storage: storage,
 			Leaf:    true,
 		},
 		{
 			Key:     AnotherContractLeafKey.Bytes(),
-			Value:   anotherValueBytes,
+			Value:   AnotherValueBytes,
 			Storage: emptyStorage,
 			Leaf:    true,
 		},
 	}
 
-	UpdatedAccountDiffs = []statediff.AccountDiff{{
-		Key:     ContractLeafKey.Bytes(),
-		Value:   valueBytes,
-		Storage: storage,
-		Leaf:    true,
-	}}
-
-	DeletedAccountDiffs = []statediff.AccountDiff{{
-		Key:     ContractLeafKey.Bytes(),
-		Value:   valueBytes,
-		Storage: storage,
-		Leaf:    true,
-	}}
-
 	MockStateDiff = statediff.StateDiff{
 		BlockNumber:     BlockNumber,
 		BlockHash:       MockBlock.Hash(),
@@ -152,11 +141,11 @@ var (
 	MockStateDiffBytes, _ = rlp.EncodeToBytes(MockStateDiff)
 	MockStateNodes        = map[common.Hash]ipfs.StateNode{
 		ContractLeafKey: {
-			Value: valueBytes,
+			Value: ValueBytes,
 			Leaf:  true,
 		},
 		AnotherContractLeafKey: {
-			Value: anotherValueBytes,
+			Value: AnotherValueBytes,
 			Leaf:  true,
 		},
 	}
@@ -177,12 +166,6 @@ var (
 		ReceiptsRlp:  ReceiptsRlp,
 	}
 
-	EmptyStateDiffPayload = statediff.Payload{
-		BlockRlp:     []byte{},
-		StateDiffRlp: []byte{},
-		ReceiptsRlp:  []byte{},
-	}
-
 	MockIPLDPayload = &ipfs.IPLDPayload{
 		BlockNumber: big.NewInt(1),
 		BlockHash:   MockBlock.Hash(),
@@ -301,6 +284,46 @@ var (
 			},
 		},
 	}
+
+	MockIPLDWrapper = ipfs.IPLDWrapper{
+		BlockNumber: big.NewInt(1),
+		Headers: []blocks.Block{
+			blocks.NewBlock(MockHeaderRlp),
+		},
+		Transactions: []blocks.Block{
+			blocks.NewBlock(MockTransactions.GetRlp(0)),
+			blocks.NewBlock(MockTransactions.GetRlp(1)),
+		},
+		Receipts: []blocks.Block{
+			blocks.NewBlock(MockReceipts.GetRlp(0)),
+			blocks.NewBlock(MockReceipts.GetRlp(1)),
+		},
+		StateNodes: map[common.Hash]blocks.Block{
+			ContractLeafKey:        blocks.NewBlock(ValueBytes),
+			AnotherContractLeafKey: blocks.NewBlock(AnotherValueBytes),
+		},
+		StorageNodes: map[common.Hash]map[common.Hash]blocks.Block{
+			ContractLeafKey: {
+				common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000001"): blocks.NewBlock(StorageValue),
+			},
+		},
+	}
+
+	MockSeeNodePayload = streamer.SeedNodePayload{
+		BlockNumber:     big.NewInt(1),
+		HeadersRlp:      [][]byte{MockHeaderRlp},
+		TransactionsRlp: [][]byte{MockTransactions.GetRlp(0), MockTransactions.GetRlp(1)},
+		ReceiptsRlp:     [][]byte{MockTransactions.GetRlp(0), MockTransactions.GetRlp(1)},
+		StateNodesRlp: map[common.Hash][]byte{
+			ContractLeafKey:        ValueBytes,
+			AnotherContractLeafKey: AnotherValueBytes,
+		},
+		StorageNodesRlp: map[common.Hash]map[common.Hash][]byte{
+			ContractLeafKey: {
+				common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000001"): StorageValue,
+			},
+		},
+	}
 )
 
 // createTransactionsAndReceipts is a helper function to generate signed mock transactions and mock receipts with mock logs

pkg/ipfs/publisher.go

@@ -179,7 +179,7 @@ func (pub *Publisher) publishReceipts(receipts types.Receipts, receiptMeta []*Re
 
 func (pub *Publisher) publishStateNodes(stateNodes map[common.Hash]StateNode) (map[common.Hash]StateNodeCID, error) {
 	stateNodeCids := make(map[common.Hash]StateNodeCID)
-	for addr, node := range stateNodes {
+	for addrKey, node := range stateNodes {
 		stateNodeCid, err := pub.StatePutter.DagPut(node.Value)
 		if err != nil {
 			return nil, err
@@ -187,7 +187,7 @@ func (pub *Publisher) publishStateNodes(stateNodes map[common.Hash]StateNode) (m
 		if len(stateNodeCid) != 1 {
 			return nil, errors.New("single CID expected to be returned for state leaf")
 		}
-		stateNodeCids[addr] = StateNodeCID{
+		stateNodeCids[addrKey] = StateNodeCID{
 			CID:  stateNodeCid[0],
 			Leaf: node.Leaf,
 		}
@@ -197,8 +197,8 @@ func (pub *Publisher) publishStateNodes(stateNodes map[common.Hash]StateNode) (m
 
 func (pub *Publisher) publishStorageNodes(storageNodes map[common.Hash][]StorageNode) (map[common.Hash][]StorageNodeCID, error) {
 	storageLeafCids := make(map[common.Hash][]StorageNodeCID)
-	for addr, storageTrie := range storageNodes {
+	for addrKey, storageTrie := range storageNodes {
-		storageLeafCids[addr] = make([]StorageNodeCID, 0, len(storageTrie))
+		storageLeafCids[addrKey] = make([]StorageNodeCID, 0, len(storageTrie))
 		for _, node := range storageTrie {
 			storageNodeCid, err := pub.StoragePutter.DagPut(node.Value)
 			if err != nil {
@@ -207,7 +207,7 @@ func (pub *Publisher) publishStorageNodes(storageNodes map[common.Hash][]Storage
 			if len(storageNodeCid) != 1 {
 				return nil, errors.New("single CID expected to be returned for storage leaf")
 			}
-			storageLeafCids[addr] = append(storageLeafCids[addr], StorageNodeCID{
+			storageLeafCids[addrKey] = append(storageLeafCids[addrKey], StorageNodeCID{
 				Key:  node.Key.Hex(),
 				CID:  storageNodeCid[0],
 				Leaf: node.Leaf,

pkg/ipfs/resolver.go

@@ -24,7 +24,7 @@ import (
 
 // IPLDResolver is the interface to resolving IPLDs
 type IPLDResolver interface {
-	ResolveIPLDs(ipfsBlocks IPLDWrapper) (*streamer.SeedNodePayload, error)
+	ResolveIPLDs(ipfsBlocks IPLDWrapper) (streamer.SeedNodePayload, error)
 }
 
 // EthIPLDResolver is the underlying struct to support the IPLDResolver interface
@@ -36,16 +36,19 @@ func NewIPLDResolver() *EthIPLDResolver {
 }
 
 // ResolveIPLDs is the exported method for resolving all of the ETH IPLDs packaged in an IpfsBlockWrapper
-func (eir *EthIPLDResolver) ResolveIPLDs(ipfsBlocks IPLDWrapper) (*streamer.SeedNodePayload, error) {
+func (eir *EthIPLDResolver) ResolveIPLDs(ipfsBlocks IPLDWrapper) (streamer.SeedNodePayload, error) {
-	response := new(streamer.SeedNodePayload)
+	response := &streamer.SeedNodePayload{
-	response.BlockNumber = ipfsBlocks.BlockNumber
+		BlockNumber:     ipfsBlocks.BlockNumber,
+		StateNodesRlp:   make(map[common.Hash][]byte),
+		StorageNodesRlp: make(map[common.Hash]map[common.Hash][]byte),
+	}
 	eir.resolveHeaders(ipfsBlocks.Headers, response)
 	eir.resolveUncles(ipfsBlocks.Uncles, response)
 	eir.resolveTransactions(ipfsBlocks.Transactions, response)
 	eir.resolveReceipts(ipfsBlocks.Receipts, response)
 	eir.resolveState(ipfsBlocks.StateNodes, response)
 	eir.resolveStorage(ipfsBlocks.StorageNodes, response)
-	return response, nil
+	return *response, nil
 }
 
 func (eir *EthIPLDResolver) resolveHeaders(blocks []blocks.Block, response *streamer.SeedNodePayload) {
@@ -77,9 +80,6 @@ func (eir *EthIPLDResolver) resolveReceipts(blocks []blocks.Block, response *str
 }
 
 func (eir *EthIPLDResolver) resolveState(blocks map[common.Hash]blocks.Block, response *streamer.SeedNodePayload) {
-	if response.StateNodesRlp == nil {
-		response.StateNodesRlp = make(map[common.Hash][]byte)
-	}
 	for key, block := range blocks {
 		raw := block.RawData()
 		response.StateNodesRlp[key] = raw
@@ -87,10 +87,8 @@ func (eir *EthIPLDResolver) resolveState(blocks map[common.Hash]blocks.Block, re
 }
 
 func (eir *EthIPLDResolver) resolveStorage(blocks map[common.Hash]map[common.Hash]blocks.Block, response *streamer.SeedNodePayload) {
-	if response.StateNodesRlp == nil {
-		response.StorageNodesRlp = make(map[common.Hash]map[common.Hash][]byte)
-	}
 	for stateKey, storageBlocks := range blocks {
+		response.StorageNodesRlp[stateKey] = make(map[common.Hash][]byte)
 		for storageKey, storageVal := range storageBlocks {
 			raw := storageVal.RawData()
 			response.StorageNodesRlp[stateKey][storageKey] = raw

pkg/ipfs/resolver_test.go

@@ -0,0 +1,51 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package ipfs_test
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/vulcanize/vulcanizedb/pkg/seed_node"
+
+	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
+	"github.com/vulcanize/vulcanizedb/pkg/ipfs/mocks"
+)
+
+var (
+	resolver ipfs.IPLDResolver
+)
+
+var _ = Describe("Resolver", func() {
+	Describe("ResolveIPLDs", func() {
+		It("Resolves IPLD data to their correct geth data types and packages them to send to requesting transformers", func() {
+			resolver = ipfs.NewIPLDResolver()
+			seedNodePayload, err := resolver.ResolveIPLDs(mocks.MockIPLDWrapper)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(seedNodePayload.BlockNumber.Int64()).To(Equal(int64(1)))
+			Expect(seedNodePayload.HeadersRlp).To(Equal(mocks.MockSeeNodePayload.HeadersRlp))
+			Expect(seedNodePayload.UnclesRlp).To(Equal(mocks.MockSeeNodePayload.UnclesRlp))
+			Expect(len(seedNodePayload.TransactionsRlp)).To(Equal(2))
+			Expect(seed_node.ListContainsBytes(seedNodePayload.TransactionsRlp, mocks.MockTransactions.GetRlp(0))).To(BeTrue())
+			Expect(seed_node.ListContainsBytes(seedNodePayload.TransactionsRlp, mocks.MockTransactions.GetRlp(1))).To(BeTrue())
+			Expect(len(seedNodePayload.ReceiptsRlp)).To(Equal(2))
+			Expect(seed_node.ListContainsBytes(seedNodePayload.ReceiptsRlp, mocks.MockReceipts.GetRlp(0))).To(BeTrue())
+			Expect(seed_node.ListContainsBytes(seedNodePayload.ReceiptsRlp, mocks.MockReceipts.GetRlp(1))).To(BeTrue())
+			Expect(len(seedNodePayload.StateNodesRlp)).To(Equal(2))
+			Expect(seedNodePayload.StorageNodesRlp).To(Equal(mocks.MockSeeNodePayload.StorageNodesRlp))
+		})
+	})
+})

pkg/seed_node/filterer.go

@@ -30,42 +30,42 @@ import (
 
 // ResponseFilterer is the inteface used to screen eth data and package appropriate data into a response payload
 type ResponseFilterer interface {
-	FilterResponse(streamFilters config.Subscription, payload ipfs.IPLDPayload) (*streamer.SeedNodePayload, error)
+	FilterResponse(streamFilters config.Subscription, payload ipfs.IPLDPayload) (streamer.SeedNodePayload, error)
 }
 
-// Filterer is the underlying struct for the ReponseFilterer interface
+// Filterer is the underlying struct for the ResponseFilterer interface
 type Filterer struct{}
 
-// NewResponseFilterer creates a new Filterer satisfyign the ReponseFilterer interface
+// NewResponseFilterer creates a new Filterer satisfying the ResponseFilterer interface
 func NewResponseFilterer() *Filterer {
 	return &Filterer{}
 }
 
 // FilterResponse is used to filter through eth data to extract and package requested data into a Payload
-func (s *Filterer) FilterResponse(streamFilters config.Subscription, payload ipfs.IPLDPayload) (*streamer.SeedNodePayload, error) {
+func (s *Filterer) FilterResponse(streamFilters config.Subscription, payload ipfs.IPLDPayload) (streamer.SeedNodePayload, error) {
 	response := new(streamer.SeedNodePayload)
 	err := s.filterHeaders(streamFilters, response, payload)
 	if err != nil {
-		return nil, err
+		return streamer.SeedNodePayload{}, err
 	}
 	txHashes, err := s.filterTransactions(streamFilters, response, payload)
 	if err != nil {
-		return nil, err
+		return streamer.SeedNodePayload{}, err
 	}
 	err = s.filerReceipts(streamFilters, response, payload, txHashes)
 	if err != nil {
-		return nil, err
+		return streamer.SeedNodePayload{}, err
 	}
 	err = s.filterState(streamFilters, response, payload)
 	if err != nil {
-		return nil, err
+		return streamer.SeedNodePayload{}, err
 	}
 	err = s.filterStorage(streamFilters, response, payload)
 	if err != nil {
-		return nil, err
+		return streamer.SeedNodePayload{}, err
 	}
 	response.BlockNumber = payload.BlockNumber
-	return response, nil
+	return *response, nil
 }
 
 func (s *Filterer) filterHeaders(streamFilters config.Subscription, response *streamer.SeedNodePayload, payload ipfs.IPLDPayload) error {

pkg/seed_node/repository_test.go

@@ -60,8 +60,8 @@ var _ = Describe("Repository", func() {
 			err = db.Select(&trxs, pgStr, 1)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(trxs)).To(Equal(2))
-			Expect(seed_node.ListContains(trxs, "mockTrxCID1")).To(BeTrue())
+			Expect(seed_node.ListContainsString(trxs, "mockTrxCID1")).To(BeTrue())
-			Expect(seed_node.ListContains(trxs, "mockTrxCID2")).To(BeTrue())
+			Expect(seed_node.ListContainsString(trxs, "mockTrxCID2")).To(BeTrue())
 			// check receipts were properly indexed
 			rcts := make([]string, 0)
 			pgStr = `SELECT receipt_cids.cid FROM receipt_cids, transaction_cids, header_cids
@@ -71,8 +71,8 @@ var _ = Describe("Repository", func() {
 			err = db.Select(&rcts, pgStr, 1)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(rcts)).To(Equal(2))
-			Expect(seed_node.ListContains(rcts, "mockRctCID1")).To(BeTrue())
+			Expect(seed_node.ListContainsString(rcts, "mockRctCID1")).To(BeTrue())
-			Expect(seed_node.ListContains(rcts, "mockRctCID2")).To(BeTrue())
+			Expect(seed_node.ListContainsString(rcts, "mockRctCID2")).To(BeTrue())
 			// check that state nodes were properly indexed
 			stateNodes := make([]ipfs.StateNodeCID, 0)
 			pgStr = `SELECT state_cids.cid, state_cids.state_key, state_cids.leaf FROM state_cids INNER JOIN header_cids ON (state_cids.header_id = header_cids.id)

pkg/seed_node/retriever_test.go

@@ -60,11 +60,11 @@ var _ = Describe("Retriever", func() {
 			Expect(len(cidWrapper.Headers)).To(Equal(1))
 			Expect(cidWrapper.Headers).To(Equal(mocks.MockCIDWrapper.Headers))
 			Expect(len(cidWrapper.Transactions)).To(Equal(2))
-			Expect(seed_node.ListContains(cidWrapper.Transactions, mocks.MockCIDWrapper.Transactions[0])).To(BeTrue())
+			Expect(seed_node.ListContainsString(cidWrapper.Transactions, mocks.MockCIDWrapper.Transactions[0])).To(BeTrue())
-			Expect(seed_node.ListContains(cidWrapper.Transactions, mocks.MockCIDWrapper.Transactions[1])).To(BeTrue())
+			Expect(seed_node.ListContainsString(cidWrapper.Transactions, mocks.MockCIDWrapper.Transactions[1])).To(BeTrue())
 			Expect(len(cidWrapper.Receipts)).To(Equal(2))
-			Expect(seed_node.ListContains(cidWrapper.Receipts, mocks.MockCIDWrapper.Receipts[0])).To(BeTrue())
+			Expect(seed_node.ListContainsString(cidWrapper.Receipts, mocks.MockCIDWrapper.Receipts[0])).To(BeTrue())
-			Expect(seed_node.ListContains(cidWrapper.Receipts, mocks.MockCIDWrapper.Receipts[1])).To(BeTrue())
+			Expect(seed_node.ListContainsString(cidWrapper.Receipts, mocks.MockCIDWrapper.Receipts[1])).To(BeTrue())
 			Expect(len(cidWrapper.StateNodes)).To(Equal(2))
 			for _, stateNode := range cidWrapper.StateNodes {
 				if stateNode.CID == "mockStateCID1" {
@@ -222,8 +222,8 @@ var _ = Describe("Retriever", func() {
 			Expect(len(cidWrapper5.StateNodes)).To(Equal(0))
 			Expect(len(cidWrapper5.StorageNodes)).To(Equal(0))
 			Expect(len(cidWrapper5.Receipts)).To(Equal(2))
-			Expect(seed_node.ListContains(cidWrapper5.Receipts, "mockRctCID1")).To(BeTrue())
+			Expect(seed_node.ListContainsString(cidWrapper5.Receipts, "mockRctCID1")).To(BeTrue())
-			Expect(seed_node.ListContains(cidWrapper5.Receipts, "mockRctCID2")).To(BeTrue())
+			Expect(seed_node.ListContainsString(cidWrapper5.Receipts, "mockRctCID2")).To(BeTrue())
 
 			rctsForSelectCollectedTrxs := config.Subscription{
 				StartingBlock: big.NewInt(0),

pkg/seed_node/service.go

@@ -250,7 +250,7 @@ func (sap *Service) sendResponse(payload ipfs.IPLDPayload) error {
 		}
 		for id, sub := range subs {
 			select {
-			case sub.PayloadChan <- *response:
+			case sub.PayloadChan <- response:
 				log.Infof("sending seed node payload to subscription %s", id)
 			default:
 				log.Infof("unable to send payload to subscription %s; channel has no receiver", id)
@@ -349,7 +349,7 @@ func (sap *Service) backFill(sub Subscription, id rpc.ID, con config.Subscriptio
 				continue
 			}
 			select {
-			case sub.PayloadChan <- *backFillIplds:
+			case sub.PayloadChan <- backFillIplds:
 				log.Infof("sending seed node back-fill payload to subscription %s", id)
 			default:
 				log.Infof("unable to send back-fill payload to subscription %s; channel has no receiver", id)

pkg/seed_node/test_helpers.go

@@ -17,6 +17,8 @@
 package seed_node
 
 import (
+	"bytes"
+
 	. "github.com/onsi/gomega"
 
 	"github.com/vulcanize/vulcanizedb/pkg/config"
@@ -53,7 +55,7 @@ func TearDownDB(db *postgres.DB) {
 	Expect(err).NotTo(HaveOccurred())
 }
 
-func ListContains(sss []string, s string) bool {
+func ListContainsString(sss []string, s string) bool {
 	for _, str := range sss {
 		if s == str {
 			return true
@@ -61,3 +63,12 @@ func ListContains(sss []string, s string) bool {
 	}
 	return false
 }
+
+func ListContainsBytes(bbb [][]byte, b []byte) bool {
+	for _, by := range bbb {
+		if bytes.Equal(by, b) {
+			return true
+		}
+	}
+	return false
+}