upgrade ipld-eth-indexer from v0.2.0-alpha to v0.5.0-alpha and test fixes

cmd/subscribe.go

@@ -28,8 +28,6 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
-	eth2 "github.com/vulcanize/ipld-eth-indexer/pkg/eth"
-
 	"github.com/vulcanize/ipld-eth-server/pkg/client"
 	"github.com/vulcanize/ipld-eth-server/pkg/eth"
 	w "github.com/vulcanize/ipld-eth-server/pkg/serve"
@@ -83,7 +81,7 @@ func subscribe() {
 				logWithCommand.Error(payload.Err)
 				continue
 			}
-			var ethData eth2.IPLDs
+			var ethData eth.IPLDs
 			if err := rlp.DecodeBytes(payload.Data, &ethData); err != nil {
 				logWithCommand.Error(err)
 				continue

db/migrations/00002_create_nodes_table.sql

@@ -5,7 +5,8 @@ CREATE TABLE nodes (
   genesis_block VARCHAR(66),
   network_id    VARCHAR,
   node_id       VARCHAR(128),
-  CONSTRAINT node_uc UNIQUE (genesis_block, network_id, node_id)
+  chain_id      INTEGER,
+  CONSTRAINT node_uc UNIQUE (genesis_block, network_id, node_id, chain_id)
 );
 
 -- +goose Down

db/schema.sql

@@ -361,7 +361,8 @@ CREATE TABLE public.nodes (
     client_name character varying,
     genesis_block character varying(66),
     network_id character varying,
-    node_id character varying(128)
+    node_id character varying(128),
+    chain_id integer
 );
 
 
@@ -595,7 +596,7 @@ ALTER TABLE ONLY public.goose_db_version
 --
 
 ALTER TABLE ONLY public.nodes
-    ADD CONSTRAINT node_uc UNIQUE (genesis_block, network_id, node_id);
+    ADD CONSTRAINT node_uc UNIQUE (genesis_block, network_id, node_id, chain_id);
 
 
 --

pkg/eth/cid_retriever.go

@@ -36,7 +36,7 @@ import (
 type Retriever interface {
 	RetrieveFirstBlockNumber() (int64, error)
 	RetrieveLastBlockNumber() (int64, error)
-	Retrieve(filter SubscriptionSettings, blockNumber int64) ([]eth2.CIDWrapper, bool, error)
+	Retrieve(filter SubscriptionSettings, blockNumber int64) ([]CIDWrapper, bool, error)
 }
 
 // CIDRetriever satisfies the CIDRetriever interface for ethereum
@@ -66,7 +66,7 @@ func (ecr *CIDRetriever) RetrieveLastBlockNumber() (int64, error) {
 }
 
 // Retrieve is used to retrieve all of the CIDs which conform to the passed StreamFilters
-func (ecr *CIDRetriever) Retrieve(filter SubscriptionSettings, blockNumber int64) ([]eth2.CIDWrapper, bool, error) {
+func (ecr *CIDRetriever) Retrieve(filter SubscriptionSettings, blockNumber int64) ([]CIDWrapper, bool, error) {
 	log.Debug("retrieving cids")
 
 	// Begin new db tx
@@ -91,10 +91,10 @@ func (ecr *CIDRetriever) Retrieve(filter SubscriptionSettings, blockNumber int64
 		log.Error("header cid retrieval error")
 		return nil, true, err
 	}
-	cws := make([]eth2.CIDWrapper, len(headers))
+	cws := make([]CIDWrapper, len(headers))
 	empty := true
 	for i, header := range headers {
-		cw := new(eth2.CIDWrapper)
+		cw := new(CIDWrapper)
 		cw.BlockNumber = big.NewInt(blockNumber)
 		if !filter.HeaderFilter.Off {
 			cw.Header = header

pkg/eth/cid_retriever_test.go

@@ -17,6 +17,7 @@
 package eth_test
 
 import (
+	"github.com/vulcanize/ipld-eth-server/pkg/eth/mocks"
 	"math/big"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -25,7 +26,7 @@ import (
 	. "github.com/onsi/gomega"
 
 	eth2 "github.com/vulcanize/ipld-eth-indexer/pkg/eth"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/eth/mocks"
+	mocks2 "github.com/vulcanize/ipld-eth-indexer/pkg/eth/mocks"
 	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
 
 	"github.com/vulcanize/ipld-eth-server/pkg/eth"
@@ -52,7 +53,7 @@ var (
 			Off: true,
 		},
 		ReceiptFilter: eth.ReceiptFilter{
-			LogAddresses: []string{mocks.Address.String()},
+			LogAddresses: []string{mocks2.Address.String()},
 		},
 		StateFilter: eth.StateFilter{
 			Off: true,
@@ -94,7 +95,7 @@ var (
 				{"0x0000000000000000000000000000000000000000000000000000000000000004"},
 				{"0x0000000000000000000000000000000000000000000000000000000000000006"},
 			},
-			LogAddresses: []string{mocks.Address.String()},
+			LogAddresses: []string{mocks2.Address.String()},
 		},
 		StateFilter: eth.StateFilter{
 			Off: true,
@@ -115,9 +116,9 @@ var (
 		ReceiptFilter: eth.ReceiptFilter{
 			Topics: [][]string{
 				{"0x0000000000000000000000000000000000000000000000000000000000000004"},
-				{"0x0000000000000000000000000000000000000000000000000000000000000007"}, // This topic won't match on the mocks.Address.String() contract receipt
+				{"0x0000000000000000000000000000000000000000000000000000000000000007"}, // This topic won't match on the mocks2.Address.String() contract receipt
 			},
-			LogAddresses: []string{mocks.Address.String()},
+			LogAddresses: []string{mocks2.Address.String()},
 		},
 		StateFilter: eth.StateFilter{
 			Off: true,
@@ -137,7 +138,7 @@ var (
 		},
 		ReceiptFilter: eth.ReceiptFilter{
 			Topics:       [][]string{{"0x0000000000000000000000000000000000000000000000000000000000000005"}},
-			LogAddresses: []string{mocks.Address.String(), mocks.AnotherAddress.String()},
+			LogAddresses: []string{mocks2.Address.String(), mocks2.AnotherAddress.String()},
 		},
 		StateFilter: eth.StateFilter{
 			Off: true,
@@ -172,7 +173,7 @@ var (
 			Off: true,
 		},
 		TxFilter: eth.TxFilter{
-			Dst: []string{mocks.AnotherAddress.String()}, // We only filter for one of the trxs so we will only get the one corresponding receipt
+			Dst: []string{mocks2.AnotherAddress.String()}, // We only filter for one of the trxs so we will only get the one corresponding receipt
 		},
 		ReceiptFilter: eth.ReceiptFilter{
 			MatchTxs:     true,
@@ -199,7 +200,7 @@ var (
 			Off: true,
 		},
 		StateFilter: eth.StateFilter{
-			Addresses: []string{mocks.AccountAddresss.Hex()},
+			Addresses: []string{mocks2.AccountAddresss.Hex()},
 		},
 		StorageFilter: eth.StorageFilter{
 			Off: true,
@@ -226,7 +227,7 @@ var _ = Describe("Retriever", func() {
 
 	Describe("Retrieve", func() {
 		BeforeEach(func() {
-			err := repo.Publish(mocks.MockConvertedPayload)
+			err := repo.Publish(mocks2.MockConvertedPayload)
 			Expect(err).ToNot(HaveOccurred())
 		})
 		It("Retrieves all CIDs for the given blocknumber when provided an open filter", func() {
@@ -249,13 +250,13 @@ var _ = Describe("Retriever", func() {
 			Expect(eth.ReceiptModelsContainsCID(cids[0].Receipts, mocks.MockCIDWrapper.Receipts[2].CID)).To(BeTrue())
 			Expect(len(cids[0].StateNodes)).To(Equal(2))
 			for _, stateNode := range cids[0].StateNodes {
-				if stateNode.CID == mocks.State1CID.String() {
+				if stateNode.CID == mocks2.State1CID.String() {
-					Expect(stateNode.StateKey).To(Equal(common.BytesToHash(mocks.ContractLeafKey).Hex()))
+					Expect(stateNode.StateKey).To(Equal(common.BytesToHash(mocks2.ContractLeafKey).Hex()))
 					Expect(stateNode.NodeType).To(Equal(2))
 					Expect(stateNode.Path).To(Equal([]byte{'\x06'}))
 				}
-				if stateNode.CID == mocks.State2CID.String() {
+				if stateNode.CID == mocks2.State2CID.String() {
-					Expect(stateNode.StateKey).To(Equal(common.BytesToHash(mocks.AccountLeafKey).Hex()))
+					Expect(stateNode.StateKey).To(Equal(common.BytesToHash(mocks2.AccountLeafKey).Hex()))
 					Expect(stateNode.NodeType).To(Equal(2))
 					Expect(stateNode.Path).To(Equal([]byte{'\x0c'}))
 				}
@@ -335,15 +336,15 @@ var _ = Describe("Retriever", func() {
 			Expect(cids5[0].BlockNumber).To(Equal(mocks.MockCIDWrapper.BlockNumber))
 			Expect(cids5[0].Header).To(Equal(eth2.HeaderModel{}))
 			Expect(len(cids5[0].Transactions)).To(Equal(3))
-			Expect(eth.TxModelsContainsCID(cids5[0].Transactions, mocks.Trx1CID.String())).To(BeTrue())
+			Expect(eth.TxModelsContainsCID(cids5[0].Transactions, mocks2.Trx1CID.String())).To(BeTrue())
-			Expect(eth.TxModelsContainsCID(cids5[0].Transactions, mocks.Trx2CID.String())).To(BeTrue())
+			Expect(eth.TxModelsContainsCID(cids5[0].Transactions, mocks2.Trx2CID.String())).To(BeTrue())
-			Expect(eth.TxModelsContainsCID(cids5[0].Transactions, mocks.Trx3CID.String())).To(BeTrue())
+			Expect(eth.TxModelsContainsCID(cids5[0].Transactions, mocks2.Trx3CID.String())).To(BeTrue())
 			Expect(len(cids5[0].StateNodes)).To(Equal(0))
 			Expect(len(cids5[0].StorageNodes)).To(Equal(0))
 			Expect(len(cids5[0].Receipts)).To(Equal(3))
-			Expect(eth.ReceiptModelsContainsCID(cids5[0].Receipts, mocks.Rct1CID.String())).To(BeTrue())
+			Expect(eth.ReceiptModelsContainsCID(cids5[0].Receipts, mocks2.Rct1CID.String())).To(BeTrue())
-			Expect(eth.ReceiptModelsContainsCID(cids5[0].Receipts, mocks.Rct2CID.String())).To(BeTrue())
+			Expect(eth.ReceiptModelsContainsCID(cids5[0].Receipts, mocks2.Rct2CID.String())).To(BeTrue())
-			Expect(eth.ReceiptModelsContainsCID(cids5[0].Receipts, mocks.Rct3CID.String())).To(BeTrue())
+			Expect(eth.ReceiptModelsContainsCID(cids5[0].Receipts, mocks2.Rct3CID.String())).To(BeTrue())
 
 			cids6, empty, err := retriever.Retrieve(rctsForSelectCollectedTrxs, 1)
 			Expect(err).ToNot(HaveOccurred())
@@ -378,9 +379,9 @@ var _ = Describe("Retriever", func() {
 				ID:       cids7[0].StateNodes[0].ID,
 				HeaderID: cids7[0].StateNodes[0].HeaderID,
 				NodeType: 2,
-				StateKey: common.BytesToHash(mocks.AccountLeafKey).Hex(),
+				StateKey: common.BytesToHash(mocks2.AccountLeafKey).Hex(),
-				CID:      mocks.State2CID.String(),
+				CID:      mocks2.State2CID.String(),
-				MhKey:    mocks.State2MhKey,
+				MhKey:    mocks2.State2MhKey,
 				Path:     []byte{'\x0c'},
 			}))
 
@@ -396,7 +397,7 @@ var _ = Describe("Retriever", func() {
 			Expect(err).To(HaveOccurred())
 		})
 		It("Gets the number of the first block that has data in the database", func() {
-			err := repo.Publish(mocks.MockConvertedPayload)
+			err := repo.Publish(mocks2.MockConvertedPayload)
 			Expect(err).ToNot(HaveOccurred())
 			num, err := retriever.RetrieveFirstBlockNumber()
 			Expect(err).ToNot(HaveOccurred())
@@ -404,7 +405,7 @@ var _ = Describe("Retriever", func() {
 		})
 
 		It("Gets the number of the first block that has data in the database", func() {
-			payload := mocks.MockConvertedPayload
+			payload := mocks2.MockConvertedPayload
 			payload.Block = newMockBlock(1010101)
 			err := repo.Publish(payload)
 			Expect(err).ToNot(HaveOccurred())
@@ -414,7 +415,7 @@ var _ = Describe("Retriever", func() {
 		})
 
 		It("Gets the number of the first block that has data in the database", func() {
-			payload1 := mocks.MockConvertedPayload
+			payload1 := mocks2.MockConvertedPayload
 			payload1.Block = newMockBlock(1010101)
 			payload2 := payload1
 			payload2.Block = newMockBlock(5)
@@ -434,7 +435,7 @@ var _ = Describe("Retriever", func() {
 			Expect(err).To(HaveOccurred())
 		})
 		It("Gets the number of the latest block that has data in the database", func() {
-			err := repo.Publish(mocks.MockConvertedPayload)
+			err := repo.Publish(mocks2.MockConvertedPayload)
 			Expect(err).ToNot(HaveOccurred())
 			num, err := retriever.RetrieveLastBlockNumber()
 			Expect(err).ToNot(HaveOccurred())
@@ -442,7 +443,7 @@ var _ = Describe("Retriever", func() {
 		})
 
 		It("Gets the number of the latest block that has data in the database", func() {
-			payload := mocks.MockConvertedPayload
+			payload := mocks2.MockConvertedPayload
 			payload.Block = newMockBlock(1010101)
 			err := repo.Publish(payload)
 			Expect(err).ToNot(HaveOccurred())
@@ -452,7 +453,7 @@ var _ = Describe("Retriever", func() {
 		})
 
 		It("Gets the number of the latest block that has data in the database", func() {
-			payload1 := mocks.MockConvertedPayload
+			payload1 := mocks2.MockConvertedPayload
 			payload1.Block = newMockBlock(1010101)
 			payload2 := payload1
 			payload2.Block = newMockBlock(5)
@@ -468,7 +469,7 @@ var _ = Describe("Retriever", func() {
 })
 
 func newMockBlock(blockNumber uint64) *types.Block {
-	header := mocks.MockHeader
+	header := mocks2.MockHeader
 	header.Number.SetUint64(blockNumber)
-	return types.NewBlock(&mocks.MockHeader, mocks.MockTransactions, nil, mocks.MockReceipts)
+	return types.NewBlock(&mocks2.MockHeader, mocks2.MockTransactions, nil, mocks2.MockReceipts)
 }

pkg/eth/filterer.go

@@ -33,7 +33,7 @@ import (
 
 // Filterer interface for substituing mocks in tests
 type Filterer interface {
-	Filter(filter SubscriptionSettings, payload eth.ConvertedPayload) (*eth.IPLDs, error)
+	Filter(filter SubscriptionSettings, payload eth.ConvertedPayload) (*IPLDs, error)
 }
 
 // ResponseFilterer satisfies the ResponseFilterer interface for ethereum
@@ -45,9 +45,9 @@ func NewResponseFilterer() *ResponseFilterer {
 }
 
 // Filter is used to filter through eth data to extract and package requested data into a Payload
-func (s *ResponseFilterer) Filter(filter SubscriptionSettings, payload eth.ConvertedPayload) (*eth.IPLDs, error) {
+func (s *ResponseFilterer) Filter(filter SubscriptionSettings, payload eth.ConvertedPayload) (*IPLDs, error) {
 	if checkRange(filter.Start.Int64(), filter.End.Int64(), payload.Block.Number().Int64()) {
-		response := new(eth.IPLDs)
+		response := new(IPLDs)
 		response.TotalDifficulty = payload.TotalDifficulty
 		if err := s.filterHeaders(filter.HeaderFilter, response, payload); err != nil {
 			return nil, err
@@ -72,7 +72,7 @@ func (s *ResponseFilterer) Filter(filter SubscriptionSettings, payload eth.Conve
 	return nil, nil
 }
 
-func (s *ResponseFilterer) filterHeaders(headerFilter HeaderFilter, response *eth.IPLDs, payload eth.ConvertedPayload) error {
+func (s *ResponseFilterer) filterHeaders(headerFilter HeaderFilter, response *IPLDs, payload eth.ConvertedPayload) error {
 	if !headerFilter.Off {
 		headerRLP, err := rlp.EncodeToBytes(payload.Block.Header())
 		if err != nil {
@@ -114,7 +114,7 @@ func checkRange(start, end, actual int64) bool {
 	return false
 }
 
-func (s *ResponseFilterer) filterTransactions(trxFilter TxFilter, response *eth.IPLDs, payload eth.ConvertedPayload) ([]common.Hash, error) {
+func (s *ResponseFilterer) filterTransactions(trxFilter TxFilter, response *IPLDs, payload eth.ConvertedPayload) ([]common.Hash, error) {
 	var trxHashes []common.Hash
 	if !trxFilter.Off {
 		trxLen := len(payload.Block.Body().Transactions)
@@ -162,7 +162,7 @@ func checkTransactionAddrs(wantedSrc, wantedDst []string, actualSrc, actualDst s
 	return false
 }
 
-func (s *ResponseFilterer) filerReceipts(receiptFilter ReceiptFilter, response *eth.IPLDs, payload eth.ConvertedPayload, trxHashes []common.Hash) error {
+func (s *ResponseFilterer) filerReceipts(receiptFilter ReceiptFilter, response *IPLDs, payload eth.ConvertedPayload, trxHashes []common.Hash) error {
 	if !receiptFilter.Off {
 		response.Receipts = make([]ipfs.BlockModel, 0, len(payload.Receipts))
 		for i, receipt := range payload.Receipts {
@@ -252,9 +252,9 @@ func slicesShareString(slice1, slice2 []string) int {
 }
 
 // filterStateAndStorage filters state and storage nodes into the response according to the provided filters
-func (s *ResponseFilterer) filterStateAndStorage(stateFilter StateFilter, storageFilter StorageFilter, response *eth.IPLDs, payload eth.ConvertedPayload) error {
+func (s *ResponseFilterer) filterStateAndStorage(stateFilter StateFilter, storageFilter StorageFilter, response *IPLDs, payload eth.ConvertedPayload) error {
-	response.StateNodes = make([]eth.StateNode, 0, len(payload.StateNodes))
+	response.StateNodes = make([]StateNode, 0, len(payload.StateNodes))
-	response.StorageNodes = make([]eth.StorageNode, 0)
+	response.StorageNodes = make([]StorageNode, 0)
 	stateAddressFilters := make([]common.Hash, len(stateFilter.Addresses))
 	for i, addr := range stateFilter.Addresses {
 		stateAddressFilters[i] = crypto.Keccak256Hash(common.HexToAddress(addr).Bytes())
@@ -274,7 +274,7 @@ func (s *ResponseFilterer) filterStateAndStorage(stateFilter StateFilter, storag
 				if err != nil {
 					return err
 				}
-				response.StateNodes = append(response.StateNodes, eth.StateNode{
+				response.StateNodes = append(response.StateNodes, StateNode{
 					StateLeafKey: stateNode.LeafKey,
 					Path:         stateNode.Path,
 					IPLD: ipfs.BlockModel{
@@ -292,7 +292,7 @@ func (s *ResponseFilterer) filterStateAndStorage(stateFilter StateFilter, storag
 					if err != nil {
 						return err
 					}
-					response.StorageNodes = append(response.StorageNodes, eth.StorageNode{
+					response.StorageNodes = append(response.StorageNodes, StorageNode{
 						StateLeafKey:   stateNode.LeafKey,
 						StorageLeafKey: storageNode.LeafKey,
 						IPLD: ipfs.BlockModel{

pkg/eth/filterer_test.go

@@ -18,12 +18,13 @@ package eth_test
 
 import (
 	"bytes"
+	"github.com/vulcanize/ipld-eth-server/pkg/eth/mocks"
 
 	"github.com/ethereum/go-ethereum/statediff"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	"github.com/vulcanize/ipld-eth-indexer/pkg/eth/mocks"
+	mocks2 "github.com/vulcanize/ipld-eth-indexer/pkg/eth/mocks"
 	"github.com/vulcanize/ipld-eth-indexer/pkg/ipfs"
 
 	"github.com/vulcanize/ipld-eth-server/pkg/eth"
@@ -41,7 +42,7 @@ var _ = Describe("Filterer", func() {
 		})
 
 		It("Transcribes all the data from the IPLDPayload into the StreamPayload if given an open filter", func() {
-			iplds, err := filterer.Filter(openFilter, mocks.MockConvertedPayload)
+			iplds, err := filterer.Filter(openFilter, mocks2.MockConvertedPayload)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(iplds).ToNot(BeNil())
 			Expect(iplds.BlockNumber.Int64()).To(Equal(mocks.MockIPLDs.BlockNumber.Int64()))
@@ -49,26 +50,26 @@ var _ = Describe("Filterer", func() {
 			var expectedEmptyUncles []ipfs.BlockModel
 			Expect(iplds.Uncles).To(Equal(expectedEmptyUncles))
 			Expect(len(iplds.Transactions)).To(Equal(3))
-			Expect(shared.IPLDsContainBytes(iplds.Transactions, mocks.MockTransactions.GetRlp(0))).To(BeTrue())
+			Expect(shared.IPLDsContainBytes(iplds.Transactions, mocks2.MockTransactions.GetRlp(0))).To(BeTrue())
-			Expect(shared.IPLDsContainBytes(iplds.Transactions, mocks.MockTransactions.GetRlp(1))).To(BeTrue())
+			Expect(shared.IPLDsContainBytes(iplds.Transactions, mocks2.MockTransactions.GetRlp(1))).To(BeTrue())
-			Expect(shared.IPLDsContainBytes(iplds.Transactions, mocks.MockTransactions.GetRlp(2))).To(BeTrue())
+			Expect(shared.IPLDsContainBytes(iplds.Transactions, mocks2.MockTransactions.GetRlp(2))).To(BeTrue())
 			Expect(len(iplds.Receipts)).To(Equal(3))
-			Expect(shared.IPLDsContainBytes(iplds.Receipts, mocks.MockReceipts.GetRlp(0))).To(BeTrue())
+			Expect(shared.IPLDsContainBytes(iplds.Receipts, mocks2.MockReceipts.GetRlp(0))).To(BeTrue())
-			Expect(shared.IPLDsContainBytes(iplds.Receipts, mocks.MockReceipts.GetRlp(1))).To(BeTrue())
+			Expect(shared.IPLDsContainBytes(iplds.Receipts, mocks2.MockReceipts.GetRlp(1))).To(BeTrue())
-			Expect(shared.IPLDsContainBytes(iplds.Receipts, mocks.MockReceipts.GetRlp(2))).To(BeTrue())
+			Expect(shared.IPLDsContainBytes(iplds.Receipts, mocks2.MockReceipts.GetRlp(2))).To(BeTrue())
 			Expect(len(iplds.StateNodes)).To(Equal(2))
 			for _, stateNode := range iplds.StateNodes {
 				Expect(stateNode.Type).To(Equal(statediff.Leaf))
-				if bytes.Equal(stateNode.StateLeafKey.Bytes(), mocks.AccountLeafKey) {
+				if bytes.Equal(stateNode.StateLeafKey.Bytes(), mocks2.AccountLeafKey) {
 					Expect(stateNode.IPLD).To(Equal(ipfs.BlockModel{
-						Data: mocks.State2IPLD.RawData(),
+						Data: mocks2.State2IPLD.RawData(),
-						CID:  mocks.State2IPLD.Cid().String(),
+						CID:  mocks2.State2IPLD.Cid().String(),
 					}))
 				}
-				if bytes.Equal(stateNode.StateLeafKey.Bytes(), mocks.ContractLeafKey) {
+				if bytes.Equal(stateNode.StateLeafKey.Bytes(), mocks2.ContractLeafKey) {
 					Expect(stateNode.IPLD).To(Equal(ipfs.BlockModel{
-						Data: mocks.State1IPLD.RawData(),
+						Data: mocks2.State1IPLD.RawData(),
-						CID:  mocks.State1IPLD.Cid().String(),
+						CID:  mocks2.State1IPLD.Cid().String(),
 					}))
 				}
 			}
@@ -76,7 +77,7 @@ var _ = Describe("Filterer", func() {
 		})
 
 		It("Applies filters from the provided config.Subscription", func() {
-			iplds1, err := filterer.Filter(rctAddressFilter, mocks.MockConvertedPayload)
+			iplds1, err := filterer.Filter(rctAddressFilter, mocks2.MockConvertedPayload)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(iplds1).ToNot(BeNil())
 			Expect(iplds1.BlockNumber.Int64()).To(Equal(mocks.MockIPLDs.BlockNumber.Int64()))
@@ -87,11 +88,11 @@ var _ = Describe("Filterer", func() {
 			Expect(len(iplds1.StateNodes)).To(Equal(0))
 			Expect(len(iplds1.Receipts)).To(Equal(1))
 			Expect(iplds1.Receipts[0]).To(Equal(ipfs.BlockModel{
-				Data: mocks.Rct1IPLD.RawData(),
+				Data: mocks2.Rct1IPLD.RawData(),
-				CID:  mocks.Rct1IPLD.Cid().String(),
+				CID:  mocks2.Rct1IPLD.Cid().String(),
 			}))
 
-			iplds2, err := filterer.Filter(rctTopicsFilter, mocks.MockConvertedPayload)
+			iplds2, err := filterer.Filter(rctTopicsFilter, mocks2.MockConvertedPayload)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(iplds2).ToNot(BeNil())
 			Expect(iplds2.BlockNumber.Int64()).To(Equal(mocks.MockIPLDs.BlockNumber.Int64()))
@@ -102,11 +103,11 @@ var _ = Describe("Filterer", func() {
 			Expect(len(iplds2.StateNodes)).To(Equal(0))
 			Expect(len(iplds2.Receipts)).To(Equal(1))
 			Expect(iplds2.Receipts[0]).To(Equal(ipfs.BlockModel{
-				Data: mocks.Rct1IPLD.RawData(),
+				Data: mocks2.Rct1IPLD.RawData(),
-				CID:  mocks.Rct1IPLD.Cid().String(),
+				CID:  mocks2.Rct1IPLD.Cid().String(),
 			}))
 
-			iplds3, err := filterer.Filter(rctTopicsAndAddressFilter, mocks.MockConvertedPayload)
+			iplds3, err := filterer.Filter(rctTopicsAndAddressFilter, mocks2.MockConvertedPayload)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(iplds3).ToNot(BeNil())
 			Expect(iplds3.BlockNumber.Int64()).To(Equal(mocks.MockIPLDs.BlockNumber.Int64()))
@@ -117,11 +118,11 @@ var _ = Describe("Filterer", func() {
 			Expect(len(iplds3.StateNodes)).To(Equal(0))
 			Expect(len(iplds3.Receipts)).To(Equal(1))
 			Expect(iplds3.Receipts[0]).To(Equal(ipfs.BlockModel{
-				Data: mocks.Rct1IPLD.RawData(),
+				Data: mocks2.Rct1IPLD.RawData(),
-				CID:  mocks.Rct1IPLD.Cid().String(),
+				CID:  mocks2.Rct1IPLD.Cid().String(),
 			}))
 
-			iplds4, err := filterer.Filter(rctAddressesAndTopicFilter, mocks.MockConvertedPayload)
+			iplds4, err := filterer.Filter(rctAddressesAndTopicFilter, mocks2.MockConvertedPayload)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(iplds4).ToNot(BeNil())
 			Expect(iplds4.BlockNumber.Int64()).To(Equal(mocks.MockIPLDs.BlockNumber.Int64()))
@@ -132,44 +133,44 @@ var _ = Describe("Filterer", func() {
 			Expect(len(iplds4.StateNodes)).To(Equal(0))
 			Expect(len(iplds4.Receipts)).To(Equal(1))
 			Expect(iplds4.Receipts[0]).To(Equal(ipfs.BlockModel{
-				Data: mocks.Rct2IPLD.RawData(),
+				Data: mocks2.Rct2IPLD.RawData(),
-				CID:  mocks.Rct2IPLD.Cid().String(),
+				CID:  mocks2.Rct2IPLD.Cid().String(),
 			}))
 
-			iplds5, err := filterer.Filter(rctsForAllCollectedTrxs, mocks.MockConvertedPayload)
+			iplds5, err := filterer.Filter(rctsForAllCollectedTrxs, mocks2.MockConvertedPayload)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(iplds5).ToNot(BeNil())
 			Expect(iplds5.BlockNumber.Int64()).To(Equal(mocks.MockIPLDs.BlockNumber.Int64()))
 			Expect(iplds5.Header).To(Equal(ipfs.BlockModel{}))
 			Expect(len(iplds5.Uncles)).To(Equal(0))
 			Expect(len(iplds5.Transactions)).To(Equal(3))
-			Expect(shared.IPLDsContainBytes(iplds5.Transactions, mocks.MockTransactions.GetRlp(0))).To(BeTrue())
+			Expect(shared.IPLDsContainBytes(iplds5.Transactions, mocks2.MockTransactions.GetRlp(0))).To(BeTrue())
-			Expect(shared.IPLDsContainBytes(iplds5.Transactions, mocks.MockTransactions.GetRlp(1))).To(BeTrue())
+			Expect(shared.IPLDsContainBytes(iplds5.Transactions, mocks2.MockTransactions.GetRlp(1))).To(BeTrue())
-			Expect(shared.IPLDsContainBytes(iplds5.Transactions, mocks.MockTransactions.GetRlp(2))).To(BeTrue())
+			Expect(shared.IPLDsContainBytes(iplds5.Transactions, mocks2.MockTransactions.GetRlp(2))).To(BeTrue())
 			Expect(len(iplds5.StorageNodes)).To(Equal(0))
 			Expect(len(iplds5.StateNodes)).To(Equal(0))
 			Expect(len(iplds5.Receipts)).To(Equal(3))
-			Expect(shared.IPLDsContainBytes(iplds5.Receipts, mocks.MockReceipts.GetRlp(0))).To(BeTrue())
+			Expect(shared.IPLDsContainBytes(iplds5.Receipts, mocks2.MockReceipts.GetRlp(0))).To(BeTrue())
-			Expect(shared.IPLDsContainBytes(iplds5.Receipts, mocks.MockReceipts.GetRlp(1))).To(BeTrue())
+			Expect(shared.IPLDsContainBytes(iplds5.Receipts, mocks2.MockReceipts.GetRlp(1))).To(BeTrue())
-			Expect(shared.IPLDsContainBytes(iplds5.Receipts, mocks.MockReceipts.GetRlp(2))).To(BeTrue())
+			Expect(shared.IPLDsContainBytes(iplds5.Receipts, mocks2.MockReceipts.GetRlp(2))).To(BeTrue())
 
-			iplds6, err := filterer.Filter(rctsForSelectCollectedTrxs, mocks.MockConvertedPayload)
+			iplds6, err := filterer.Filter(rctsForSelectCollectedTrxs, mocks2.MockConvertedPayload)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(iplds6).ToNot(BeNil())
 			Expect(iplds6.BlockNumber.Int64()).To(Equal(mocks.MockIPLDs.BlockNumber.Int64()))
 			Expect(iplds6.Header).To(Equal(ipfs.BlockModel{}))
 			Expect(len(iplds6.Uncles)).To(Equal(0))
 			Expect(len(iplds6.Transactions)).To(Equal(1))
-			Expect(shared.IPLDsContainBytes(iplds5.Transactions, mocks.MockTransactions.GetRlp(1))).To(BeTrue())
+			Expect(shared.IPLDsContainBytes(iplds5.Transactions, mocks2.MockTransactions.GetRlp(1))).To(BeTrue())
 			Expect(len(iplds6.StorageNodes)).To(Equal(0))
 			Expect(len(iplds6.StateNodes)).To(Equal(0))
 			Expect(len(iplds6.Receipts)).To(Equal(1))
 			Expect(iplds4.Receipts[0]).To(Equal(ipfs.BlockModel{
-				Data: mocks.Rct2IPLD.RawData(),
+				Data: mocks2.Rct2IPLD.RawData(),
-				CID:  mocks.Rct2IPLD.Cid().String(),
+				CID:  mocks2.Rct2IPLD.Cid().String(),
 			}))
 
-			iplds7, err := filterer.Filter(stateFilter, mocks.MockConvertedPayload)
+			iplds7, err := filterer.Filter(stateFilter, mocks2.MockConvertedPayload)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(iplds7).ToNot(BeNil())
 			Expect(iplds7.BlockNumber.Int64()).To(Equal(mocks.MockIPLDs.BlockNumber.Int64()))
@@ -179,13 +180,13 @@ var _ = Describe("Filterer", func() {
 			Expect(len(iplds7.StorageNodes)).To(Equal(0))
 			Expect(len(iplds7.Receipts)).To(Equal(0))
 			Expect(len(iplds7.StateNodes)).To(Equal(1))
-			Expect(iplds7.StateNodes[0].StateLeafKey.Bytes()).To(Equal(mocks.AccountLeafKey))
+			Expect(iplds7.StateNodes[0].StateLeafKey.Bytes()).To(Equal(mocks2.AccountLeafKey))
 			Expect(iplds7.StateNodes[0].IPLD).To(Equal(ipfs.BlockModel{
-				Data: mocks.State2IPLD.RawData(),
+				Data: mocks2.State2IPLD.RawData(),
-				CID:  mocks.State2IPLD.Cid().String(),
+				CID:  mocks2.State2IPLD.Cid().String(),
 			}))
 
-			iplds8, err := filterer.Filter(rctTopicsAndAddressFilterFail, mocks.MockConvertedPayload)
+			iplds8, err := filterer.Filter(rctTopicsAndAddressFilterFail, mocks2.MockConvertedPayload)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(iplds8).ToNot(BeNil())
 			Expect(iplds8.BlockNumber.Int64()).To(Equal(mocks.MockIPLDs.BlockNumber.Int64()))

pkg/eth/ipld_fetcher.go

@@ -34,7 +34,7 @@ import (
 
 // Fetcher interface for substituting mocks in tests
 type Fetcher interface {
-	Fetch(cids eth.CIDWrapper) (*eth.IPLDs, error)
+	Fetch(cids CIDWrapper) (*IPLDs, error)
 }
 
 // IPLDFetcher satisfies the IPLDFetcher interface for ethereum
@@ -51,9 +51,9 @@ func NewIPLDFetcher(db *postgres.DB) *IPLDFetcher {
 }
 
 // Fetch is the exported method for fetching and returning all the IPLDS specified in the CIDWrapper
-func (f *IPLDFetcher) Fetch(cids eth.CIDWrapper) (*eth.IPLDs, error) {
+func (f *IPLDFetcher) Fetch(cids CIDWrapper) (*IPLDs, error) {
 	log.Debug("fetching iplds")
-	iplds := new(eth.IPLDs)
+	iplds := new(IPLDs)
 	var ok bool
 	iplds.TotalDifficulty, ok = new(big.Int).SetString(cids.Header.TotalDifficulty, 10)
 	if !ok {
@@ -168,9 +168,9 @@ func (f *IPLDFetcher) FetchRcts(tx *sqlx.Tx, cids []eth.ReceiptModel) ([]ipfs.Bl
 }
 
 // FetchState fetches state nodes
-func (f *IPLDFetcher) FetchState(tx *sqlx.Tx, cids []eth.StateNodeModel) ([]eth.StateNode, error) {
+func (f *IPLDFetcher) FetchState(tx *sqlx.Tx, cids []eth.StateNodeModel) ([]StateNode, error) {
 	log.Debug("fetching state iplds")
-	stateNodes := make([]eth.StateNode, 0, len(cids))
+	stateNodes := make([]StateNode, 0, len(cids))
 	for _, stateNode := range cids {
 		if stateNode.CID == "" {
 			continue
@@ -179,7 +179,7 @@ func (f *IPLDFetcher) FetchState(tx *sqlx.Tx, cids []eth.StateNodeModel) ([]eth.
 		if err != nil {
 			return nil, err
 		}
-		stateNodes = append(stateNodes, eth.StateNode{
+		stateNodes = append(stateNodes, StateNode{
 			IPLD: ipfs.BlockModel{
 				Data: stateBytes,
 				CID:  stateNode.CID,
@@ -193,9 +193,9 @@ func (f *IPLDFetcher) FetchState(tx *sqlx.Tx, cids []eth.StateNodeModel) ([]eth.
 }
 
 // FetchStorage fetches storage nodes
-func (f *IPLDFetcher) FetchStorage(tx *sqlx.Tx, cids []eth.StorageNodeWithStateKeyModel) ([]eth.StorageNode, error) {
+func (f *IPLDFetcher) FetchStorage(tx *sqlx.Tx, cids []eth.StorageNodeWithStateKeyModel) ([]StorageNode, error) {
 	log.Debug("fetching storage iplds")
-	storageNodes := make([]eth.StorageNode, 0, len(cids))
+	storageNodes := make([]StorageNode, 0, len(cids))
 	for _, storageNode := range cids {
 		if storageNode.CID == "" || storageNode.StateKey == "" {
 			continue
@@ -204,7 +204,7 @@ func (f *IPLDFetcher) FetchStorage(tx *sqlx.Tx, cids []eth.StorageNodeWithStateK
 		if err != nil {
 			return nil, err
 		}
-		storageNodes = append(storageNodes, eth.StorageNode{
+		storageNodes = append(storageNodes, StorageNode{
 			IPLD: ipfs.BlockModel{
 				Data: storageBytes,
 				CID:  storageNode.CID,

pkg/eth/ipld_fetcher_test.go

@@ -19,9 +19,10 @@ package eth_test
 import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+	"github.com/vulcanize/ipld-eth-server/pkg/eth/mocks"
 
 	eth2 "github.com/vulcanize/ipld-eth-indexer/pkg/eth"
-	"github.com/vulcanize/ipld-eth-indexer/pkg/eth/mocks"
+	mocks2 "github.com/vulcanize/ipld-eth-indexer/pkg/eth/mocks"
 	"github.com/vulcanize/ipld-eth-indexer/pkg/postgres"
 
 	"github.com/vulcanize/ipld-eth-server/pkg/eth"
@@ -41,7 +42,7 @@ var _ = Describe("IPLDFetcher", func() {
 			db, err = shared.SetupDB()
 			Expect(err).ToNot(HaveOccurred())
 			pubAndIndexer = eth2.NewIPLDPublisher(db)
-			err = pubAndIndexer.Publish(mocks.MockConvertedPayload)
+			err = pubAndIndexer.Publish(mocks2.MockConvertedPayload)
 			Expect(err).ToNot(HaveOccurred())
 			fetcher = eth.NewIPLDFetcher(db)
 		})
@@ -53,8 +54,8 @@ var _ = Describe("IPLDFetcher", func() {
 			iplds, err := fetcher.Fetch(*mocks.MockCIDWrapper)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(iplds).ToNot(BeNil())
-			Expect(iplds.TotalDifficulty).To(Equal(mocks.MockConvertedPayload.TotalDifficulty))
+			Expect(iplds.TotalDifficulty).To(Equal(mocks2.MockConvertedPayload.TotalDifficulty))
-			Expect(iplds.BlockNumber).To(Equal(mocks.MockConvertedPayload.Block.Number()))
+			Expect(iplds.BlockNumber).To(Equal(mocks2.MockConvertedPayload.Block.Number()))
 			Expect(iplds.Header).To(Equal(mocks.MockIPLDs.Header))
 			Expect(len(iplds.Uncles)).To(Equal(0))
 			Expect(iplds.Transactions).To(Equal(mocks.MockIPLDs.Transactions))

pkg/eth/mocks/test_data.go

@@ -0,0 +1,511 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package mocks
+
+import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/state"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/statediff"
+	"github.com/ethereum/go-ethereum/statediff/testhelpers"
+	"github.com/ipfs/go-block-format"
+	"github.com/multiformats/go-multihash"
+	log "github.com/sirupsen/logrus"
+	eth2 "github.com/vulcanize/ipld-eth-indexer/pkg/eth"
+	"github.com/vulcanize/ipld-eth-indexer/pkg/ipfs"
+	"github.com/vulcanize/ipld-eth-indexer/pkg/ipfs/ipld"
+	"github.com/vulcanize/ipld-eth-indexer/pkg/shared"
+	"github.com/vulcanize/ipld-eth-server/pkg/eth"
+	"math/big"
+)
+
+var (
+	// block data
+	BlockNumber = big.NewInt(1)
+	MockHeader  = types.Header{
+		Time:        0,
+		Number:      new(big.Int).Set(BlockNumber),
+		Root:        common.HexToHash("0x0"),
+		TxHash:      common.HexToHash("0x0"),
+		ReceiptHash: common.HexToHash("0x0"),
+		Difficulty:  big.NewInt(5000000),
+		Extra:       []byte{},
+	}
+	MockTransactions, MockReceipts, SenderAddr = createTransactionsAndReceipts()
+	ReceiptsRlp, _                             = rlp.EncodeToBytes(MockReceipts)
+	MockBlock                                  = types.NewBlock(&MockHeader, MockTransactions, nil, MockReceipts)
+	MockBlockRlp, _                            = rlp.EncodeToBytes(MockBlock)
+	MockHeaderRlp, _                           = rlp.EncodeToBytes(MockBlock.Header())
+	Address                                    = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592")
+	AnotherAddress                             = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476593")
+	ContractAddress                            = crypto.CreateAddress(SenderAddr, MockTransactions[2].Nonce())
+	ContractHash                               = crypto.Keccak256Hash(ContractAddress.Bytes()).String()
+	MockContractByteCode                       = []byte{0, 1, 2, 3, 4, 5}
+	mockTopic11                                = common.HexToHash("0x04")
+	mockTopic12                                = common.HexToHash("0x06")
+	mockTopic21                                = common.HexToHash("0x05")
+	mockTopic22                                = common.HexToHash("0x07")
+	MockLog1                                   = &types.Log{
+		Address: Address,
+		Topics:  []common.Hash{mockTopic11, mockTopic12},
+		Data:    []byte{},
+	}
+	MockLog2 = &types.Log{
+		Address: AnotherAddress,
+		Topics:  []common.Hash{mockTopic21, mockTopic22},
+		Data:    []byte{},
+	}
+	HeaderCID, _  = ipld.RawdataToCid(ipld.MEthHeader, MockHeaderRlp, multihash.KECCAK_256)
+	HeaderMhKey   = shared.MultihashKeyFromCID(HeaderCID)
+	Trx1CID, _    = ipld.RawdataToCid(ipld.MEthTx, MockTransactions.GetRlp(0), multihash.KECCAK_256)
+	Trx1MhKey     = shared.MultihashKeyFromCID(Trx1CID)
+	Trx2CID, _    = ipld.RawdataToCid(ipld.MEthTx, MockTransactions.GetRlp(1), multihash.KECCAK_256)
+	Trx2MhKey     = shared.MultihashKeyFromCID(Trx2CID)
+	Trx3CID, _    = ipld.RawdataToCid(ipld.MEthTx, MockTransactions.GetRlp(2), multihash.KECCAK_256)
+	Trx3MhKey     = shared.MultihashKeyFromCID(Trx3CID)
+	Rct1CID, _    = ipld.RawdataToCid(ipld.MEthTxReceipt, MockReceipts.GetRlp(0), multihash.KECCAK_256)
+	Rct1MhKey     = shared.MultihashKeyFromCID(Rct1CID)
+	Rct2CID, _    = ipld.RawdataToCid(ipld.MEthTxReceipt, MockReceipts.GetRlp(1), multihash.KECCAK_256)
+	Rct2MhKey     = shared.MultihashKeyFromCID(Rct2CID)
+	Rct3CID, _    = ipld.RawdataToCid(ipld.MEthTxReceipt, MockReceipts.GetRlp(2), multihash.KECCAK_256)
+	Rct3MhKey     = shared.MultihashKeyFromCID(Rct3CID)
+	State1CID, _  = ipld.RawdataToCid(ipld.MEthStateTrie, ContractLeafNode, multihash.KECCAK_256)
+	State1MhKey   = shared.MultihashKeyFromCID(State1CID)
+	State2CID, _  = ipld.RawdataToCid(ipld.MEthStateTrie, AccountLeafNode, multihash.KECCAK_256)
+	State2MhKey   = shared.MultihashKeyFromCID(State2CID)
+	StorageCID, _ = ipld.RawdataToCid(ipld.MEthStorageTrie, StorageLeafNode, multihash.KECCAK_256)
+	StorageMhKey  = shared.MultihashKeyFromCID(StorageCID)
+
+	MockTrxMeta = []eth2.TxModel{
+		{
+			CID:        "", // This is empty until we go to publish to ipfs
+			MhKey:      "",
+			Src:        SenderAddr.Hex(),
+			Dst:        Address.String(),
+			Index:      0,
+			TxHash:     MockTransactions[0].Hash().String(),
+			Data:       []byte{},
+			Deployment: false,
+		},
+		{
+			CID:        "",
+			MhKey:      "",
+			Src:        SenderAddr.Hex(),
+			Dst:        AnotherAddress.String(),
+			Index:      1,
+			TxHash:     MockTransactions[1].Hash().String(),
+			Data:       []byte{},
+			Deployment: false,
+		},
+		{
+			CID:        "",
+			MhKey:      "",
+			Src:        SenderAddr.Hex(),
+			Dst:        "",
+			Index:      2,
+			TxHash:     MockTransactions[2].Hash().String(),
+			Data:       MockContractByteCode,
+			Deployment: true,
+		},
+	}
+	MockTrxMetaPostPublsh = []eth2.TxModel{
+		{
+			CID:        Trx1CID.String(), // This is empty until we go to publish to ipfs
+			MhKey:      Trx1MhKey,
+			Src:        SenderAddr.Hex(),
+			Dst:        Address.String(),
+			Index:      0,
+			TxHash:     MockTransactions[0].Hash().String(),
+			Data:       []byte{},
+			Deployment: false,
+		},
+		{
+			CID:        Trx2CID.String(),
+			MhKey:      Trx2MhKey,
+			Src:        SenderAddr.Hex(),
+			Dst:        AnotherAddress.String(),
+			Index:      1,
+			TxHash:     MockTransactions[1].Hash().String(),
+			Data:       []byte{},
+			Deployment: false,
+		},
+		{
+			CID:        Trx3CID.String(),
+			MhKey:      Trx3MhKey,
+			Src:        SenderAddr.Hex(),
+			Dst:        "",
+			Index:      2,
+			TxHash:     MockTransactions[2].Hash().String(),
+			Data:       MockContractByteCode,
+			Deployment: true,
+		},
+	}
+	MockRctMeta = []eth2.ReceiptModel{
+		{
+			CID:   "",
+			MhKey: "",
+			Topic0s: []string{
+				mockTopic11.String(),
+			},
+			Topic1s: []string{
+				mockTopic12.String(),
+			},
+			Contract:     "",
+			ContractHash: "",
+			LogContracts: []string{
+				Address.String(),
+			},
+		},
+		{
+			CID:   "",
+			MhKey: "",
+			Topic0s: []string{
+				mockTopic21.String(),
+			},
+			Topic1s: []string{
+				mockTopic22.String(),
+			},
+			Contract:     "",
+			ContractHash: "",
+			LogContracts: []string{
+				AnotherAddress.String(),
+			},
+		},
+		{
+			CID:          "",
+			MhKey:        "",
+			Contract:     ContractAddress.String(),
+			ContractHash: ContractHash,
+			LogContracts: []string{},
+		},
+	}
+	MockRctMetaPostPublish = []eth2.ReceiptModel{
+		{
+			CID:   Rct1CID.String(),
+			MhKey: Rct1MhKey,
+			Topic0s: []string{
+				mockTopic11.String(),
+			},
+			Topic1s: []string{
+				mockTopic12.String(),
+			},
+			Contract:     "",
+			ContractHash: "",
+			LogContracts: []string{
+				Address.String(),
+			},
+		},
+		{
+			CID:   Rct2CID.String(),
+			MhKey: Rct2MhKey,
+			Topic0s: []string{
+				mockTopic21.String(),
+			},
+			Topic1s: []string{
+				mockTopic22.String(),
+			},
+			Contract:     "",
+			ContractHash: "",
+			LogContracts: []string{
+				AnotherAddress.String(),
+			},
+		},
+		{
+			CID:          Rct3CID.String(),
+			MhKey:        Rct3MhKey,
+			Contract:     ContractAddress.String(),
+			ContractHash: ContractHash,
+			LogContracts: []string{},
+		},
+	}
+
+	// statediff data
+	storageLocation    = common.HexToHash("0")
+	StorageLeafKey     = crypto.Keccak256Hash(storageLocation[:]).Bytes()
+	StorageValue       = common.Hex2Bytes("01")
+	StoragePartialPath = common.Hex2Bytes("20290decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563")
+	StorageLeafNode, _ = rlp.EncodeToBytes([]interface{}{
+		StoragePartialPath,
+		StorageValue,
+	})
+
+	nonce1             = uint64(1)
+	ContractRoot       = "0x821e2556a290c86405f8160a2d662042a431ba456b9db265c79bb837c04be5f0"
+	ContractCodeHash   = common.HexToHash("0x753f98a8d4328b15636e46f66f2cb4bc860100aa17967cc145fcd17d1d4710ea")
+	contractPath       = common.Bytes2Hex([]byte{'\x06'})
+	ContractLeafKey    = testhelpers.AddressToLeafKey(ContractAddress)
+	ContractAccount, _ = rlp.EncodeToBytes(state.Account{
+		Nonce:    nonce1,
+		Balance:  big.NewInt(0),
+		CodeHash: ContractCodeHash.Bytes(),
+		Root:     common.HexToHash(ContractRoot),
+	})
+	ContractPartialPath = common.Hex2Bytes("3114658a74d9cc9f7acf2c5cd696c3494d7c344d78bfec3add0d91ec4e8d1c45")
+	ContractLeafNode, _ = rlp.EncodeToBytes([]interface{}{
+		ContractPartialPath,
+		ContractAccount,
+	})
+
+	nonce0          = uint64(0)
+	AccountRoot     = "0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421"
+	AccountCodeHash = common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")
+	accountPath     = common.Bytes2Hex([]byte{'\x0c'})
+	AccountAddresss = common.HexToAddress("0x0D3ab14BBaD3D99F4203bd7a11aCB94882050E7e")
+	AccountLeafKey  = testhelpers.Account2LeafKey
+	Account, _      = rlp.EncodeToBytes(state.Account{
+		Nonce:    nonce0,
+		Balance:  big.NewInt(1000),
+		CodeHash: AccountCodeHash.Bytes(),
+		Root:     common.HexToHash(AccountRoot),
+	})
+	AccountPartialPath = common.Hex2Bytes("3957f3e2f04a0764c3a0491b175f69926da61efbcc8f61fa1455fd2d2b4cdd45")
+	AccountLeafNode, _ = rlp.EncodeToBytes([]interface{}{
+		AccountPartialPath,
+		Account,
+	})
+
+	StateDiffs = []statediff.StateNode{
+		{
+			Path:      []byte{'\x06'},
+			NodeType:  statediff.Leaf,
+			LeafKey:   ContractLeafKey,
+			NodeValue: ContractLeafNode,
+			StorageNodes: []statediff.StorageNode{
+				{
+					Path:      []byte{},
+					NodeType:  statediff.Leaf,
+					LeafKey:   StorageLeafKey,
+					NodeValue: StorageLeafNode,
+				},
+			},
+		},
+		{
+			Path:         []byte{'\x0c'},
+			NodeType:     statediff.Leaf,
+			LeafKey:      AccountLeafKey,
+			NodeValue:    AccountLeafNode,
+			StorageNodes: []statediff.StorageNode{},
+		},
+	}
+
+	MockStateDiff = statediff.StateObject{
+		BlockNumber: new(big.Int).Set(BlockNumber),
+		BlockHash:   MockBlock.Hash(),
+		Nodes:       StateDiffs,
+	}
+	MockStateDiffBytes, _ = rlp.EncodeToBytes(MockStateDiff)
+	MockStateNodes        = []eth2.TrieNode{
+		{
+			LeafKey: common.BytesToHash(ContractLeafKey),
+			Path:    []byte{'\x06'},
+			Value:   ContractLeafNode,
+			Type:    statediff.Leaf,
+		},
+		{
+			LeafKey: common.BytesToHash(AccountLeafKey),
+			Path:    []byte{'\x0c'},
+			Value:   AccountLeafNode,
+			Type:    statediff.Leaf,
+		},
+	}
+	MockStateMetaPostPublish = []eth2.StateNodeModel{
+		{
+			CID:      State1CID.String(),
+			MhKey:    State1MhKey,
+			Path:     []byte{'\x06'},
+			NodeType: 2,
+			StateKey: common.BytesToHash(ContractLeafKey).Hex(),
+		},
+		{
+			CID:      State2CID.String(),
+			MhKey:    State2MhKey,
+			Path:     []byte{'\x0c'},
+			NodeType: 2,
+			StateKey: common.BytesToHash(AccountLeafKey).Hex(),
+		},
+	}
+	MockStorageNodes = map[string][]eth2.TrieNode{
+		contractPath: {
+			{
+				LeafKey: common.BytesToHash(StorageLeafKey),
+				Value:   StorageLeafNode,
+				Type:    statediff.Leaf,
+				Path:    []byte{},
+			},
+		},
+	}
+
+	MockCIDWrapper = &eth.CIDWrapper{
+		BlockNumber: new(big.Int).Set(BlockNumber),
+		Header: eth2.HeaderModel{
+			BlockNumber:     "1",
+			BlockHash:       MockBlock.Hash().String(),
+			ParentHash:      "0x0000000000000000000000000000000000000000000000000000000000000000",
+			CID:             HeaderCID.String(),
+			MhKey:           HeaderMhKey,
+			TotalDifficulty: MockBlock.Difficulty().String(),
+			Reward:          "5000000000000000000",
+			StateRoot:       MockBlock.Root().String(),
+			RctRoot:         MockBlock.ReceiptHash().String(),
+			TxRoot:          MockBlock.TxHash().String(),
+			UncleRoot:       MockBlock.UncleHash().String(),
+			Bloom:           MockBlock.Bloom().Bytes(),
+			Timestamp:       MockBlock.Time(),
+			TimesValidated:  1,
+		},
+		Transactions: MockTrxMetaPostPublsh,
+		Receipts:     MockRctMetaPostPublish,
+		Uncles:       []eth2.UncleModel{},
+		StateNodes:   MockStateMetaPostPublish,
+		StorageNodes: []eth2.StorageNodeWithStateKeyModel{
+			{
+				Path:       []byte{},
+				CID:        StorageCID.String(),
+				MhKey:      StorageMhKey,
+				NodeType:   2,
+				StateKey:   common.BytesToHash(ContractLeafKey).Hex(),
+				StorageKey: common.BytesToHash(StorageLeafKey).Hex(),
+			},
+		},
+	}
+
+	HeaderIPLD, _  = blocks.NewBlockWithCid(MockHeaderRlp, HeaderCID)
+	Trx1IPLD, _    = blocks.NewBlockWithCid(MockTransactions.GetRlp(0), Trx1CID)
+	Trx2IPLD, _    = blocks.NewBlockWithCid(MockTransactions.GetRlp(1), Trx2CID)
+	Trx3IPLD, _    = blocks.NewBlockWithCid(MockTransactions.GetRlp(2), Trx3CID)
+	Rct1IPLD, _    = blocks.NewBlockWithCid(MockReceipts.GetRlp(0), Rct1CID)
+	Rct2IPLD, _    = blocks.NewBlockWithCid(MockReceipts.GetRlp(1), Rct2CID)
+	Rct3IPLD, _    = blocks.NewBlockWithCid(MockReceipts.GetRlp(2), Rct3CID)
+	State1IPLD, _  = blocks.NewBlockWithCid(ContractLeafNode, State1CID)
+	State2IPLD, _  = blocks.NewBlockWithCid(AccountLeafNode, State2CID)
+	StorageIPLD, _ = blocks.NewBlockWithCid(StorageLeafNode, StorageCID)
+
+	MockIPLDs = eth.IPLDs{
+		BlockNumber: new(big.Int).Set(BlockNumber),
+		Header: ipfs.BlockModel{
+			Data: HeaderIPLD.RawData(),
+			CID:  HeaderIPLD.Cid().String(),
+		},
+		Transactions: []ipfs.BlockModel{
+			{
+				Data: Trx1IPLD.RawData(),
+				CID:  Trx1IPLD.Cid().String(),
+			},
+			{
+				Data: Trx2IPLD.RawData(),
+				CID:  Trx2IPLD.Cid().String(),
+			},
+			{
+				Data: Trx3IPLD.RawData(),
+				CID:  Trx3IPLD.Cid().String(),
+			},
+		},
+		Receipts: []ipfs.BlockModel{
+			{
+				Data: Rct1IPLD.RawData(),
+				CID:  Rct1IPLD.Cid().String(),
+			},
+			{
+				Data: Rct2IPLD.RawData(),
+				CID:  Rct2IPLD.Cid().String(),
+			},
+			{
+				Data: Rct3IPLD.RawData(),
+				CID:  Rct3IPLD.Cid().String(),
+			},
+		},
+		StateNodes: []eth.StateNode{
+			{
+				StateLeafKey: common.BytesToHash(ContractLeafKey),
+				Type:         statediff.Leaf,
+				IPLD: ipfs.BlockModel{
+					Data: State1IPLD.RawData(),
+					CID:  State1IPLD.Cid().String(),
+				},
+				Path: []byte{'\x06'},
+			},
+			{
+				StateLeafKey: common.BytesToHash(AccountLeafKey),
+				Type:         statediff.Leaf,
+				IPLD: ipfs.BlockModel{
+					Data: State2IPLD.RawData(),
+					CID:  State2IPLD.Cid().String(),
+				},
+				Path: []byte{'\x0c'},
+			},
+		},
+		StorageNodes: []eth.StorageNode{
+			{
+				StateLeafKey:   common.BytesToHash(ContractLeafKey),
+				StorageLeafKey: common.BytesToHash(StorageLeafKey),
+				Type:           statediff.Leaf,
+				IPLD: ipfs.BlockModel{
+					Data: StorageIPLD.RawData(),
+					CID:  StorageIPLD.Cid().String(),
+				},
+				Path: []byte{},
+			},
+		},
+	}
+)
+
+// createTransactionsAndReceipts is a helper function to generate signed mock transactions and mock receipts with mock logs
+func createTransactionsAndReceipts() (types.Transactions, types.Receipts, common.Address) {
+	// make transactions
+	trx1 := types.NewTransaction(0, Address, big.NewInt(1000), 50, big.NewInt(100), []byte{})
+	trx2 := types.NewTransaction(1, AnotherAddress, big.NewInt(2000), 100, big.NewInt(200), []byte{})
+	trx3 := types.NewContractCreation(2, big.NewInt(1500), 75, big.NewInt(150), MockContractByteCode)
+	transactionSigner := types.MakeSigner(params.MainnetChainConfig, new(big.Int).Set(BlockNumber))
+	mockCurve := elliptic.P256()
+	mockPrvKey, err := ecdsa.GenerateKey(mockCurve, rand.Reader)
+	if err != nil {
+		log.Fatal(err)
+	}
+	signedTrx1, err := types.SignTx(trx1, transactionSigner, mockPrvKey)
+	if err != nil {
+		log.Fatal(err)
+	}
+	signedTrx2, err := types.SignTx(trx2, transactionSigner, mockPrvKey)
+	if err != nil {
+		log.Fatal(err)
+	}
+	signedTrx3, err := types.SignTx(trx3, transactionSigner, mockPrvKey)
+	if err != nil {
+		log.Fatal(err)
+	}
+	SenderAddr, err := types.Sender(transactionSigner, signedTrx1) // same for both trx
+	if err != nil {
+		log.Fatal(err)
+	}
+	// make receipts
+	mockReceipt1 := types.NewReceipt(common.HexToHash("0x0").Bytes(), false, 50)
+	mockReceipt1.Logs = []*types.Log{MockLog1}
+	mockReceipt1.TxHash = signedTrx1.Hash()
+	mockReceipt2 := types.NewReceipt(common.HexToHash("0x1").Bytes(), false, 100)
+	mockReceipt2.Logs = []*types.Log{MockLog2}
+	mockReceipt2.TxHash = signedTrx2.Hash()
+	mockReceipt3 := types.NewReceipt(common.HexToHash("0x2").Bytes(), false, 75)
+	mockReceipt3.Logs = []*types.Log{}
+	mockReceipt3.TxHash = signedTrx3.Hash()
+	return types.Transactions{signedTrx1, signedTrx2, signedTrx3}, types.Receipts{mockReceipt1, mockReceipt2, mockReceipt3}, SenderAddr
+}

pkg/eth/types.go

@@ -0,0 +1,50 @@
+package eth
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/statediff"
+	eth2 "github.com/vulcanize/ipld-eth-indexer/pkg/eth"
+	"github.com/vulcanize/ipld-eth-indexer/pkg/ipfs"
+	"math/big"
+)
+
+// CIDWrapper is used to direct fetching of IPLDs from IPFS
+// Returned by CIDRetriever
+// Passed to IPLDFetcher
+type CIDWrapper struct {
+	BlockNumber  *big.Int
+	Header       eth2.HeaderModel
+	Uncles       []eth2.UncleModel
+	Transactions []eth2.TxModel
+	Receipts     []eth2.ReceiptModel
+	StateNodes   []eth2.StateNodeModel
+	StorageNodes []eth2.StorageNodeWithStateKeyModel
+}
+
+// IPLDs is used to package raw IPLD block data fetched from IPFS and returned by the server
+// Returned by IPLDFetcher and ResponseFilterer
+type IPLDs struct {
+	BlockNumber     *big.Int
+	TotalDifficulty *big.Int
+	Header          ipfs.BlockModel
+	Uncles          []ipfs.BlockModel
+	Transactions    []ipfs.BlockModel
+	Receipts        []ipfs.BlockModel
+	StateNodes      []StateNode
+	StorageNodes    []StorageNode
+}
+
+type StateNode struct {
+	Type         statediff.NodeType
+	StateLeafKey common.Hash
+	Path         []byte
+	IPLD         ipfs.BlockModel
+}
+
+type StorageNode struct {
+	Type           statediff.NodeType
+	StateLeafKey   common.Hash
+	StorageLeafKey common.Hash
+	Path           []byte
+	IPLD           ipfs.BlockModel
+}

pkg/shared/test_helpers.go

@@ -34,7 +34,9 @@ func SetupDB() (*postgres.DB, error) {
 	return postgres.NewDB(postgres.Config{
 		Hostname: "localhost",
 		Name:     "vulcanize_testing",
-		Port:     5432,
+		Port:     8077,
+		User:     "vdbm",
+		Password: "password",
 	}, node.Info{})
 }