unit test for ipfs fetcher

integration_test/seed_node.go

@@ -1 +0,0 @@
-package integration

pkg/ipfs/fetcher.go

@@ -63,26 +63,32 @@ func (f *EthIPLDFetcher) FetchCIDs(cids CIDWrapper) (*IPLDWrapper, error) {
 
 	err := f.fetchHeaders(cids, blocks)
 	if err != nil {
+		println(1)
 		return nil, err
 	}
 	err = f.fetchUncles(cids, blocks)
 	if err != nil {
+		println(2)
 		return nil, err
 	}
 	err = f.fetchTrxs(cids, blocks)
 	if err != nil {
+		println(3)
 		return nil, err
 	}
 	err = f.fetchRcts(cids, blocks)
 	if err != nil {
+		println(4)
 		return nil, err
 	}
 	err = f.fetchStorage(cids, blocks)
 	if err != nil {
+		println(5)
 		return nil, err
 	}
 	err = f.fetchState(cids, blocks)
 	if err != nil {
+		println(6)
 		return nil, err
 	}
 
@@ -190,7 +196,7 @@ func (f *EthIPLDFetcher) fetchState(cids CIDWrapper, blocks *IPLDWrapper) error
 // fetchStorage fetches storage nodes
 // It uses the single f.fetch method instead of the batch fetch, because it
 // needs to maintain the data's relation to state and storage keys
-func (f *EthIPLDFetcher) fetchStorage(cids CIDWrapper, blocks *IPLDWrapper) error {
+func (f *EthIPLDFetcher) fetchStorage(cids CIDWrapper, blks *IPLDWrapper) error {
 	log.Debug("fetching storage iplds")
 	for _, storageNode := range cids.StorageNodes {
 		if storageNode.CID == "" || storageNode.Key == "" || storageNode.StateKey == "" {
@@ -200,11 +206,14 @@ func (f *EthIPLDFetcher) fetchStorage(cids CIDWrapper, blocks *IPLDWrapper) erro
 		if err != nil {
 			return err
 		}
-		block, err := f.fetch(dc)
+		blk, err := f.fetch(dc)
 		if err != nil {
 			return err
 		}
-		blocks.StorageNodes[common.HexToHash(storageNode.StateKey)][common.HexToHash(storageNode.Key)] = block
+		if blks.StorageNodes[common.HexToHash(storageNode.StateKey)] == nil {
+			blks.StorageNodes[common.HexToHash(storageNode.StateKey)] = make(map[common.Hash]blocks.Block)
+		}
+		blks.StorageNodes[common.HexToHash(storageNode.StateKey)][common.HexToHash(storageNode.Key)] = blk
 	}
 	return nil
 }

pkg/ipfs/fetcher_test.go

@@ -0,0 +1,113 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package ipfs_test
+
+import (
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ipfs/go-block-format"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
+	"github.com/vulcanize/vulcanizedb/pkg/ipfs/mocks"
+)
+
+var (
+	// these need to be actual typed objects so that the cid.Decode works
+	mockHeaderData    = []byte{0, 1, 2, 3, 4}
+	mockUncleData     = []byte{1, 2, 3, 4, 5}
+	mockTrxData       = []byte{2, 3, 4, 5, 6}
+	mockReceiptData   = []byte{3, 4, 5, 6, 7}
+	mockStateData     = []byte{4, 5, 6, 7, 8}
+	mockStorageData   = []byte{5, 6, 7, 8, 9}
+	mockStorageData2  = []byte{6, 7, 8, 9, 1}
+	mockHeaderBlock   = blocks.NewBlock(mockHeaderData)
+	mockUncleBlock    = blocks.NewBlock(mockUncleData)
+	mockTrxBlock      = blocks.NewBlock(mockTrxData)
+	mockReceiptBlock  = blocks.NewBlock(mockReceiptData)
+	mockStateBlock    = blocks.NewBlock(mockStateData)
+	mockStorageBlock1 = blocks.NewBlock(mockStorageData)
+	mockStorageBlock2 = blocks.NewBlock(mockStorageData2)
+	mockBlocks        = []blocks.Block{mockHeaderBlock, mockUncleBlock, mockTrxBlock, mockReceiptBlock, mockStateBlock, mockStorageBlock1, mockStorageBlock2}
+	mockBlockService  *mocks.MockIPFSBlockService
+	mockCIDWrapper    = ipfs.CIDWrapper{
+		BlockNumber:  big.NewInt(9000),
+		Headers:      []string{mockHeaderBlock.Cid().String()},
+		Uncles:       []string{mockUncleBlock.Cid().String()},
+		Transactions: []string{mockTrxBlock.Cid().String()},
+		Receipts:     []string{mockReceiptBlock.Cid().String()},
+		StateNodes: []ipfs.StateNodeCID{{
+			CID:  mockStateBlock.Cid().String(),
+			Leaf: true,
+			Key:  "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470",
+		}},
+		StorageNodes: []ipfs.StorageNodeCID{{
+			CID:      mockStorageBlock1.Cid().String(),
+			Leaf:     true,
+			StateKey: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470",
+			Key:      "0000000000000000000000000000000000000000000000000000000000000001",
+		},
+			{
+				CID:      mockStorageBlock2.Cid().String(),
+				Leaf:     true,
+				StateKey: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470",
+				Key:      "0000000000000000000000000000000000000000000000000000000000000002",
+			}},
+	}
+)
+
+var _ = Describe("Fetcher", func() {
+	Describe("FetchCIDs", func() {
+		BeforeEach(func() {
+			mockBlockService = new(mocks.MockIPFSBlockService)
+			err := mockBlockService.AddBlocks(mockBlocks)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(mockBlockService.Blocks)).To(Equal(7))
+		})
+
+		It("Fetches and returns IPLDs for the CIDs provided in the CIDWrapper", func() {
+			fetcher := new(ipfs.EthIPLDFetcher)
+			fetcher.BlockService = mockBlockService
+			iplds, err := fetcher.FetchCIDs(mockCIDWrapper)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(iplds.BlockNumber).To(Equal(mockCIDWrapper.BlockNumber))
+			Expect(len(iplds.Headers)).To(Equal(1))
+			Expect(iplds.Headers[0]).To(Equal(mockHeaderBlock))
+			Expect(len(iplds.Uncles)).To(Equal(1))
+			Expect(iplds.Uncles[0]).To(Equal(mockUncleBlock))
+			Expect(len(iplds.Transactions)).To(Equal(1))
+			Expect(iplds.Transactions[0]).To(Equal(mockTrxBlock))
+			Expect(len(iplds.Receipts)).To(Equal(1))
+			Expect(iplds.Receipts[0]).To(Equal(mockReceiptBlock))
+			Expect(len(iplds.StateNodes)).To(Equal(1))
+			stateNode, ok := iplds.StateNodes[common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")]
+			Expect(ok).To(BeTrue())
+			Expect(stateNode).To(Equal(mockStateBlock))
+			Expect(len(iplds.StorageNodes)).To(Equal(1))
+			storageNodes := iplds.StorageNodes[common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")]
+			Expect(len(storageNodes)).To(Equal(2))
+			storageNode1, ok := storageNodes[common.HexToHash("0000000000000000000000000000000000000000000000000000000000000001")]
+			Expect(ok).To(BeTrue())
+			Expect(storageNode1).To(Equal(mockStorageBlock1))
+			storageNode2, ok := storageNodes[common.HexToHash("0000000000000000000000000000000000000000000000000000000000000002")]
+			Expect(storageNode2).To(Equal(mockStorageBlock2))
+			Expect(ok).To(BeTrue())
+		})
+	})
+})

pkg/ipfs/mocks/blockservice.go

@@ -0,0 +1,86 @@
+package mocks
+
+import (
+	"context"
+	"errors"
+
+	"github.com/ipfs/go-block-format"
+	"github.com/ipfs/go-cid"
+	"github.com/ipfs/go-ipfs-blockstore"
+	"github.com/ipfs/go-ipfs-exchange-interface"
+)
+
+// MockIPFSBlockService is a mock for testing the ipfs fetcher
+type MockIPFSBlockService struct {
+	Blocks map[cid.Cid]blocks.Block
+}
+
+// GetBlock is used to retrieve a block from the mock BlockService
+func (bs *MockIPFSBlockService) GetBlock(ctx context.Context, c cid.Cid) (blocks.Block, error) {
+	if bs.Blocks == nil {
+		return nil, errors.New("BlockService has not been initialized")
+	}
+	blk, ok := bs.Blocks[c]
+	if ok {
+		return blk, nil
+	}
+	return nil, nil
+}
+
+// GetBlocks is used to retrieve a set of blocks from the mock BlockService
+func (bs *MockIPFSBlockService) GetBlocks(ctx context.Context, cs []cid.Cid) <-chan blocks.Block {
+	if bs.Blocks == nil {
+		panic("BlockService has not been initialized")
+	}
+	blkChan := make(chan blocks.Block)
+	go func() {
+		for _, c := range cs {
+			blk, ok := bs.Blocks[c]
+			if ok {
+				blkChan <- blk
+			}
+		}
+		close(blkChan)
+	}()
+	return blkChan
+}
+
+// AddBlock adds a block to the mock BlockService
+func (bs *MockIPFSBlockService) AddBlock(blk blocks.Block) error {
+	if bs.Blocks == nil {
+		bs.Blocks = make(map[cid.Cid]blocks.Block)
+	}
+	bs.Blocks[blk.Cid()] = blk
+	return nil
+}
+
+// AddBlocks adds a set of blocks to the mock BlockService
+func (bs *MockIPFSBlockService) AddBlocks(blks []blocks.Block) error {
+	if bs.Blocks == nil {
+		bs.Blocks = make(map[cid.Cid]blocks.Block)
+	}
+	for _, block := range blks {
+		bs.Blocks[block.Cid()] = block
+	}
+	return nil
+}
+
+// Close is here to satisfy the interface
+func (*MockIPFSBlockService) Close() error {
+	panic("implement me")
+}
+
+// Blockstore is here to satisfy the interface
+func (*MockIPFSBlockService) Blockstore() blockstore.Blockstore {
+	panic("implement me")
+}
+
+// DeleteBlock is here to satisfy the interface
+func (*MockIPFSBlockService) DeleteBlock(c cid.Cid) error {
+	panic("implement me")
+}
+
+// Exchange is here to satisfy the interface
+func (*MockIPFSBlockService) Exchange() exchange.Interface {
+	panic("implement me")
+}

pkg/ipfs/types.go

@@ -24,7 +24,7 @@ import (
 	"github.com/ipfs/go-block-format"
 )
 
-// CIDWrapper is used to package CIDs retrieved from the local Postgres cache
+// CIDWrapper is used to package CIDs retrieved from the local Postgres cache and direct fetching of IPLDs
 type CIDWrapper struct {
 	BlockNumber  *big.Int
 	Headers      []string
@@ -35,7 +35,7 @@ type CIDWrapper struct {
 	StorageNodes []StorageNodeCID
 }
 
-// IPLDWrapper is used to package raw IPLD block data for resolution
+// IPLDWrapper is used to package raw IPLD block data fetched from IPFS
 type IPLDWrapper struct {
 	BlockNumber  *big.Int
 	Headers      []blocks.Block
@@ -46,7 +46,7 @@ type IPLDWrapper struct {
 	StorageNodes map[common.Hash]map[common.Hash]blocks.Block
 }
 
-// IPLDPayload is a custom type which packages ETH data for the IPFS publisher
+// IPLDPayload is a custom type which packages raw ETH data for the IPFS publisher
 type IPLDPayload struct {
 	HeaderRLP       []byte
 	BlockNumber     *big.Int