From d79cc90cb24633f825ee5eedacf37c4aab8ceafd Mon Sep 17 00:00:00 2001 From: Ian Norden Date: Fri, 23 Aug 2019 16:34:54 -0500 Subject: [PATCH] unit test for ipfs fetcher --- integration_test/seed_node.go | 1 - pkg/ipfs/fetcher.go | 15 ++++- pkg/ipfs/fetcher_test.go | 113 +++++++++++++++++++++++++++++++++ pkg/ipfs/mocks/blockservice.go | 86 +++++++++++++++++++++++++ pkg/ipfs/types.go | 6 +- 5 files changed, 214 insertions(+), 7 deletions(-) delete mode 100644 integration_test/seed_node.go create mode 100644 pkg/ipfs/fetcher_test.go create mode 100644 pkg/ipfs/mocks/blockservice.go diff --git a/integration_test/seed_node.go b/integration_test/seed_node.go deleted file mode 100644 index 76ab1b72..00000000 --- a/integration_test/seed_node.go +++ /dev/null @@ -1 +0,0 @@ -package integration diff --git a/pkg/ipfs/fetcher.go b/pkg/ipfs/fetcher.go index 9ac58001..5a65e893 100644 --- a/pkg/ipfs/fetcher.go +++ b/pkg/ipfs/fetcher.go @@ -63,26 +63,32 @@ func (f *EthIPLDFetcher) FetchCIDs(cids CIDWrapper) (*IPLDWrapper, error) { err := f.fetchHeaders(cids, blocks) if err != nil { + println(1) return nil, err } err = f.fetchUncles(cids, blocks) if err != nil { + println(2) return nil, err } err = f.fetchTrxs(cids, blocks) if err != nil { + println(3) return nil, err } err = f.fetchRcts(cids, blocks) if err != nil { + println(4) return nil, err } err = f.fetchStorage(cids, blocks) if err != nil { + println(5) return nil, err } err = f.fetchState(cids, blocks) if err != nil { + println(6) return nil, err } @@ -190,7 +196,7 @@ func (f *EthIPLDFetcher) fetchState(cids CIDWrapper, blocks *IPLDWrapper) error // fetchStorage fetches storage nodes // It uses the single f.fetch method instead of the batch fetch, because it // needs to maintain the data's relation to state and storage keys -func (f *EthIPLDFetcher) fetchStorage(cids CIDWrapper, blocks *IPLDWrapper) error { +func (f *EthIPLDFetcher) fetchStorage(cids CIDWrapper, blks *IPLDWrapper) error { log.Debug("fetching storage iplds") for _, storageNode := range cids.StorageNodes { if storageNode.CID == "" || storageNode.Key == "" || storageNode.StateKey == "" { @@ -200,11 +206,14 @@ func (f *EthIPLDFetcher) fetchStorage(cids CIDWrapper, blocks *IPLDWrapper) erro if err != nil { return err } - block, err := f.fetch(dc) + blk, err := f.fetch(dc) if err != nil { return err } - blocks.StorageNodes[common.HexToHash(storageNode.StateKey)][common.HexToHash(storageNode.Key)] = block + if blks.StorageNodes[common.HexToHash(storageNode.StateKey)] == nil { + blks.StorageNodes[common.HexToHash(storageNode.StateKey)] = make(map[common.Hash]blocks.Block) + } + blks.StorageNodes[common.HexToHash(storageNode.StateKey)][common.HexToHash(storageNode.Key)] = blk } return nil } diff --git a/pkg/ipfs/fetcher_test.go b/pkg/ipfs/fetcher_test.go new file mode 100644 index 00000000..e582630d --- /dev/null +++ b/pkg/ipfs/fetcher_test.go @@ -0,0 +1,113 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ipfs_test + +import ( + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/ipfs/go-block-format" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vulcanize/vulcanizedb/pkg/ipfs" + "github.com/vulcanize/vulcanizedb/pkg/ipfs/mocks" +) + +var ( + // these need to be actual typed objects so that the cid.Decode works + mockHeaderData = []byte{0, 1, 2, 3, 4} + mockUncleData = []byte{1, 2, 3, 4, 5} + mockTrxData = []byte{2, 3, 4, 5, 6} + mockReceiptData = []byte{3, 4, 5, 6, 7} + mockStateData = []byte{4, 5, 6, 7, 8} + mockStorageData = []byte{5, 6, 7, 8, 9} + mockStorageData2 = []byte{6, 7, 8, 9, 1} + mockHeaderBlock = blocks.NewBlock(mockHeaderData) + mockUncleBlock = blocks.NewBlock(mockUncleData) + mockTrxBlock = blocks.NewBlock(mockTrxData) + mockReceiptBlock = blocks.NewBlock(mockReceiptData) + mockStateBlock = blocks.NewBlock(mockStateData) + mockStorageBlock1 = blocks.NewBlock(mockStorageData) + mockStorageBlock2 = blocks.NewBlock(mockStorageData2) + mockBlocks = []blocks.Block{mockHeaderBlock, mockUncleBlock, mockTrxBlock, mockReceiptBlock, mockStateBlock, mockStorageBlock1, mockStorageBlock2} + mockBlockService *mocks.MockIPFSBlockService + mockCIDWrapper = ipfs.CIDWrapper{ + BlockNumber: big.NewInt(9000), + Headers: []string{mockHeaderBlock.Cid().String()}, + Uncles: []string{mockUncleBlock.Cid().String()}, + Transactions: []string{mockTrxBlock.Cid().String()}, + Receipts: []string{mockReceiptBlock.Cid().String()}, + StateNodes: []ipfs.StateNodeCID{{ + CID: mockStateBlock.Cid().String(), + Leaf: true, + Key: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", + }}, + StorageNodes: []ipfs.StorageNodeCID{{ + CID: mockStorageBlock1.Cid().String(), + Leaf: true, + StateKey: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", + Key: "0000000000000000000000000000000000000000000000000000000000000001", + }, + { + CID: mockStorageBlock2.Cid().String(), + Leaf: true, + StateKey: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470", + Key: "0000000000000000000000000000000000000000000000000000000000000002", + }}, + } +) + +var _ = Describe("Fetcher", func() { + Describe("FetchCIDs", func() { + BeforeEach(func() { + mockBlockService = new(mocks.MockIPFSBlockService) + err := mockBlockService.AddBlocks(mockBlocks) + Expect(err).ToNot(HaveOccurred()) + Expect(len(mockBlockService.Blocks)).To(Equal(7)) + }) + + It("Fetches and returns IPLDs for the CIDs provided in the CIDWrapper", func() { + fetcher := new(ipfs.EthIPLDFetcher) + fetcher.BlockService = mockBlockService + iplds, err := fetcher.FetchCIDs(mockCIDWrapper) + Expect(err).ToNot(HaveOccurred()) + Expect(iplds.BlockNumber).To(Equal(mockCIDWrapper.BlockNumber)) + Expect(len(iplds.Headers)).To(Equal(1)) + Expect(iplds.Headers[0]).To(Equal(mockHeaderBlock)) + Expect(len(iplds.Uncles)).To(Equal(1)) + Expect(iplds.Uncles[0]).To(Equal(mockUncleBlock)) + Expect(len(iplds.Transactions)).To(Equal(1)) + Expect(iplds.Transactions[0]).To(Equal(mockTrxBlock)) + Expect(len(iplds.Receipts)).To(Equal(1)) + Expect(iplds.Receipts[0]).To(Equal(mockReceiptBlock)) + Expect(len(iplds.StateNodes)).To(Equal(1)) + stateNode, ok := iplds.StateNodes[common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")] + Expect(ok).To(BeTrue()) + Expect(stateNode).To(Equal(mockStateBlock)) + Expect(len(iplds.StorageNodes)).To(Equal(1)) + storageNodes := iplds.StorageNodes[common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")] + Expect(len(storageNodes)).To(Equal(2)) + storageNode1, ok := storageNodes[common.HexToHash("0000000000000000000000000000000000000000000000000000000000000001")] + Expect(ok).To(BeTrue()) + Expect(storageNode1).To(Equal(mockStorageBlock1)) + storageNode2, ok := storageNodes[common.HexToHash("0000000000000000000000000000000000000000000000000000000000000002")] + Expect(storageNode2).To(Equal(mockStorageBlock2)) + Expect(ok).To(BeTrue()) + }) + }) +}) diff --git a/pkg/ipfs/mocks/blockservice.go b/pkg/ipfs/mocks/blockservice.go new file mode 100644 index 00000000..fdab2fd9 --- /dev/null +++ b/pkg/ipfs/mocks/blockservice.go @@ -0,0 +1,86 @@ +package mocks + +import ( + "context" + "errors" + + "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + "github.com/ipfs/go-ipfs-blockstore" + "github.com/ipfs/go-ipfs-exchange-interface" +) + +// MockIPFSBlockService is a mock for testing the ipfs fetcher +type MockIPFSBlockService struct { + Blocks map[cid.Cid]blocks.Block +} + +// GetBlock is used to retrieve a block from the mock BlockService +func (bs *MockIPFSBlockService) GetBlock(ctx context.Context, c cid.Cid) (blocks.Block, error) { + if bs.Blocks == nil { + return nil, errors.New("BlockService has not been initialized") + } + blk, ok := bs.Blocks[c] + if ok { + return blk, nil + } + return nil, nil +} + +// GetBlocks is used to retrieve a set of blocks from the mock BlockService +func (bs *MockIPFSBlockService) GetBlocks(ctx context.Context, cs []cid.Cid) <-chan blocks.Block { + if bs.Blocks == nil { + panic("BlockService has not been initialized") + } + blkChan := make(chan blocks.Block) + go func() { + for _, c := range cs { + blk, ok := bs.Blocks[c] + if ok { + blkChan <- blk + } + } + close(blkChan) + }() + return blkChan +} + +// AddBlock adds a block to the mock BlockService +func (bs *MockIPFSBlockService) AddBlock(blk blocks.Block) error { + if bs.Blocks == nil { + bs.Blocks = make(map[cid.Cid]blocks.Block) + } + bs.Blocks[blk.Cid()] = blk + return nil +} + +// AddBlocks adds a set of blocks to the mock BlockService +func (bs *MockIPFSBlockService) AddBlocks(blks []blocks.Block) error { + if bs.Blocks == nil { + bs.Blocks = make(map[cid.Cid]blocks.Block) + } + for _, block := range blks { + bs.Blocks[block.Cid()] = block + } + return nil +} + +// Close is here to satisfy the interface +func (*MockIPFSBlockService) Close() error { + panic("implement me") +} + +// Blockstore is here to satisfy the interface +func (*MockIPFSBlockService) Blockstore() blockstore.Blockstore { + panic("implement me") +} + +// DeleteBlock is here to satisfy the interface +func (*MockIPFSBlockService) DeleteBlock(c cid.Cid) error { + panic("implement me") +} + +// Exchange is here to satisfy the interface +func (*MockIPFSBlockService) Exchange() exchange.Interface { + panic("implement me") +} diff --git a/pkg/ipfs/types.go b/pkg/ipfs/types.go index d560fac4..95b3cb60 100644 --- a/pkg/ipfs/types.go +++ b/pkg/ipfs/types.go @@ -24,7 +24,7 @@ import ( "github.com/ipfs/go-block-format" ) -// CIDWrapper is used to package CIDs retrieved from the local Postgres cache +// CIDWrapper is used to package CIDs retrieved from the local Postgres cache and direct fetching of IPLDs type CIDWrapper struct { BlockNumber *big.Int Headers []string @@ -35,7 +35,7 @@ type CIDWrapper struct { StorageNodes []StorageNodeCID } -// IPLDWrapper is used to package raw IPLD block data for resolution +// IPLDWrapper is used to package raw IPLD block data fetched from IPFS type IPLDWrapper struct { BlockNumber *big.Int Headers []blocks.Block @@ -46,7 +46,7 @@ type IPLDWrapper struct { StorageNodes map[common.Hash]map[common.Hash]blocks.Block } -// IPLDPayload is a custom type which packages ETH data for the IPFS publisher +// IPLDPayload is a custom type which packages raw ETH data for the IPFS publisher type IPLDPayload struct { HeaderRLP []byte BlockNumber *big.Int