cerc-io/ipld-eth-server
18
0
Fork 1
Code Issues 10 Pull Requests 1 Actions Packages Projects Releases 37 Wiki Activity

unit test for ipfs fetcher

Browse Source
This commit is contained in:
Ian Norden 2019-08-23 16:34:54 -05:00
parent 069646905e
commit d79cc90cb2
5 changed files with 214 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
integration_test/seed_node.go
View File

@ -1 +0,0 @@
package integration

15
pkg/ipfs/fetcher.go
View File

@ -63,26 +63,32 @@ func (f *EthIPLDFetcher) FetchCIDs(cids CIDWrapper) (*IPLDWrapper, error) {
err := f.fetchHeaders(cids, blocks)
if err != nil {
println(1)
return nil, err
}
err = f.fetchUncles(cids, blocks)
if err != nil {
println(2)
return nil, err
}
err = f.fetchTrxs(cids, blocks)
if err != nil {
println(3)
return nil, err
}
err = f.fetchRcts(cids, blocks)
if err != nil {
println(4)
return nil, err
}
err = f.fetchStorage(cids, blocks)
if err != nil {
println(5)
return nil, err
}
err = f.fetchState(cids, blocks)
if err != nil {
println(6)
return nil, err
}
@ -190,7 +196,7 @@ func (f *EthIPLDFetcher) fetchState(cids CIDWrapper, blocks *IPLDWrapper) error
// fetchStorage fetches storage nodes
// It uses the single f.fetch method instead of the batch fetch, because it
// needs to maintain the data's relation to state and storage keys
func (f *EthIPLDFetcher) fetchStorage(cids CIDWrapper, blocks *IPLDWrapper) error {
func (f *EthIPLDFetcher) fetchStorage(cids CIDWrapper, blks *IPLDWrapper) error {
log.Debug("fetching storage iplds")
for _, storageNode := range cids.StorageNodes {
if storageNode.CID == "" || storageNode.Key == "" || storageNode.StateKey == "" {
@ -200,11 +206,14 @@ func (f *EthIPLDFetcher) fetchStorage(cids CIDWrapper, blocks *IPLDWrapper) erro
if err != nil {
return err
}
block, err := f.fetch(dc)
blk, err := f.fetch(dc)
if err != nil {
return err
}
blocks.StorageNodes[common.HexToHash(storageNode.StateKey)][common.HexToHash(storageNode.Key)] = block
if blks.StorageNodes[common.HexToHash(storageNode.StateKey)] == nil {
blks.StorageNodes[common.HexToHash(storageNode.StateKey)] = make(map[common.Hash]blocks.Block)
}
blks.StorageNodes[common.HexToHash(storageNode.StateKey)][common.HexToHash(storageNode.Key)] = blk
}
return nil
}

113
pkg/ipfs/fetcher_test.go Normal file
View File

@ -0,0 +1,113 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package ipfs_test
import (
"math/big"
"github.com/ethereum/go-ethereum/common"
"github.com/ipfs/go-block-format"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/ipfs"
"github.com/vulcanize/vulcanizedb/pkg/ipfs/mocks"
)
var (
// these need to be actual typed objects so that the cid.Decode works
mockHeaderData = []byte{0, 1, 2, 3, 4}
mockUncleData = []byte{1, 2, 3, 4, 5}
mockTrxData = []byte{2, 3, 4, 5, 6}
mockReceiptData = []byte{3, 4, 5, 6, 7}
mockStateData = []byte{4, 5, 6, 7, 8}
mockStorageData = []byte{5, 6, 7, 8, 9}
mockStorageData2 = []byte{6, 7, 8, 9, 1}
mockHeaderBlock = blocks.NewBlock(mockHeaderData)
mockUncleBlock = blocks.NewBlock(mockUncleData)
mockTrxBlock = blocks.NewBlock(mockTrxData)
mockReceiptBlock = blocks.NewBlock(mockReceiptData)
mockStateBlock = blocks.NewBlock(mockStateData)
mockStorageBlock1 = blocks.NewBlock(mockStorageData)
mockStorageBlock2 = blocks.NewBlock(mockStorageData2)
mockBlocks = []blocks.Block{mockHeaderBlock, mockUncleBlock, mockTrxBlock, mockReceiptBlock, mockStateBlock, mockStorageBlock1, mockStorageBlock2}
mockBlockService *mocks.MockIPFSBlockService
mockCIDWrapper = ipfs.CIDWrapper{
BlockNumber: big.NewInt(9000),
Headers: []string{mockHeaderBlock.Cid().String()},
Uncles: []string{mockUncleBlock.Cid().String()},
Transactions: []string{mockTrxBlock.Cid().String()},
Receipts: []string{mockReceiptBlock.Cid().String()},
StateNodes: []ipfs.StateNodeCID{{
CID: mockStateBlock.Cid().String(),
Leaf: true,
Key: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470",
}},
StorageNodes: []ipfs.StorageNodeCID{{
CID: mockStorageBlock1.Cid().String(),
Leaf: true,
StateKey: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470",
Key: "0000000000000000000000000000000000000000000000000000000000000001",
},
{
CID: mockStorageBlock2.Cid().String(),
Leaf: true,
StateKey: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470",
Key: "0000000000000000000000000000000000000000000000000000000000000002",
}},
}
)
var _ = Describe("Fetcher", func() {
Describe("FetchCIDs", func() {
BeforeEach(func() {
mockBlockService = new(mocks.MockIPFSBlockService)
err := mockBlockService.AddBlocks(mockBlocks)
Expect(err).ToNot(HaveOccurred())
Expect(len(mockBlockService.Blocks)).To(Equal(7))
})
It("Fetches and returns IPLDs for the CIDs provided in the CIDWrapper", func() {
fetcher := new(ipfs.EthIPLDFetcher)
fetcher.BlockService = mockBlockService
iplds, err := fetcher.FetchCIDs(mockCIDWrapper)
Expect(err).ToNot(HaveOccurred())
Expect(iplds.BlockNumber).To(Equal(mockCIDWrapper.BlockNumber))
Expect(len(iplds.Headers)).To(Equal(1))
Expect(iplds.Headers[0]).To(Equal(mockHeaderBlock))
Expect(len(iplds.Uncles)).To(Equal(1))
Expect(iplds.Uncles[0]).To(Equal(mockUncleBlock))
Expect(len(iplds.Transactions)).To(Equal(1))
Expect(iplds.Transactions[0]).To(Equal(mockTrxBlock))
Expect(len(iplds.Receipts)).To(Equal(1))
Expect(iplds.Receipts[0]).To(Equal(mockReceiptBlock))
Expect(len(iplds.StateNodes)).To(Equal(1))
stateNode, ok := iplds.StateNodes[common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")]
Expect(ok).To(BeTrue())
Expect(stateNode).To(Equal(mockStateBlock))
Expect(len(iplds.StorageNodes)).To(Equal(1))
storageNodes := iplds.StorageNodes[common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")]
Expect(len(storageNodes)).To(Equal(2))
storageNode1, ok := storageNodes[common.HexToHash("0000000000000000000000000000000000000000000000000000000000000001")]
Expect(ok).To(BeTrue())
Expect(storageNode1).To(Equal(mockStorageBlock1))
storageNode2, ok := storageNodes[common.HexToHash("0000000000000000000000000000000000000000000000000000000000000002")]
Expect(storageNode2).To(Equal(mockStorageBlock2))
Expect(ok).To(BeTrue())
})
})
})

86
pkg/ipfs/mocks/blockservice.go Normal file
View File

@ -0,0 +1,86 @@
package mocks
import (
"context"
"errors"
"github.com/ipfs/go-block-format"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-ipfs-blockstore"
"github.com/ipfs/go-ipfs-exchange-interface"
)
// MockIPFSBlockService is a mock for testing the ipfs fetcher
type MockIPFSBlockService struct {
Blocks map[cid.Cid]blocks.Block
}
// GetBlock is used to retrieve a block from the mock BlockService
func (bs *MockIPFSBlockService) GetBlock(ctx context.Context, c cid.Cid) (blocks.Block, error) {
if bs.Blocks == nil {
return nil, errors.New("BlockService has not been initialized")
}
blk, ok := bs.Blocks[c]
if ok {
return blk, nil
}
return nil, nil
}
// GetBlocks is used to retrieve a set of blocks from the mock BlockService
func (bs *MockIPFSBlockService) GetBlocks(ctx context.Context, cs []cid.Cid) <-chan blocks.Block {
if bs.Blocks == nil {
panic("BlockService has not been initialized")
}
blkChan := make(chan blocks.Block)
go func() {
for _, c := range cs {
blk, ok := bs.Blocks[c]
if ok {
blkChan <- blk
}
}
close(blkChan)
}()
return blkChan
}
// AddBlock adds a block to the mock BlockService
func (bs *MockIPFSBlockService) AddBlock(blk blocks.Block) error {
if bs.Blocks == nil {
bs.Blocks = make(map[cid.Cid]blocks.Block)
}
bs.Blocks[blk.Cid()] = blk
return nil
}
// AddBlocks adds a set of blocks to the mock BlockService
func (bs *MockIPFSBlockService) AddBlocks(blks []blocks.Block) error {
if bs.Blocks == nil {
bs.Blocks = make(map[cid.Cid]blocks.Block)
}
for _, block := range blks {
bs.Blocks[block.Cid()] = block
}
return nil
}
// Close is here to satisfy the interface
func (*MockIPFSBlockService) Close() error {
panic("implement me")
}
// Blockstore is here to satisfy the interface
func (*MockIPFSBlockService) Blockstore() blockstore.Blockstore {
panic("implement me")
}
// DeleteBlock is here to satisfy the interface
func (*MockIPFSBlockService) DeleteBlock(c cid.Cid) error {
panic("implement me")
}
// Exchange is here to satisfy the interface
func (*MockIPFSBlockService) Exchange() exchange.Interface {
panic("implement me")
}

6
pkg/ipfs/types.go
View File

@ -24,7 +24,7 @@ import (
"github.com/ipfs/go-block-format"
)
// CIDWrapper is used to package CIDs retrieved from the local Postgres cache
// CIDWrapper is used to package CIDs retrieved from the local Postgres cache and direct fetching of IPLDs
type CIDWrapper struct {
BlockNumber *big.Int
Headers []string
@ -35,7 +35,7 @@ type CIDWrapper struct {
StorageNodes []StorageNodeCID
}
// IPLDWrapper is used to package raw IPLD block data for resolution
// IPLDWrapper is used to package raw IPLD block data fetched from IPFS
type IPLDWrapper struct {
BlockNumber *big.Int
Headers []blocks.Block
@ -46,7 +46,7 @@ type IPLDWrapper struct {
StorageNodes map[common.Hash]map[common.Hash]blocks.Block
}
// IPLDPayload is a custom type which packages ETH data for the IPFS publisher
// IPLDPayload is a custom type which packages raw ETH data for the IPFS publisher
type IPLDPayload struct {
HeaderRLP []byte
BlockNumber *big.Int