major refactoring of super_node to make it easier to support other chains

pkg/super_node/eth/mocks/blockservice.go

@@ -0,0 +1,86 @@
+package mocks
+
+import (
+	"context"
+	"errors"
+
+	"github.com/ipfs/go-block-format"
+	"github.com/ipfs/go-cid"
+	"github.com/ipfs/go-ipfs-blockstore"
+	"github.com/ipfs/go-ipfs-exchange-interface"
+)
+
+// MockIPFSBlockService is a mock for testing the ipfs fetcher
+type MockIPFSBlockService struct {
+	Blocks map[cid.Cid]blocks.Block
+}
+
+// GetBlock is used to retrieve a block from the mock BlockService
+func (bs *MockIPFSBlockService) GetBlock(ctx context.Context, c cid.Cid) (blocks.Block, error) {
+	if bs.Blocks == nil {
+		return nil, errors.New("BlockService has not been initialized")
+	}
+	blk, ok := bs.Blocks[c]
+	if ok {
+		return blk, nil
+	}
+	return nil, nil
+}
+
+// GetBlocks is used to retrieve a set of blocks from the mock BlockService
+func (bs *MockIPFSBlockService) GetBlocks(ctx context.Context, cs []cid.Cid) <-chan blocks.Block {
+	if bs.Blocks == nil {
+		panic("BlockService has not been initialized")
+	}
+	blkChan := make(chan blocks.Block)
+	go func() {
+		for _, c := range cs {
+			blk, ok := bs.Blocks[c]
+			if ok {
+				blkChan <- blk
+			}
+		}
+		close(blkChan)
+	}()
+	return blkChan
+}
+
+// AddBlock adds a block to the mock BlockService
+func (bs *MockIPFSBlockService) AddBlock(blk blocks.Block) error {
+	if bs.Blocks == nil {
+		bs.Blocks = make(map[cid.Cid]blocks.Block)
+	}
+	bs.Blocks[blk.Cid()] = blk
+	return nil
+}
+
+// AddBlocks adds a set of blocks to the mock BlockService
+func (bs *MockIPFSBlockService) AddBlocks(blks []blocks.Block) error {
+	if bs.Blocks == nil {
+		bs.Blocks = make(map[cid.Cid]blocks.Block)
+	}
+	for _, block := range blks {
+		bs.Blocks[block.Cid()] = block
+	}
+	return nil
+}
+
+// Close is here to satisfy the interface
+func (*MockIPFSBlockService) Close() error {
+	panic("implement me")
+}
+
+// Blockstore is here to satisfy the interface
+func (*MockIPFSBlockService) Blockstore() blockstore.Blockstore {
+	panic("implement me")
+}
+
+// DeleteBlock is here to satisfy the interface
+func (*MockIPFSBlockService) DeleteBlock(c cid.Cid) error {
+	panic("implement me")
+}
+
+// Exchange is here to satisfy the interface
+func (*MockIPFSBlockService) Exchange() exchange.Interface {
+	panic("implement me")
+}

pkg/super_node/eth/mocks/converter.go

@@ -0,0 +1,65 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package mocks
+
+import (
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/statediff"
+
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
+)
+
+// PayloadConverter is the underlying struct for the Converter interface
+type PayloadConverter struct {
+	PassedStatediffPayload statediff.Payload
+	ReturnIPLDPayload      *eth.IPLDPayload
+	ReturnErr              error
+}
+
+// Convert method is used to convert a geth statediff.Payload to a IPLDPayload
+func (pc *PayloadConverter) Convert(payload interface{}) (interface{}, error) {
+	stateDiffPayload, ok := payload.(statediff.Payload)
+	if !ok {
+		return nil, fmt.Errorf("convert expected payload type %T got %T", statediff.Payload{}, payload)
+	}
+	pc.PassedStatediffPayload = stateDiffPayload
+	return pc.ReturnIPLDPayload, pc.ReturnErr
+}
+
+// IterativePayloadConverter is the underlying struct for the Converter interface
+type IterativePayloadConverter struct {
+	PassedStatediffPayload []statediff.Payload
+	ReturnIPLDPayload      []*eth.IPLDPayload
+	ReturnErr              error
+	iteration              int
+}
+
+// Convert method is used to convert a geth statediff.Payload to a IPLDPayload
+func (pc *IterativePayloadConverter) Convert(payload interface{}) (interface{}, error) {
+	stateDiffPayload, ok := payload.(statediff.Payload)
+	if !ok {
+		return nil, fmt.Errorf("convert expected payload type %T got %T", statediff.Payload{}, payload)
+	}
+	pc.PassedStatediffPayload = append(pc.PassedStatediffPayload, stateDiffPayload)
+	if len(pc.PassedStatediffPayload) < pc.iteration+1 {
+		return nil, fmt.Errorf("IterativePayloadConverter does not have a payload to return at iteration %d", pc.iteration)
+	}
+	returnPayload := pc.ReturnIPLDPayload[pc.iteration]
+	pc.iteration++
+	return returnPayload, pc.ReturnErr
+}

pkg/super_node/eth/mocks/dag_putters.go

@@ -0,0 +1,53 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package mocks
+
+import (
+	"errors"
+
+	"github.com/ethereum/go-ethereum/common"
+)
+
+// DagPutter is a mock for testing the ipfs publisher
+type DagPutter struct {
+	CIDsToReturn []string
+	ErrToReturn  error
+}
+
+// DagPut returns the pre-loaded CIDs or error
+func (dp *DagPutter) DagPut(raw interface{}) ([]string, error) {
+	return dp.CIDsToReturn, dp.ErrToReturn
+}
+
+// MappedDagPutter is a mock for testing the ipfs publisher
+type MappedDagPutter struct {
+	CIDsToReturn map[common.Hash][]string
+	ErrToReturn  error
+}
+
+// DagPut returns the pre-loaded CIDs or error
+func (mdp *MappedDagPutter) DagPut(raw interface{}) ([]string, error) {
+	if mdp.CIDsToReturn == nil {
+		return nil, errors.New("mapped dag putter needs to be initialized with a map of cids to return")
+	}
+	by, ok := raw.([]byte)
+	if !ok {
+		return nil, errors.New("mapped dag putters can only dag put []byte values")
+	}
+	hash := common.BytesToHash(by)
+	return mdp.CIDsToReturn[hash], nil
+}

pkg/super_node/eth/mocks/fetcher.go

@@ -0,0 +1,50 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package mocks
+
+import (
+	"errors"
+	"sync/atomic"
+
+	"github.com/ethereum/go-ethereum/statediff"
+)
+
+// StateDiffFetcher mock for tests
+type StateDiffFetcher struct {
+	PayloadsToReturn     map[uint64]statediff.Payload
+	FetchErrs            map[uint64]error
+	CalledAtBlockHeights [][]uint64
+	CalledTimes          int64
+}
+
+// FetchStateDiffsAt mock method
+func (fetcher *StateDiffFetcher) FetchAt(blockHeights []uint64) ([]interface{}, error) {
+	if fetcher.PayloadsToReturn == nil {
+		return nil, errors.New("mock StateDiffFetcher needs to be initialized with payloads to return")
+	}
+	atomic.AddInt64(&fetcher.CalledTimes, 1) // thread-safe increment
+	fetcher.CalledAtBlockHeights = append(fetcher.CalledAtBlockHeights, blockHeights)
+	results := make([]interface{}, 0, len(blockHeights))
+	for _, height := range blockHeights {
+		results = append(results, fetcher.PayloadsToReturn[height])
+		err, ok := fetcher.FetchErrs[height]
+		if ok && err != nil {
+			return nil, err
+		}
+	}
+	return results, nil
+}

pkg/super_node/eth/mocks/indexer.go

@@ -0,0 +1,39 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package mocks
+
+import (
+	"fmt"
+
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
+)
+
+// CIDIndexer is the underlying struct for the Indexer interface
+type CIDIndexer struct {
+	PassedCIDPayload []*eth.CIDPayload
+	ReturnErr        error
+}
+
+// Index indexes a cidPayload in Postgres
+func (repo *CIDIndexer) Index(cids interface{}) error {
+	cidPayload, ok := cids.(*eth.CIDPayload)
+	if !ok {
+		return fmt.Errorf("index expected cids type %T got %T", &eth.CIDPayload{}, cids)
+	}
+	repo.PassedCIDPayload = append(repo.PassedCIDPayload, cidPayload)
+	return repo.ReturnErr
+}

pkg/super_node/eth/mocks/publisher.go

@@ -0,0 +1,63 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package mocks
+
+import (
+	"fmt"
+
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
+)
+
+// IPLDPublisher is the underlying struct for the Publisher interface
+type IPLDPublisher struct {
+	PassedIPLDPayload *eth.IPLDPayload
+	ReturnCIDPayload  *eth.CIDPayload
+	ReturnErr         error
+}
+
+// Publish publishes an IPLDPayload to IPFS and returns the corresponding CIDPayload
+func (pub *IPLDPublisher) Publish(payload interface{}) (interface{}, error) {
+	ipldPayload, ok := payload.(*eth.IPLDPayload)
+	if !ok {
+		return nil, fmt.Errorf("publish expected payload type %T got %T", &eth.IPLDPayload{}, payload)
+	}
+	pub.PassedIPLDPayload = ipldPayload
+	return pub.ReturnCIDPayload, pub.ReturnErr
+}
+
+// IterativeIPLDPublisher is the underlying struct for the Publisher interface; used in testing
+type IterativeIPLDPublisher struct {
+	PassedIPLDPayload []*eth.IPLDPayload
+	ReturnCIDPayload  []*eth.CIDPayload
+	ReturnErr         error
+	iteration         int
+}
+
+// Publish publishes an IPLDPayload to IPFS and returns the corresponding CIDPayload
+func (pub *IterativeIPLDPublisher) Publish(payload interface{}) (interface{}, error) {
+	ipldPayload, ok := payload.(*eth.IPLDPayload)
+	if !ok {
+		return nil, fmt.Errorf("publish expected payload type %T got %T", &eth.IPLDPayload{}, payload)
+	}
+	pub.PassedIPLDPayload = append(pub.PassedIPLDPayload, ipldPayload)
+	if len(pub.ReturnCIDPayload) < pub.iteration+1 {
+		return nil, fmt.Errorf("IterativeIPLDPublisher does not have a payload to return at iteration %d", pub.iteration)
+	}
+	returnPayload := pub.ReturnCIDPayload[pub.iteration]
+	pub.iteration++
+	return returnPayload, pub.ReturnErr
+}

pkg/super_node/eth/mocks/retriever.go

@@ -0,0 +1,64 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package mocks
+
+import (
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
+)
+
+// MockCIDRetriever is a mock CID retriever for use in tests
+type MockCIDRetriever struct {
+	GapsToRetrieve              []shared.Gap
+	GapsToRetrieveErr           error
+	CalledTimes                 int
+	FirstBlockNumberToReturn    int64
+	RetrieveFirstBlockNumberErr error
+}
+
+// RetrieveCIDs mock method
+func (*MockCIDRetriever) Retrieve(filter interface{}, blockNumber int64) (interface{}, bool, error) {
+	panic("implement me")
+}
+
+// RetrieveLastBlockNumber mock method
+func (*MockCIDRetriever) RetrieveLastBlockNumber() (int64, error) {
+	panic("implement me")
+}
+
+// RetrieveFirstBlockNumber mock method
+func (mcr *MockCIDRetriever) RetrieveFirstBlockNumber() (int64, error) {
+	return mcr.FirstBlockNumberToReturn, mcr.RetrieveFirstBlockNumberErr
+}
+
+// RetrieveGapsInData mock method
+func (mcr *MockCIDRetriever) RetrieveGapsInData() ([]shared.Gap, error) {
+	mcr.CalledTimes++
+	return mcr.GapsToRetrieve, mcr.GapsToRetrieveErr
+}
+
+// SetGapsToRetrieve mock method
+func (mcr *MockCIDRetriever) SetGapsToRetrieve(gaps []shared.Gap) {
+	if mcr.GapsToRetrieve == nil {
+		mcr.GapsToRetrieve = make([]shared.Gap, 0)
+	}
+	mcr.GapsToRetrieve = append(mcr.GapsToRetrieve, gaps...)
+}
+
+func (mcr *MockCIDRetriever) Database() *postgres.DB {
+	panic("implement me")
+}

pkg/super_node/eth/mocks/streamer.go

@@ -0,0 +1,43 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package mocks
+
+import (
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/statediff"
+)
+
+// StateDiffStreamer is the underlying struct for the Streamer interface
+type StateDiffStreamer struct {
+	PassedPayloadChan chan interface{}
+	ReturnSub         *rpc.ClientSubscription
+	ReturnErr         error
+	StreamPayloads    []statediff.Payload
+}
+
+// Stream is the main loop for subscribing to data from the Geth state diff process
+func (sds *StateDiffStreamer) Stream(payloadChan chan interface{}) (*rpc.ClientSubscription, error) {
+	sds.PassedPayloadChan = payloadChan
+
+	go func() {
+		for _, payload := range sds.StreamPayloads {
+			sds.PassedPayloadChan <- payload
+		}
+	}()
+
+	return sds.ReturnSub, sds.ReturnErr
+}

pkg/super_node/eth/mocks/test_data.go

@@ -0,0 +1,412 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package mocks
+
+import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"math/big"
+	rand2 "math/rand"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/state"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/statediff"
+	"github.com/ipfs/go-block-format"
+	log "github.com/sirupsen/logrus"
+
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
+	eth2 "github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
+)
+
+// Test variables
+var (
+	// block data
+	BlockNumber = big.NewInt(1)
+	MockHeader  = types.Header{
+		Time:        0,
+		Number:      BlockNumber,
+		Root:        common.HexToHash("0x0"),
+		TxHash:      common.HexToHash("0x0"),
+		ReceiptHash: common.HexToHash("0x0"),
+	}
+	MockTransactions, MockReceipts, senderAddr = createTransactionsAndReceipts()
+	ReceiptsRlp, _                             = rlp.EncodeToBytes(MockReceipts)
+	MockBlock                                  = types.NewBlock(&MockHeader, MockTransactions, nil, MockReceipts)
+	MockBlockRlp, _                            = rlp.EncodeToBytes(MockBlock)
+	MockHeaderRlp, _                           = rlp.EncodeToBytes(MockBlock.Header())
+	MockTrxMeta                                = []eth.TxModel{
+		{
+			CID: "", // This is empty until we go to publish to ipfs
+			Src: senderAddr.Hex(),
+			Dst: "0x0000000000000000000000000000000000000000",
+		},
+		{
+			CID: "",
+			Src: senderAddr.Hex(),
+			Dst: "0x0000000000000000000000000000000000000001",
+		},
+	}
+	MockRctMeta = []eth.ReceiptModel{
+		{
+			CID: "",
+			Topic0s: []string{
+				"0x0000000000000000000000000000000000000000000000000000000000000004",
+			},
+			Contract: "0x0000000000000000000000000000000000000000",
+		},
+		{
+			CID: "",
+			Topic0s: []string{
+				"0x0000000000000000000000000000000000000000000000000000000000000005",
+			},
+			Contract: "0x0000000000000000000000000000000000000001",
+		},
+	}
+
+	// statediff data
+	CodeHash            = common.Hex2Bytes("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")
+	NonceValue          = rand2.Uint64()
+	anotherNonceValue   = rand2.Uint64()
+	BalanceValue        = rand2.Int63()
+	anotherBalanceValue = rand2.Int63()
+	ContractRoot        = common.HexToHash("0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421")
+	StoragePath         = common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes()
+	StorageKey          = common.HexToHash("0000000000000000000000000000000000000000000000000000000000000001").Bytes()
+	StorageValue        = common.Hex2Bytes("0x03")
+	storage             = []statediff.StorageDiff{{
+		Key:   StorageKey,
+		Value: StorageValue,
+		Path:  StoragePath,
+		Proof: [][]byte{},
+		Leaf:  true,
+	}}
+	emptyStorage           = make([]statediff.StorageDiff, 0)
+	Address                = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592")
+	ContractLeafKey        = eth.AddressToKey(Address)
+	AnotherAddress         = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476593")
+	AnotherContractLeafKey = eth.AddressToKey(AnotherAddress)
+	testAccount            = state.Account{
+		Nonce:    NonceValue,
+		Balance:  big.NewInt(BalanceValue),
+		Root:     ContractRoot,
+		CodeHash: CodeHash,
+	}
+	anotherTestAccount = state.Account{
+		Nonce:    anotherNonceValue,
+		Balance:  big.NewInt(anotherBalanceValue),
+		Root:     common.HexToHash("0x"),
+		CodeHash: nil,
+	}
+	ValueBytes, _        = rlp.EncodeToBytes(testAccount)
+	AnotherValueBytes, _ = rlp.EncodeToBytes(anotherTestAccount)
+	CreatedAccountDiffs  = []statediff.AccountDiff{
+		{
+			Key:     ContractLeafKey.Bytes(),
+			Value:   ValueBytes,
+			Storage: storage,
+			Leaf:    true,
+		},
+		{
+			Key:     AnotherContractLeafKey.Bytes(),
+			Value:   AnotherValueBytes,
+			Storage: emptyStorage,
+			Leaf:    true,
+		},
+	}
+
+	MockStateDiff = statediff.StateDiff{
+		BlockNumber:     BlockNumber,
+		BlockHash:       MockBlock.Hash(),
+		CreatedAccounts: CreatedAccountDiffs,
+	}
+	MockStateDiffBytes, _ = rlp.EncodeToBytes(MockStateDiff)
+	MockStateNodes        = []eth.TrieNode{
+		{
+			Key:   ContractLeafKey,
+			Value: ValueBytes,
+			Leaf:  true,
+		},
+		{
+			Key:   AnotherContractLeafKey,
+			Value: AnotherValueBytes,
+			Leaf:  true,
+		},
+	}
+	MockStorageNodes = map[common.Hash][]eth.TrieNode{
+		ContractLeafKey: {
+			{
+				Key:   common.BytesToHash(StorageKey),
+				Value: StorageValue,
+				Leaf:  true,
+			},
+		},
+	}
+
+	// aggregate payloads
+	MockStateDiffPayload = statediff.Payload{
+		BlockRlp:        MockBlockRlp,
+		StateDiffRlp:    MockStateDiffBytes,
+		ReceiptsRlp:     ReceiptsRlp,
+		TotalDifficulty: big.NewInt(1337),
+	}
+
+	MockIPLDPayload = &eth.IPLDPayload{
+		TotalDifficulty: big.NewInt(1337),
+		Block:           MockBlock,
+		Receipts:        MockReceipts,
+		HeaderRLP:       MockHeaderRlp,
+		TrxMetaData: []eth.TxModel{
+			{
+				CID: "",
+				Src: senderAddr.Hex(),
+				Dst: "0x0000000000000000000000000000000000000000",
+			},
+			{
+				CID: "",
+				Src: senderAddr.Hex(),
+				Dst: "0x0000000000000000000000000000000000000001",
+			},
+		},
+		ReceiptMetaData: []eth.ReceiptModel{
+			{
+				CID: "",
+				Topic0s: []string{
+					"0x0000000000000000000000000000000000000000000000000000000000000004",
+				},
+				Contract: "0x0000000000000000000000000000000000000000",
+			},
+			{
+				CID: "",
+				Topic0s: []string{
+					"0x0000000000000000000000000000000000000000000000000000000000000005",
+				},
+				Contract: "0x0000000000000000000000000000000000000001",
+			},
+		},
+		StorageNodes: MockStorageNodes,
+		StateNodes:   MockStateNodes,
+	}
+
+	MockCIDPayload = &eth.CIDPayload{
+		HeaderCID: eth2.HeaderModel{
+			BlockHash:       MockBlock.Hash().String(),
+			BlockNumber:     MockBlock.Number().String(),
+			CID:             "mockHeaderCID",
+			Uncle:           false,
+			ParentHash:      MockBlock.ParentHash().String(),
+			TotalDifficulty: "1337",
+		},
+		UncleCIDs: []eth2.HeaderModel{},
+		TransactionCIDs: []eth.TxModel{
+			{
+				TxHash: MockTransactions[0].Hash().String(),
+				CID:    "mockTrxCID1",
+				Dst:    "0x0000000000000000000000000000000000000000",
+				Src:    senderAddr.Hex(),
+			},
+			{
+				TxHash: MockTransactions[1].Hash().String(),
+				CID:    "mockTrxCID2",
+				Dst:    "0x0000000000000000000000000000000000000001",
+				Src:    senderAddr.Hex(),
+			},
+		},
+		ReceiptCIDs: map[common.Hash]eth.ReceiptModel{
+			MockTransactions[0].Hash(): {
+				CID:      "mockRctCID1",
+				Topic0s:  []string{"0x0000000000000000000000000000000000000000000000000000000000000004"},
+				Contract: "0x0000000000000000000000000000000000000000",
+			},
+			MockTransactions[1].Hash(): {
+				CID:      "mockRctCID2",
+				Topic0s:  []string{"0x0000000000000000000000000000000000000000000000000000000000000005"},
+				Contract: "0x0000000000000000000000000000000000000001",
+			},
+		},
+		StateNodeCIDs: []eth.StateNodeModel{
+			{
+				CID:      "mockStateCID1",
+				Leaf:     true,
+				StateKey: ContractLeafKey.String(),
+			},
+			{
+				CID:      "mockStateCID2",
+				Leaf:     true,
+				StateKey: AnotherContractLeafKey.String(),
+			},
+		},
+		StorageNodeCIDs: map[common.Hash][]eth.StorageNodeModel{
+			ContractLeafKey: {
+				{
+					CID:        "mockStorageCID",
+					StorageKey: "0x0000000000000000000000000000000000000000000000000000000000000001",
+					Leaf:       true,
+				},
+			},
+		},
+	}
+
+	MockCIDWrapper = &eth.CIDWrapper{
+		BlockNumber: big.NewInt(1),
+		Headers: []eth2.HeaderModel{
+			{
+				BlockNumber:     "1",
+				BlockHash:       MockBlock.Hash().String(),
+				ParentHash:      "0x0000000000000000000000000000000000000000000000000000000000000000",
+				CID:             "mockHeaderCID",
+				Uncle:           false,
+				TotalDifficulty: "1337",
+			},
+		},
+		Transactions: []eth2.TxModel{
+			{
+				CID: "mockTrxCID1",
+			},
+			{
+				TxHash: MockTransactions[1].Hash().String(),
+				CID:    "mockTrxCID2",
+				Dst:    "0x0000000000000000000000000000000000000001",
+				Src:    senderAddr.String(),
+			},
+		},
+		Receipts: []eth2.ReceiptModel{
+			{
+				CID:      "mockRctCID1",
+				Contract: "0x0000000000000000000000000000000000000000",
+				Topic0s: []string{
+					"0x0000000000000000000000000000000000000000000000000000000000000004",
+				},
+			},
+			{
+				CID:      "mockRctCID2",
+				Contract: "0x0000000000000000000000000000000000000001",
+				Topic0s: []string{
+					"0x0000000000000000000000000000000000000000000000000000000000000005",
+				},
+			},
+		},
+		Uncles: []eth2.HeaderModel{},
+		StateNodes: []eth.StateNodeModel{
+			{
+				CID:      "mockStateCID1",
+				Leaf:     true,
+				StateKey: ContractLeafKey.Hex(),
+			},
+			{
+				CID:      "mockStateCID2",
+				Leaf:     true,
+				StateKey: AnotherContractLeafKey.Hex(),
+			},
+		},
+		StorageNodes: []eth.StorageNodeWithStateKeyModel{
+			{
+				CID:        "mockStorageCID",
+				Leaf:       true,
+				StateKey:   ContractLeafKey.Hex(),
+				StorageKey: "0x0000000000000000000000000000000000000000000000000000000000000001",
+			},
+		},
+	}
+
+	MockIPLDWrapper = &eth.IPLDWrapper{
+		BlockNumber: big.NewInt(1),
+		Headers: []blocks.Block{
+			blocks.NewBlock(MockHeaderRlp),
+		},
+		Transactions: []blocks.Block{
+			blocks.NewBlock(MockTransactions.GetRlp(0)),
+			blocks.NewBlock(MockTransactions.GetRlp(1)),
+		},
+		Receipts: []blocks.Block{
+			blocks.NewBlock(MockReceipts.GetRlp(0)),
+			blocks.NewBlock(MockReceipts.GetRlp(1)),
+		},
+		StateNodes: map[common.Hash]blocks.Block{
+			ContractLeafKey:        blocks.NewBlock(ValueBytes),
+			AnotherContractLeafKey: blocks.NewBlock(AnotherValueBytes),
+		},
+		StorageNodes: map[common.Hash]map[common.Hash]blocks.Block{
+			ContractLeafKey: {
+				common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000001"): blocks.NewBlock(StorageValue),
+			},
+		},
+	}
+
+	MockSeedNodePayload = eth2.StreamPayload{
+		BlockNumber:     big.NewInt(1),
+		HeadersRlp:      [][]byte{MockHeaderRlp},
+		UnclesRlp:       [][]byte{},
+		TransactionsRlp: [][]byte{MockTransactions.GetRlp(0), MockTransactions.GetRlp(1)},
+		ReceiptsRlp:     [][]byte{MockTransactions.GetRlp(0), MockTransactions.GetRlp(1)},
+		StateNodesRlp: map[common.Hash][]byte{
+			ContractLeafKey:        ValueBytes,
+			AnotherContractLeafKey: AnotherValueBytes,
+		},
+		StorageNodesRlp: map[common.Hash]map[common.Hash][]byte{
+			ContractLeafKey: {
+				common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000001"): StorageValue,
+			},
+		},
+	}
+)
+
+// createTransactionsAndReceipts is a helper function to generate signed mock transactions and mock receipts with mock logs
+func createTransactionsAndReceipts() (types.Transactions, types.Receipts, common.Address) {
+	// make transactions
+	trx1 := types.NewTransaction(0, common.HexToAddress("0x0"), big.NewInt(1000), 50, big.NewInt(100), nil)
+	trx2 := types.NewTransaction(1, common.HexToAddress("0x1"), big.NewInt(2000), 100, big.NewInt(200), nil)
+	transactionSigner := types.MakeSigner(params.MainnetChainConfig, BlockNumber)
+	mockCurve := elliptic.P256()
+	mockPrvKey, err := ecdsa.GenerateKey(mockCurve, rand.Reader)
+	if err != nil {
+		log.Fatal(err)
+	}
+	signedTrx1, err := types.SignTx(trx1, transactionSigner, mockPrvKey)
+	if err != nil {
+		log.Fatal(err)
+	}
+	signedTrx2, err := types.SignTx(trx2, transactionSigner, mockPrvKey)
+	if err != nil {
+		log.Fatal(err)
+	}
+	senderAddr, err := types.Sender(transactionSigner, signedTrx1) // same for both trx
+	if err != nil {
+		log.Fatal(err)
+	}
+	// make receipts
+	mockTopic1 := common.HexToHash("0x04")
+	mockReceipt1 := types.NewReceipt(common.HexToHash("0x0").Bytes(), false, 50)
+	mockReceipt1.ContractAddress = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592")
+	mockLog1 := &types.Log{
+		Topics: []common.Hash{mockTopic1},
+	}
+	mockReceipt1.Logs = []*types.Log{mockLog1}
+	mockReceipt1.TxHash = signedTrx1.Hash()
+	mockTopic2 := common.HexToHash("0x05")
+	mockReceipt2 := types.NewReceipt(common.HexToHash("0x1").Bytes(), false, 100)
+	mockReceipt2.ContractAddress = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476593")
+	mockLog2 := &types.Log{
+		Topics: []common.Hash{mockTopic2},
+	}
+	mockReceipt2.Logs = []*types.Log{mockLog2}
+	mockReceipt2.TxHash = signedTrx2.Hash()
+	return types.Transactions{signedTrx1, signedTrx2}, types.Receipts{mockReceipt1, mockReceipt2}, senderAddr
+}