major refactoring of super_node to make it easier to support other chains

pkg/super_node/eth/api.go

@@ -1,3 +1,19 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
 package eth
 
 import (
@@ -9,18 +25,18 @@ import (
 	"github.com/ethereum/go-ethereum/rpc"
 )
 
-// APIName is the namespace used for the state diffing service API
+// APIName is the namespace for the super node's eth api
 const APIName = "eth"
 
-// APIVersion is the version of the state diffing service API
+// APIVersion is the version of the super node's eth api
 const APIVersion = "0.0.1"
 
 type PublicEthAPI struct {
-	b Backend
+	b *Backend
 }
 
 // NewPublicEthAPI creates a new PublicEthAPI with the provided underlying Backend
-func NewPublicEthAPI(b Backend) *PublicEthAPI {
+func NewPublicEthAPI(b *Backend) *PublicEthAPI {
 	return &PublicEthAPI{
 		b: b,
 	}

pkg/super_node/eth/backend.go

@@ -1,3 +1,19 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
 package eth
 
 import (
@@ -6,16 +22,13 @@ import (
 	"fmt"
 	"math/big"
 
-	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
-
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/hashicorp/golang-lru"
 
-	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
-	"github.com/vulcanize/vulcanizedb/pkg/super_node"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 )
 
 var (
@@ -23,8 +36,8 @@ var (
 )
 
 type Backend struct {
-	retriever super_node.CIDRetriever
-	fetcher   ipfs.IPLDFetcher
+	retriever *CIDRetriever
+	fetcher   *IPLDFetcher
 	db        *postgres.DB
 
 	headerCache *lru.Cache // Cache for the most recent block headers
@@ -32,12 +45,17 @@ type Backend struct {
 	numberCache *lru.Cache // Cache for the most recent block numbers
 }
 
-func NewEthBackend(r super_node.CIDRetriever, f ipfs.IPLDFetcher) *Backend {
+func NewEthBackend(db *postgres.DB, ipfsPath string) (*Backend, error) {
+	r := NewCIDRetriever(db)
+	f, err := NewIPLDFetcher(ipfsPath)
+	if err != nil {
+		return nil, err
+	}
 	return &Backend{
 		retriever: r,
 		fetcher:   f,
 		db:        r.Database(),
-	}
+	}, nil
 }
 
 func (b *Backend) HeaderByNumber(ctx context.Context, blockNumber rpc.BlockNumber) (*types.Header, error) {

pkg/super_node/eth/converter.go

@@ -0,0 +1,175 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package eth
+
+import (
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/statediff"
+)
+
+// PayloadConverter satisfies the PayloadConverter interface for ethereum
+type PayloadConverter struct {
+	chainConfig *params.ChainConfig
+}
+
+// NewPayloadConverter creates a pointer to a new Converter which satisfies the PayloadConverter interface
+func NewPayloadConverter(chainConfig *params.ChainConfig) *PayloadConverter {
+	return &PayloadConverter{
+		chainConfig: chainConfig,
+	}
+}
+
+// Convert method is used to convert a geth statediff.Payload to a IPLDPayload
+func (pc *PayloadConverter) Convert(payload interface{}) (interface{}, error) {
+	stateDiffPayload, ok := payload.(statediff.Payload)
+	if !ok {
+		return nil, fmt.Errorf("eth converter: expected payload type %T got %T", statediff.Payload{}, payload)
+	}
+	// Unpack block rlp to access fields
+	block := new(types.Block)
+	if err := rlp.DecodeBytes(stateDiffPayload.BlockRlp, block); err != nil {
+		return nil, err
+	}
+	// Process and publish headers
+	header := block.Header()
+	headerRlp, err := rlp.EncodeToBytes(header)
+	if err != nil {
+		return nil, err
+	}
+	trxLen := len(block.Transactions())
+	convertedPayload := &IPLDPayload{
+		TotalDifficulty: stateDiffPayload.TotalDifficulty,
+		Block:           block,
+		HeaderRLP:       headerRlp,
+		TrxMetaData:     make([]TxModel, 0, trxLen),
+		Receipts:        make(types.Receipts, 0, trxLen),
+		ReceiptMetaData: make([]ReceiptModel, 0, trxLen),
+		StateNodes:      make([]TrieNode, 0),
+		StorageNodes:    make(map[common.Hash][]TrieNode),
+	}
+	signer := types.MakeSigner(pc.chainConfig, block.Number())
+	transactions := block.Transactions()
+	for _, trx := range transactions {
+		// Extract to and from data from the the transactions for indexing
+		from, err := types.Sender(signer, trx)
+		if err != nil {
+			return nil, err
+		}
+		txMeta := TxModel{
+			Dst: handleNullAddr(trx.To()),
+			Src: handleNullAddr(&from),
+		}
+		// txMeta will have same index as its corresponding trx in the convertedPayload.BlockBody
+		convertedPayload.TrxMetaData = append(convertedPayload.TrxMetaData, txMeta)
+	}
+
+	// Decode receipts for this block
+	receipts := make(types.Receipts, 0)
+	if err := rlp.DecodeBytes(stateDiffPayload.ReceiptsRlp, &receipts); err != nil {
+		return nil, err
+	}
+	// Derive any missing fields
+	if err := receipts.DeriveFields(pc.chainConfig, block.Hash(), block.NumberU64(), block.Transactions()); err != nil {
+		return nil, err
+	}
+	for i, receipt := range receipts {
+		// If the transaction for this receipt has a "to" address, the above DeriveFields() fails to assign it to the receipt's ContractAddress
+		// If it doesn't have a "to" address, it correctly derives it and assigns it to to the receipt's ContractAddress
+		// Weird, right?
+		if transactions[i].To() != nil {
+			receipt.ContractAddress = *transactions[i].To()
+		}
+		// Extract topic0 data from the receipt's logs for indexing
+		rctMeta := ReceiptModel{
+			Topic0s:  make([]string, 0, len(receipt.Logs)),
+			Contract: receipt.ContractAddress.Hex(),
+		}
+		for _, log := range receipt.Logs {
+			if len(log.Topics) < 1 {
+				continue
+			}
+			rctMeta.Topic0s = append(rctMeta.Topic0s, log.Topics[0].Hex())
+		}
+		// receipt and rctMeta will have same indexes
+		convertedPayload.Receipts = append(convertedPayload.Receipts, receipt)
+		convertedPayload.ReceiptMetaData = append(convertedPayload.ReceiptMetaData, rctMeta)
+	}
+
+	// Unpack state diff rlp to access fields
+	stateDiff := new(statediff.StateDiff)
+	if err := rlp.DecodeBytes(stateDiffPayload.StateDiffRlp, stateDiff); err != nil {
+		return nil, err
+	}
+	for _, createdAccount := range stateDiff.CreatedAccounts {
+		hashKey := common.BytesToHash(createdAccount.Key)
+		convertedPayload.StateNodes = append(convertedPayload.StateNodes, TrieNode{
+			Key:   hashKey,
+			Value: createdAccount.Value,
+			Leaf:  createdAccount.Leaf,
+		})
+		for _, storageDiff := range createdAccount.Storage {
+			convertedPayload.StorageNodes[hashKey] = append(convertedPayload.StorageNodes[hashKey], TrieNode{
+				Key:   common.BytesToHash(storageDiff.Key),
+				Value: storageDiff.Value,
+				Leaf:  storageDiff.Leaf,
+			})
+		}
+	}
+	for _, deletedAccount := range stateDiff.DeletedAccounts {
+		hashKey := common.BytesToHash(deletedAccount.Key)
+		convertedPayload.StateNodes = append(convertedPayload.StateNodes, TrieNode{
+			Key:   hashKey,
+			Value: deletedAccount.Value,
+			Leaf:  deletedAccount.Leaf,
+		})
+		for _, storageDiff := range deletedAccount.Storage {
+			convertedPayload.StorageNodes[hashKey] = append(convertedPayload.StorageNodes[hashKey], TrieNode{
+				Key:   common.BytesToHash(storageDiff.Key),
+				Value: storageDiff.Value,
+				Leaf:  storageDiff.Leaf,
+			})
+		}
+	}
+	for _, updatedAccount := range stateDiff.UpdatedAccounts {
+		hashKey := common.BytesToHash(updatedAccount.Key)
+		convertedPayload.StateNodes = append(convertedPayload.StateNodes, TrieNode{
+			Key:   hashKey,
+			Value: updatedAccount.Value,
+			Leaf:  updatedAccount.Leaf,
+		})
+		for _, storageDiff := range updatedAccount.Storage {
+			convertedPayload.StorageNodes[hashKey] = append(convertedPayload.StorageNodes[hashKey], TrieNode{
+				Key:   common.BytesToHash(storageDiff.Key),
+				Value: storageDiff.Value,
+				Leaf:  storageDiff.Leaf,
+			})
+		}
+	}
+	return convertedPayload, nil
+}
+
+func handleNullAddr(to *common.Address) string {
+	if to == nil {
+		return "0x0000000000000000000000000000000000000000000000000000000000000000"
+	}
+	return to.Hex()
+}

pkg/super_node/eth/converter_test.go

@@ -0,0 +1,52 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package eth_test
+
+import (
+	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/rlp"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth/mocks"
+)
+
+var _ = Describe("Converter", func() {
+	Describe("Convert", func() {
+		It("Converts mock statediff.Payloads into the expected IPLDPayloads", func() {
+			converter := eth.NewPayloadConverter(params.MainnetChainConfig)
+			payload, err := converter.Convert(mocks.MockStateDiffPayload)
+			Expect(err).ToNot(HaveOccurred())
+			convertedPayload, ok := payload.(*eth.IPLDPayload)
+			Expect(ok).To(BeTrue())
+			Expect(convertedPayload.Block.Number().String()).To(Equal(mocks.BlockNumber.String()))
+			Expect(convertedPayload.Block.Hash().String()).To(Equal(mocks.MockBlock.Hash().String()))
+			Expect(convertedPayload.StateNodes).To(Equal(mocks.MockStateNodes))
+			Expect(convertedPayload.StorageNodes).To(Equal(mocks.MockStorageNodes))
+			Expect(convertedPayload.TotalDifficulty.Int64()).To(Equal(mocks.MockStateDiffPayload.TotalDifficulty.Int64()))
+			gotBody, err := rlp.EncodeToBytes(convertedPayload.Block.Body())
+			Expect(err).ToNot(HaveOccurred())
+			expectedBody, err := rlp.EncodeToBytes(mocks.MockBlock.Body())
+			Expect(err).ToNot(HaveOccurred())
+			Expect(gotBody).To(Equal(expectedBody))
+			Expect(convertedPayload.HeaderRLP).To(Equal(mocks.MockHeaderRlp))
+			Expect(convertedPayload.TrxMetaData).To(Equal(mocks.MockTrxMeta))
+			Expect(convertedPayload.ReceiptMetaData).To(Equal(mocks.MockRctMeta))
+		})
+	})
+})

pkg/super_node/eth/eth_suite_test.go

@@ -0,0 +1,35 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package eth_test
+
+import (
+	"io/ioutil"
+	"testing"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/sirupsen/logrus"
+)
+
+func TestETHSuperNode(t *testing.T) {
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Super Node ETH Suite Test")
+}
+
+var _ = BeforeSuite(func() {
+	logrus.SetOutput(ioutil.Discard)
+})

pkg/super_node/eth/filterer.go

@@ -0,0 +1,250 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package eth
+
+import (
+	"bytes"
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/rlp"
+
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/config"
+)
+
+// ResponseFilterer satisfies the ResponseFilterer interface for ethereum
+type ResponseFilterer struct{}
+
+// NewResponseFilterer creates a new Filterer satisfying the ResponseFilterer interface
+func NewResponseFilterer() *ResponseFilterer {
+	return &ResponseFilterer{}
+}
+
+// Filter is used to filter through eth data to extract and package requested data into a Payload
+func (s *ResponseFilterer) Filter(filter, payload interface{}) (interface{}, error) {
+	ethFilters, ok := filter.(*config.EthSubscription)
+	if !ok {
+		return StreamPayload{}, fmt.Errorf("eth filterer expected filter type %T got %T", &config.EthSubscription{}, filter)
+	}
+	ethPayload, ok := payload.(*IPLDPayload)
+	if !ok {
+		return StreamPayload{}, fmt.Errorf("eth filterer expected payload type %T got %T", &IPLDPayload{}, payload)
+	}
+	if checkRange(ethFilters.Start.Int64(), ethFilters.End.Int64(), ethPayload.Block.Number().Int64()) {
+		response := new(StreamPayload)
+		if err := s.filterHeaders(ethFilters.HeaderFilter, response, ethPayload); err != nil {
+			return StreamPayload{}, err
+		}
+		txHashes, err := s.filterTransactions(ethFilters.TxFilter, response, ethPayload)
+		if err != nil {
+			return StreamPayload{}, err
+		}
+		if err := s.filerReceipts(ethFilters.ReceiptFilter, response, ethPayload, txHashes); err != nil {
+			return StreamPayload{}, err
+		}
+		if err := s.filterState(ethFilters.StateFilter, response, ethPayload); err != nil {
+			return StreamPayload{}, err
+		}
+		if err := s.filterStorage(ethFilters.StorageFilter, response, ethPayload); err != nil {
+			return StreamPayload{}, err
+		}
+		response.BlockNumber = ethPayload.Block.Number()
+		return *response, nil
+	}
+	return StreamPayload{}, nil
+}
+
+func (s *ResponseFilterer) filterHeaders(headerFilter config.HeaderFilter, response *StreamPayload, payload *IPLDPayload) error {
+	if !headerFilter.Off {
+		response.HeadersRlp = append(response.HeadersRlp, payload.HeaderRLP)
+		if headerFilter.Uncles {
+			response.UnclesRlp = make([][]byte, 0, len(payload.Block.Body().Uncles))
+			for _, uncle := range payload.Block.Body().Uncles {
+				uncleRlp, err := rlp.EncodeToBytes(uncle)
+				if err != nil {
+					return err
+				}
+				response.UnclesRlp = append(response.UnclesRlp, uncleRlp)
+			}
+		}
+	}
+	return nil
+}
+
+func checkRange(start, end, actual int64) bool {
+	if (end <= 0 || end >= actual) && start <= actual {
+		return true
+	}
+	return false
+}
+
+func (s *ResponseFilterer) filterTransactions(trxFilter config.TxFilter, response *StreamPayload, payload *IPLDPayload) ([]common.Hash, error) {
+	trxHashes := make([]common.Hash, 0, len(payload.Block.Body().Transactions))
+	if !trxFilter.Off {
+		for i, trx := range payload.Block.Body().Transactions {
+			if checkTransactions(trxFilter.Src, trxFilter.Dst, payload.TrxMetaData[i].Src, payload.TrxMetaData[i].Dst) {
+				trxBuffer := new(bytes.Buffer)
+				err := trx.EncodeRLP(trxBuffer)
+				if err != nil {
+					return nil, err
+				}
+				trxHashes = append(trxHashes, trx.Hash())
+				response.TransactionsRlp = append(response.TransactionsRlp, trxBuffer.Bytes())
+			}
+		}
+	}
+	return trxHashes, nil
+}
+
+func checkTransactions(wantedSrc, wantedDst []string, actualSrc, actualDst string) bool {
+	// If we aren't filtering for any addresses, every transaction is a go
+	if len(wantedDst) == 0 && len(wantedSrc) == 0 {
+		return true
+	}
+	for _, src := range wantedSrc {
+		if src == actualSrc {
+			return true
+		}
+	}
+	for _, dst := range wantedDst {
+		if dst == actualDst {
+			return true
+		}
+	}
+	return false
+}
+
+func (s *ResponseFilterer) filerReceipts(receiptFilter config.ReceiptFilter, response *StreamPayload, payload *IPLDPayload, trxHashes []common.Hash) error {
+	if !receiptFilter.Off {
+		for i, receipt := range payload.Receipts {
+			if checkReceipts(receipt, receiptFilter.Topic0s, payload.ReceiptMetaData[i].Topic0s, receiptFilter.Contracts, payload.ReceiptMetaData[i].Contract, trxHashes, receiptFilter.MatchTxs) {
+				receiptForStorage := (*types.ReceiptForStorage)(receipt)
+				receiptBuffer := new(bytes.Buffer)
+				err := receiptForStorage.EncodeRLP(receiptBuffer)
+				if err != nil {
+					return err
+				}
+				response.ReceiptsRlp = append(response.ReceiptsRlp, receiptBuffer.Bytes())
+			}
+		}
+	}
+	return nil
+}
+
+func checkReceipts(rct *types.Receipt, wantedTopics, actualTopics, wantedContracts []string, actualContract string, wantedTrxHashes []common.Hash, matchTxs bool) bool {
+	// If we aren't filtering for any topics, contracts, or corresponding trxs then all receipts are a go
+	if len(wantedTopics) == 0 && len(wantedContracts) == 0 && (len(wantedTrxHashes) == 0 || !matchTxs) {
+		return true
+	}
+	// No matter what filters we have, we keep receipts for specific trxs we are interested in
+	if matchTxs {
+		for _, wantedTrxHash := range wantedTrxHashes {
+			if bytes.Equal(wantedTrxHash.Bytes(), rct.TxHash.Bytes()) {
+				return true
+			}
+		}
+	}
+
+	if len(wantedContracts) == 0 {
+		// We keep all receipts that have logs we are interested in
+		for _, wantedTopic := range wantedTopics {
+			for _, actualTopic := range actualTopics {
+				if wantedTopic == actualTopic {
+					return true
+				}
+			}
+		}
+	} else { // We keep all receipts that belong to one of the specified contracts if we aren't filtering on topics
+		for _, wantedContract := range wantedContracts {
+			if wantedContract == actualContract {
+				if len(wantedTopics) == 0 {
+					return true
+				}
+				// Or if we have contracts and topics to filter on we only keep receipts that satisfy both conditions
+				for _, wantedTopic := range wantedTopics {
+					for _, actualTopic := range actualTopics {
+						if wantedTopic == actualTopic {
+							return true
+						}
+					}
+				}
+			}
+		}
+	}
+
+	return false
+}
+
+func (s *ResponseFilterer) filterState(stateFilter config.StateFilter, response *StreamPayload, payload *IPLDPayload) error {
+	if !stateFilter.Off {
+		response.StateNodesRlp = make(map[common.Hash][]byte)
+		keyFilters := make([]common.Hash, 0, len(stateFilter.Addresses))
+		for _, addr := range stateFilter.Addresses {
+			keyFilter := AddressToKey(common.HexToAddress(addr))
+			keyFilters = append(keyFilters, keyFilter)
+		}
+		for _, stateNode := range payload.StateNodes {
+			if checkNodeKeys(keyFilters, stateNode.Key) {
+				if stateNode.Leaf || stateFilter.IntermediateNodes {
+					response.StateNodesRlp[stateNode.Key] = stateNode.Value
+				}
+			}
+		}
+	}
+	return nil
+}
+
+func checkNodeKeys(wantedKeys []common.Hash, actualKey common.Hash) bool {
+	// If we aren't filtering for any specific keys, all nodes are a go
+	if len(wantedKeys) == 0 {
+		return true
+	}
+	for _, key := range wantedKeys {
+		if bytes.Equal(key.Bytes(), actualKey.Bytes()) {
+			return true
+		}
+	}
+	return false
+}
+
+func (s *ResponseFilterer) filterStorage(storageFilter config.StorageFilter, response *StreamPayload, payload *IPLDPayload) error {
+	if !storageFilter.Off {
+		response.StorageNodesRlp = make(map[common.Hash]map[common.Hash][]byte)
+		stateKeyFilters := make([]common.Hash, 0, len(storageFilter.Addresses))
+		for _, addr := range storageFilter.Addresses {
+			keyFilter := AddressToKey(common.HexToAddress(addr))
+			stateKeyFilters = append(stateKeyFilters, keyFilter)
+		}
+		storageKeyFilters := make([]common.Hash, 0, len(storageFilter.StorageKeys))
+		for _, store := range storageFilter.StorageKeys {
+			keyFilter := HexToKey(store)
+			storageKeyFilters = append(storageKeyFilters, keyFilter)
+		}
+		for stateKey, storageNodes := range payload.StorageNodes {
+			if checkNodeKeys(stateKeyFilters, stateKey) {
+				response.StorageNodesRlp[stateKey] = make(map[common.Hash][]byte)
+				for _, storageNode := range storageNodes {
+					if checkNodeKeys(storageKeyFilters, storageNode.Key) {
+						response.StorageNodesRlp[stateKey][storageNode.Key] = storageNode.Value
+					}
+				}
+			}
+		}
+	}
+	return nil
+}

pkg/super_node/eth/filterer_test.go

@@ -0,0 +1,172 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package eth_test
+
+import (
+	"bytes"
+
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
+
+	"github.com/ethereum/go-ethereum/core/types"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth/mocks"
+)
+
+var (
+	filterer                  *eth.ResponseFilterer
+	expectedRctForStorageRLP1 []byte
+	expectedRctForStorageRLP2 []byte
+)
+
+var _ = Describe("Filterer", func() {
+	Describe("FilterResponse", func() {
+		BeforeEach(func() {
+			filterer = eth.NewResponseFilterer()
+			expectedRctForStorageRLP1 = getReceiptForStorageRLP(mocks.MockReceipts, 0)
+			expectedRctForStorageRLP2 = getReceiptForStorageRLP(mocks.MockReceipts, 1)
+		})
+
+		It("Transcribes all the data from the IPLDPayload into the StreamPayload if given an open filter", func() {
+			payload, err := filterer.Filter(openFilter, mocks.MockIPLDPayload)
+			Expect(err).ToNot(HaveOccurred())
+			superNodePayload, ok := payload.(eth.StreamPayload)
+			Expect(ok).To(BeTrue())
+			Expect(superNodePayload.BlockNumber.Int64()).To(Equal(mocks.MockSeedNodePayload.BlockNumber.Int64()))
+			Expect(superNodePayload.HeadersRlp).To(Equal(mocks.MockSeedNodePayload.HeadersRlp))
+			var unclesRlp [][]byte
+			Expect(superNodePayload.UnclesRlp).To(Equal(unclesRlp))
+			Expect(len(superNodePayload.TransactionsRlp)).To(Equal(2))
+			Expect(shared.ListContainsBytes(superNodePayload.TransactionsRlp, mocks.MockTransactions.GetRlp(0))).To(BeTrue())
+			Expect(shared.ListContainsBytes(superNodePayload.TransactionsRlp, mocks.MockTransactions.GetRlp(1))).To(BeTrue())
+			Expect(len(superNodePayload.ReceiptsRlp)).To(Equal(2))
+			Expect(shared.ListContainsBytes(superNodePayload.ReceiptsRlp, expectedRctForStorageRLP1)).To(BeTrue())
+			Expect(shared.ListContainsBytes(superNodePayload.ReceiptsRlp, expectedRctForStorageRLP2)).To(BeTrue())
+			Expect(len(superNodePayload.StateNodesRlp)).To(Equal(2))
+			Expect(superNodePayload.StateNodesRlp[mocks.ContractLeafKey]).To(Equal(mocks.ValueBytes))
+			Expect(superNodePayload.StateNodesRlp[mocks.AnotherContractLeafKey]).To(Equal(mocks.AnotherValueBytes))
+			Expect(superNodePayload.StorageNodesRlp).To(Equal(mocks.MockSeedNodePayload.StorageNodesRlp))
+		})
+
+		It("Applies filters from the provided config.Subscription", func() {
+			payload1, err := filterer.Filter(rctContractFilter, mocks.MockIPLDPayload)
+			Expect(err).ToNot(HaveOccurred())
+			superNodePayload1, ok := payload1.(eth.StreamPayload)
+			Expect(ok).To(BeTrue())
+			Expect(superNodePayload1.BlockNumber.Int64()).To(Equal(mocks.MockSeedNodePayload.BlockNumber.Int64()))
+			Expect(len(superNodePayload1.HeadersRlp)).To(Equal(0))
+			Expect(len(superNodePayload1.UnclesRlp)).To(Equal(0))
+			Expect(len(superNodePayload1.TransactionsRlp)).To(Equal(0))
+			Expect(len(superNodePayload1.StorageNodesRlp)).To(Equal(0))
+			Expect(len(superNodePayload1.StateNodesRlp)).To(Equal(0))
+			Expect(len(superNodePayload1.ReceiptsRlp)).To(Equal(1))
+			Expect(superNodePayload1.ReceiptsRlp[0]).To(Equal(expectedRctForStorageRLP2))
+
+			payload2, err := filterer.Filter(rctTopicsFilter, mocks.MockIPLDPayload)
+			Expect(err).ToNot(HaveOccurred())
+			superNodePayload2, ok := payload2.(eth.StreamPayload)
+			Expect(ok).To(BeTrue())
+			Expect(superNodePayload2.BlockNumber.Int64()).To(Equal(mocks.MockSeedNodePayload.BlockNumber.Int64()))
+			Expect(len(superNodePayload2.HeadersRlp)).To(Equal(0))
+			Expect(len(superNodePayload2.UnclesRlp)).To(Equal(0))
+			Expect(len(superNodePayload2.TransactionsRlp)).To(Equal(0))
+			Expect(len(superNodePayload2.StorageNodesRlp)).To(Equal(0))
+			Expect(len(superNodePayload2.StateNodesRlp)).To(Equal(0))
+			Expect(len(superNodePayload2.ReceiptsRlp)).To(Equal(1))
+			Expect(superNodePayload2.ReceiptsRlp[0]).To(Equal(expectedRctForStorageRLP1))
+
+			payload3, err := filterer.Filter(rctTopicsAndContractFilter, mocks.MockIPLDPayload)
+			Expect(err).ToNot(HaveOccurred())
+			superNodePayload3, ok := payload3.(eth.StreamPayload)
+			Expect(ok).To(BeTrue())
+			Expect(superNodePayload3.BlockNumber.Int64()).To(Equal(mocks.MockSeedNodePayload.BlockNumber.Int64()))
+			Expect(len(superNodePayload3.HeadersRlp)).To(Equal(0))
+			Expect(len(superNodePayload3.UnclesRlp)).To(Equal(0))
+			Expect(len(superNodePayload3.TransactionsRlp)).To(Equal(0))
+			Expect(len(superNodePayload3.StorageNodesRlp)).To(Equal(0))
+			Expect(len(superNodePayload3.StateNodesRlp)).To(Equal(0))
+			Expect(len(superNodePayload3.ReceiptsRlp)).To(Equal(1))
+			Expect(superNodePayload3.ReceiptsRlp[0]).To(Equal(expectedRctForStorageRLP1))
+
+			payload4, err := filterer.Filter(rctContractsAndTopicFilter, mocks.MockIPLDPayload)
+			Expect(err).ToNot(HaveOccurred())
+			superNodePayload4, ok := payload4.(eth.StreamPayload)
+			Expect(ok).To(BeTrue())
+			Expect(superNodePayload4.BlockNumber.Int64()).To(Equal(mocks.MockSeedNodePayload.BlockNumber.Int64()))
+			Expect(len(superNodePayload4.HeadersRlp)).To(Equal(0))
+			Expect(len(superNodePayload4.UnclesRlp)).To(Equal(0))
+			Expect(len(superNodePayload4.TransactionsRlp)).To(Equal(0))
+			Expect(len(superNodePayload4.StorageNodesRlp)).To(Equal(0))
+			Expect(len(superNodePayload4.StateNodesRlp)).To(Equal(0))
+			Expect(len(superNodePayload4.ReceiptsRlp)).To(Equal(1))
+			Expect(superNodePayload4.ReceiptsRlp[0]).To(Equal(expectedRctForStorageRLP2))
+
+			payload5, err := filterer.Filter(rctsForAllCollectedTrxs, mocks.MockIPLDPayload)
+			Expect(err).ToNot(HaveOccurred())
+			superNodePayload5, ok := payload5.(eth.StreamPayload)
+			Expect(ok).To(BeTrue())
+			Expect(superNodePayload5.BlockNumber.Int64()).To(Equal(mocks.MockSeedNodePayload.BlockNumber.Int64()))
+			Expect(len(superNodePayload5.HeadersRlp)).To(Equal(0))
+			Expect(len(superNodePayload5.UnclesRlp)).To(Equal(0))
+			Expect(len(superNodePayload5.TransactionsRlp)).To(Equal(2))
+			Expect(shared.ListContainsBytes(superNodePayload5.TransactionsRlp, mocks.MockTransactions.GetRlp(0))).To(BeTrue())
+			Expect(shared.ListContainsBytes(superNodePayload5.TransactionsRlp, mocks.MockTransactions.GetRlp(1))).To(BeTrue())
+			Expect(len(superNodePayload5.StorageNodesRlp)).To(Equal(0))
+			Expect(len(superNodePayload5.StateNodesRlp)).To(Equal(0))
+			Expect(len(superNodePayload5.ReceiptsRlp)).To(Equal(2))
+			Expect(shared.ListContainsBytes(superNodePayload5.ReceiptsRlp, expectedRctForStorageRLP1)).To(BeTrue())
+			Expect(shared.ListContainsBytes(superNodePayload5.ReceiptsRlp, expectedRctForStorageRLP2)).To(BeTrue())
+
+			payload6, err := filterer.Filter(rctsForSelectCollectedTrxs, mocks.MockIPLDPayload)
+			Expect(err).ToNot(HaveOccurred())
+			superNodePayload6, ok := payload6.(eth.StreamPayload)
+			Expect(ok).To(BeTrue())
+			Expect(superNodePayload6.BlockNumber.Int64()).To(Equal(mocks.MockSeedNodePayload.BlockNumber.Int64()))
+			Expect(len(superNodePayload6.HeadersRlp)).To(Equal(0))
+			Expect(len(superNodePayload6.UnclesRlp)).To(Equal(0))
+			Expect(len(superNodePayload6.TransactionsRlp)).To(Equal(1))
+			Expect(shared.ListContainsBytes(superNodePayload5.TransactionsRlp, mocks.MockTransactions.GetRlp(1))).To(BeTrue())
+			Expect(len(superNodePayload6.StorageNodesRlp)).To(Equal(0))
+			Expect(len(superNodePayload6.StateNodesRlp)).To(Equal(0))
+			Expect(len(superNodePayload6.ReceiptsRlp)).To(Equal(1))
+			Expect(superNodePayload4.ReceiptsRlp[0]).To(Equal(expectedRctForStorageRLP2))
+
+			payload7, err := filterer.Filter(stateFilter, mocks.MockIPLDPayload)
+			Expect(err).ToNot(HaveOccurred())
+			superNodePayload7, ok := payload7.(eth.StreamPayload)
+			Expect(ok).To(BeTrue())
+			Expect(superNodePayload7.BlockNumber.Int64()).To(Equal(mocks.MockSeedNodePayload.BlockNumber.Int64()))
+			Expect(len(superNodePayload7.HeadersRlp)).To(Equal(0))
+			Expect(len(superNodePayload7.UnclesRlp)).To(Equal(0))
+			Expect(len(superNodePayload7.TransactionsRlp)).To(Equal(0))
+			Expect(len(superNodePayload7.StorageNodesRlp)).To(Equal(0))
+			Expect(len(superNodePayload7.ReceiptsRlp)).To(Equal(0))
+			Expect(len(superNodePayload7.StateNodesRlp)).To(Equal(1))
+			Expect(superNodePayload7.StateNodesRlp[mocks.ContractLeafKey]).To(Equal(mocks.ValueBytes))
+		})
+	})
+})
+
+func getReceiptForStorageRLP(receipts types.Receipts, i int) []byte {
+	receiptForStorage := (*types.ReceiptForStorage)(receipts[i])
+	receiptBuffer := new(bytes.Buffer)
+	err := receiptForStorage.EncodeRLP(receiptBuffer)
+	Expect(err).ToNot(HaveOccurred())
+	return receiptBuffer.Bytes()
+}

pkg/super_node/eth/helpers.go

@@ -0,0 +1,33 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package eth
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/crypto"
+)
+
+// AddressToKey hashes an address
+func AddressToKey(address common.Address) common.Hash {
+	return crypto.Keccak256Hash(address[:])
+}
+
+// HexToKey hashes a hex (0x leading or not) string
+func HexToKey(hex string) common.Hash {
+	addr := common.FromHex(hex)
+	return crypto.Keccak256Hash(addr[:])
+}

pkg/super_node/eth/indexer.go

@@ -0,0 +1,150 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package eth
+
+import (
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/common"
+
+	"github.com/jmoiron/sqlx"
+	"github.com/lib/pq"
+	log "github.com/sirupsen/logrus"
+
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+)
+
+// Indexer satisfies the Indexer interface for ethereum
+type CIDIndexer struct {
+	db *postgres.DB
+}
+
+// NewCIDIndexer creates a new pointer to a Indexer which satisfies the CIDIndexer interface
+func NewCIDIndexer(db *postgres.DB) *CIDIndexer {
+	return &CIDIndexer{
+		db: db,
+	}
+}
+
+// Index indexes a cidPayload in Postgres
+func (in *CIDIndexer) Index(cids interface{}) error {
+	cidPayload, ok := cids.(*CIDPayload)
+	if !ok {
+		return fmt.Errorf("eth indexer expected cids type %T got %T", &CIDPayload{}, cids)
+	}
+	tx, err := in.db.Beginx()
+	if err != nil {
+		return err
+	}
+	headerID, err := in.indexHeaderCID(tx, cidPayload.HeaderCID)
+	if err != nil {
+		if err := tx.Rollback(); err != nil {
+			log.Error(err)
+		}
+		return err
+	}
+	for _, uncle := range cidPayload.UncleCIDs {
+		err := in.indexUncleCID(tx, uncle)
+		if err != nil {
+			if err := tx.Rollback(); err != nil {
+				log.Error(err)
+			}
+			return err
+		}
+	}
+	if err := in.indexTransactionAndReceiptCIDs(tx, cidPayload, headerID); err != nil {
+		if err := tx.Rollback(); err != nil {
+			log.Error(err)
+		}
+		return err
+	}
+	if err := in.indexStateAndStorageCIDs(tx, cidPayload, headerID); err != nil {
+		if err := tx.Rollback(); err != nil {
+			log.Error(err)
+		}
+		return err
+	}
+	return tx.Commit()
+}
+
+func (repo *CIDIndexer) indexHeaderCID(tx *sqlx.Tx, header HeaderModel) (int64, error) {
+	var headerID int64
+	err := tx.QueryRowx(`INSERT INTO public.header_cids (block_number, block_hash, parent_hash, cid, uncle, td) VALUES ($1, $2, $3, $4, $5, $6)
+								ON CONFLICT (block_number, block_hash) DO UPDATE SET (parent_hash, cid, uncle, td) = ($3, $4, $5, $6)
+								RETURNING id`,
+		header.BlockNumber, header.BlockHash, header.ParentHash, header.CID, false, header.TotalDifficulty).Scan(&headerID)
+	return headerID, err
+}
+
+func (in *CIDIndexer) indexUncleCID(tx *sqlx.Tx, uncle HeaderModel) error {
+	_, err := tx.Exec(`INSERT INTO public.header_cids (block_number, block_hash, parent_hash, cid, uncle) VALUES ($1, $2, $3, $4, $5)
+								ON CONFLICT (block_number, block_hash) DO UPDATE SET (parent_hash, cid, uncle) = ($3, $4, $5)`,
+		uncle.BlockNumber, uncle.BlockHash, uncle.ParentHash, uncle.CID, true)
+	return err
+}
+
+func (in *CIDIndexer) indexTransactionAndReceiptCIDs(tx *sqlx.Tx, payload *CIDPayload, headerID int64) error {
+	for _, trxCidMeta := range payload.TransactionCIDs {
+		var txID int64
+		err := tx.QueryRowx(`INSERT INTO public.transaction_cids (header_id, tx_hash, cid, dst, src) VALUES ($1, $2, $3, $4, $5)
+									ON CONFLICT (header_id, tx_hash) DO UPDATE SET (cid, dst, src) = ($3, $4, $5)
+									RETURNING id`,
+			headerID, trxCidMeta.TxHash, trxCidMeta.CID, trxCidMeta.Dst, trxCidMeta.Src).Scan(&txID)
+		if err != nil {
+			return err
+		}
+		receiptCidMeta, ok := payload.ReceiptCIDs[common.HexToHash(trxCidMeta.TxHash)]
+		if ok {
+			if err := in.indexReceiptCID(tx, receiptCidMeta, txID); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (in *CIDIndexer) indexReceiptCID(tx *sqlx.Tx, cidMeta ReceiptModel, txID int64) error {
+	_, err := tx.Exec(`INSERT INTO public.receipt_cids (tx_id, cid, contract, topic0s) VALUES ($1, $2, $3, $4)`,
+		txID, cidMeta.CID, cidMeta.Contract, pq.Array(cidMeta.Topic0s))
+	return err
+}
+
+func (in *CIDIndexer) indexStateAndStorageCIDs(tx *sqlx.Tx, payload *CIDPayload, headerID int64) error {
+	for _, stateCID := range payload.StateNodeCIDs {
+		var stateID int64
+		err := tx.QueryRowx(`INSERT INTO public.state_cids (header_id, state_key, cid, leaf) VALUES ($1, $2, $3, $4)
+									ON CONFLICT (header_id, state_key) DO UPDATE SET (cid, leaf) = ($3, $4)
+									RETURNING id`,
+			headerID, stateCID.StateKey, stateCID.CID, stateCID.Leaf).Scan(&stateID)
+		if err != nil {
+			return err
+		}
+		for _, storageCID := range payload.StorageNodeCIDs[common.HexToHash(stateCID.StateKey)] {
+			if err := in.indexStorageCID(tx, storageCID, stateID); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (in *CIDIndexer) indexStorageCID(tx *sqlx.Tx, storageCID StorageNodeModel, stateID int64) error {
+	_, err := tx.Exec(`INSERT INTO public.storage_cids (state_id, storage_key, cid, leaf) VALUES ($1, $2, $3, $4) 
+								   ON CONFLICT (state_id, storage_key) DO UPDATE SET (cid, leaf) = ($3, $4)`,
+		stateID, storageCID.StorageKey, storageCID.CID, storageCID.Leaf)
+	return err
+}

pkg/super_node/eth/indexer_test.go

@@ -0,0 +1,115 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package eth_test
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
+
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
+	eth2 "github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth/mocks"
+)
+
+var _ = Describe("Indexer", func() {
+	var (
+		db   *postgres.DB
+		err  error
+		repo *eth2.CIDIndexer
+	)
+	BeforeEach(func() {
+		db, err = eth.SetupDB()
+		Expect(err).ToNot(HaveOccurred())
+		repo = eth2.NewCIDIndexer(db)
+	})
+	AfterEach(func() {
+		eth.TearDownDB(db)
+	})
+
+	Describe("Index", func() {
+		It("Indexes CIDs and related metadata into vulcanizedb", func() {
+			err = repo.Index(mocks.MockCIDPayload)
+			Expect(err).ToNot(HaveOccurred())
+			pgStr := `SELECT cid, td FROM header_cids
+				WHERE block_number = $1 AND uncle IS FALSE`
+			// check header was properly indexed
+			type res struct {
+				CID string
+				TD  string
+			}
+			headers := new(res)
+			err = db.QueryRowx(pgStr, 1).StructScan(headers)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(headers.CID).To(Equal("mockHeaderCID"))
+			Expect(headers.TD).To(Equal("1337"))
+			// check trxs were properly indexed
+			trxs := make([]string, 0)
+			pgStr = `SELECT transaction_cids.cid FROM transaction_cids INNER JOIN header_cids ON (transaction_cids.header_id = header_cids.id)
+				WHERE header_cids.block_number = $1`
+			err = db.Select(&trxs, pgStr, 1)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(trxs)).To(Equal(2))
+			Expect(shared.ListContainsString(trxs, "mockTrxCID1")).To(BeTrue())
+			Expect(shared.ListContainsString(trxs, "mockTrxCID2")).To(BeTrue())
+			// check receipts were properly indexed
+			rcts := make([]string, 0)
+			pgStr = `SELECT receipt_cids.cid FROM receipt_cids, transaction_cids, header_cids
+				WHERE receipt_cids.tx_id = transaction_cids.id 
+				AND transaction_cids.header_id = header_cids.id
+				AND header_cids.block_number = $1`
+			err = db.Select(&rcts, pgStr, 1)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(rcts)).To(Equal(2))
+			Expect(shared.ListContainsString(rcts, "mockRctCID1")).To(BeTrue())
+			Expect(shared.ListContainsString(rcts, "mockRctCID2")).To(BeTrue())
+			// check that state nodes were properly indexed
+			stateNodes := make([]eth.StateNodeModel, 0)
+			pgStr = `SELECT state_cids.cid, state_cids.state_key, state_cids.leaf FROM state_cids INNER JOIN header_cids ON (state_cids.header_id = header_cids.id)
+				WHERE header_cids.block_number = $1`
+			err = db.Select(&stateNodes, pgStr, 1)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(stateNodes)).To(Equal(2))
+			for _, stateNode := range stateNodes {
+				if stateNode.CID == "mockStateCID1" {
+					Expect(stateNode.Leaf).To(Equal(true))
+					Expect(stateNode.StateKey).To(Equal(mocks.ContractLeafKey.Hex()))
+				}
+				if stateNode.CID == "mockStateCID2" {
+					Expect(stateNode.Leaf).To(Equal(true))
+					Expect(stateNode.StateKey).To(Equal(mocks.AnotherContractLeafKey.Hex()))
+				}
+			}
+			// check that storage nodes were properly indexed
+			storageNodes := make([]eth.StorageNodeWithStateKeyModel, 0)
+			pgStr = `SELECT storage_cids.cid, state_cids.state_key, storage_cids.storage_key, storage_cids.leaf FROM storage_cids, state_cids, header_cids
+				WHERE storage_cids.state_id = state_cids.id 
+				AND state_cids.header_id = header_cids.id
+				AND header_cids.block_number = $1`
+			err = db.Select(&storageNodes, pgStr, 1)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(storageNodes)).To(Equal(1))
+			Expect(storageNodes[0]).To(Equal(eth.StorageNodeWithStateKeyModel{
+				CID:        "mockStorageCID",
+				Leaf:       true,
+				StorageKey: "0x0000000000000000000000000000000000000000000000000000000000000001",
+				StateKey:   mocks.ContractLeafKey.Hex(),
+			}))
+		})
+	})
+})

pkg/super_node/eth/ipld_fetcher.go

@@ -0,0 +1,234 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package eth
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ipfs/go-block-format"
+	"github.com/ipfs/go-blockservice"
+	"github.com/ipfs/go-cid"
+	log "github.com/sirupsen/logrus"
+
+	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
+)
+
+var (
+	errUnexpectedNumberOfIPLDs = errors.New("ipfs batch fetch returned unexpected number of IPLDs")
+)
+
+// IPLDFetcher satisfies the IPLDFetcher interface for ethereum
+type IPLDFetcher struct {
+	BlockService blockservice.BlockService
+}
+
+// NewIPLDFetcher creates a pointer to a new IPLDFetcher
+func NewIPLDFetcher(ipfsPath string) (*IPLDFetcher, error) {
+	blockService, err := ipfs.InitIPFSBlockService(ipfsPath)
+	if err != nil {
+		return nil, err
+	}
+	return &IPLDFetcher{
+		BlockService: blockService,
+	}, nil
+}
+
+// Fetch is the exported method for fetching and returning all the IPLDS specified in the CIDWrapper
+func (f *IPLDFetcher) Fetch(cids interface{}) (interface{}, error) {
+	cidWrapper, ok := cids.(*CIDWrapper)
+	if !ok {
+		return nil, fmt.Errorf("eth fetcher: expected cids type %T got %T", &CIDWrapper{}, cids)
+	}
+	log.Debug("fetching iplds")
+	iplds := new(IPLDWrapper)
+	iplds.BlockNumber = cidWrapper.BlockNumber
+	var err error
+	iplds.Headers, err = f.FetchHeaders(cidWrapper.Headers)
+	if err != nil {
+		return nil, err
+	}
+	iplds.Uncles, err = f.FetchUncles(cidWrapper.Uncles)
+	if err != nil {
+		return nil, err
+	}
+	iplds.Transactions, err = f.FetchTrxs(cidWrapper.Transactions)
+	if err != nil {
+		return nil, err
+	}
+	iplds.Receipts, err = f.FetchRcts(cidWrapper.Receipts)
+	if err != nil {
+		return nil, err
+	}
+	iplds.StateNodes, err = f.FetchState(cidWrapper.StateNodes)
+	if err != nil {
+		return nil, err
+	}
+	iplds.StorageNodes, err = f.FetchStorage(cidWrapper.StorageNodes)
+	if err != nil {
+		return nil, err
+	}
+	return iplds, nil
+}
+
+// FetchHeaders fetches headers
+// It uses the f.fetchBatch method
+func (f *IPLDFetcher) FetchHeaders(cids []HeaderModel) ([]blocks.Block, error) {
+	log.Debug("fetching header iplds")
+	headerCids := make([]cid.Cid, 0, len(cids))
+	for _, c := range cids {
+		dc, err := cid.Decode(c.CID)
+		if err != nil {
+			return nil, err
+		}
+		headerCids = append(headerCids, dc)
+	}
+	headers := f.fetchBatch(headerCids)
+	if len(headers) != len(headerCids) {
+		log.Errorf("ipfs fetcher: number of header blocks returned (%d) does not match number expected (%d)", len(headers), len(headerCids))
+		return headers, errUnexpectedNumberOfIPLDs
+	}
+	return headers, nil
+}
+
+// FetchUncles fetches uncles
+// It uses the f.fetchBatch method
+func (f *IPLDFetcher) FetchUncles(cids []HeaderModel) ([]blocks.Block, error) {
+	log.Debug("fetching uncle iplds")
+	uncleCids := make([]cid.Cid, 0, len(cids))
+	for _, c := range cids {
+		dc, err := cid.Decode(c.CID)
+		if err != nil {
+			return nil, err
+		}
+		uncleCids = append(uncleCids, dc)
+	}
+	uncles := f.fetchBatch(uncleCids)
+	if len(uncles) != len(uncleCids) {
+		log.Errorf("ipfs fetcher: number of uncle blocks returned (%d) does not match number expected (%d)", len(uncles), len(uncleCids))
+		return uncles, errUnexpectedNumberOfIPLDs
+	}
+	return uncles, nil
+}
+
+// FetchTrxs fetches transactions
+// It uses the f.fetchBatch method
+func (f *IPLDFetcher) FetchTrxs(cids []TxModel) ([]blocks.Block, error) {
+	log.Debug("fetching transaction iplds")
+	trxCids := make([]cid.Cid, 0, len(cids))
+	for _, c := range cids {
+		dc, err := cid.Decode(c.CID)
+		if err != nil {
+			return nil, err
+		}
+		trxCids = append(trxCids, dc)
+	}
+	trxs := f.fetchBatch(trxCids)
+	if len(trxs) != len(trxCids) {
+		log.Errorf("ipfs fetcher: number of transaction blocks returned (%d) does not match number expected (%d)", len(trxs), len(trxCids))
+		return trxs, errUnexpectedNumberOfIPLDs
+	}
+	return trxs, nil
+}
+
+// FetchRcts fetches receipts
+// It uses the f.fetchBatch method
+func (f *IPLDFetcher) FetchRcts(cids []ReceiptModel) ([]blocks.Block, error) {
+	log.Debug("fetching receipt iplds")
+	rctCids := make([]cid.Cid, 0, len(cids))
+	for _, c := range cids {
+		dc, err := cid.Decode(c.CID)
+		if err != nil {
+			return nil, err
+		}
+		rctCids = append(rctCids, dc)
+	}
+	rcts := f.fetchBatch(rctCids)
+	if len(rcts) != len(rctCids) {
+		log.Errorf("ipfs fetcher: number of receipt blocks returned (%d) does not match number expected (%d)", len(rcts), len(rctCids))
+		return rcts, errUnexpectedNumberOfIPLDs
+	}
+	return rcts, nil
+}
+
+// FetchState fetches state nodes
+// It uses the single f.fetch method instead of the batch fetch, because it
+// needs to maintain the data's relation to state keys
+func (f *IPLDFetcher) FetchState(cids []StateNodeModel) (map[common.Hash]blocks.Block, error) {
+	log.Debug("fetching state iplds")
+	stateNodes := make(map[common.Hash]blocks.Block)
+	for _, stateNode := range cids {
+		if stateNode.CID == "" || stateNode.StateKey == "" {
+			continue
+		}
+		dc, err := cid.Decode(stateNode.CID)
+		if err != nil {
+			return nil, err
+		}
+		state, err := f.fetch(dc)
+		if err != nil {
+			return nil, err
+		}
+		stateNodes[common.HexToHash(stateNode.StateKey)] = state
+	}
+	return stateNodes, nil
+}
+
+// FetchStorage fetches storage nodes
+// It uses the single f.fetch method instead of the batch fetch, because it
+// needs to maintain the data's relation to state and storage keys
+func (f *IPLDFetcher) FetchStorage(cids []StorageNodeWithStateKeyModel) (map[common.Hash]map[common.Hash]blocks.Block, error) {
+	log.Debug("fetching storage iplds")
+	storageNodes := make(map[common.Hash]map[common.Hash]blocks.Block)
+	for _, storageNode := range cids {
+		if storageNode.CID == "" || storageNode.StorageKey == "" || storageNode.StateKey == "" {
+			continue
+		}
+		dc, err := cid.Decode(storageNode.CID)
+		if err != nil {
+			return nil, err
+		}
+		storage, err := f.fetch(dc)
+		if err != nil {
+			return nil, err
+		}
+		if storageNodes[common.HexToHash(storageNode.StateKey)] == nil {
+			storageNodes[common.HexToHash(storageNode.StateKey)] = make(map[common.Hash]blocks.Block)
+		}
+		storageNodes[common.HexToHash(storageNode.StateKey)][common.HexToHash(storageNode.StorageKey)] = storage
+	}
+	return storageNodes, nil
+}
+
+// fetch is used to fetch a single cid
+func (f *IPLDFetcher) fetch(cid cid.Cid) (blocks.Block, error) {
+	return f.BlockService.GetBlock(context.Background(), cid)
+}
+
+// fetchBatch is used to fetch a batch of IPFS data blocks by cid
+// There is no guarantee all are fetched, and no error in such a case, so
+// downstream we will need to confirm which CIDs were fetched in the result set
+func (f *IPLDFetcher) fetchBatch(cids []cid.Cid) []blocks.Block {
+	fetchedBlocks := make([]blocks.Block, 0, len(cids))
+	blockChan := f.BlockService.GetBlocks(context.Background(), cids)
+	for block := range blockChan {
+		fetchedBlocks = append(fetchedBlocks, block)
+	}
+	return fetchedBlocks
+}

pkg/super_node/eth/ipld_fetcher_test.go

@@ -0,0 +1,130 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package eth_test
+
+import (
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ipfs/go-block-format"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth/mocks"
+)
+
+var (
+	mockHeaderData    = []byte{0, 1, 2, 3, 4}
+	mockUncleData     = []byte{1, 2, 3, 4, 5}
+	mockTrxData       = []byte{2, 3, 4, 5, 6}
+	mockReceiptData   = []byte{3, 4, 5, 6, 7}
+	mockStateData     = []byte{4, 5, 6, 7, 8}
+	mockStorageData   = []byte{5, 6, 7, 8, 9}
+	mockStorageData2  = []byte{6, 7, 8, 9, 1}
+	mockHeaderBlock   = blocks.NewBlock(mockHeaderData)
+	mockUncleBlock    = blocks.NewBlock(mockUncleData)
+	mockTrxBlock      = blocks.NewBlock(mockTrxData)
+	mockReceiptBlock  = blocks.NewBlock(mockReceiptData)
+	mockStateBlock    = blocks.NewBlock(mockStateData)
+	mockStorageBlock1 = blocks.NewBlock(mockStorageData)
+	mockStorageBlock2 = blocks.NewBlock(mockStorageData2)
+	mockBlocks        = []blocks.Block{mockHeaderBlock, mockUncleBlock, mockTrxBlock, mockReceiptBlock, mockStateBlock, mockStorageBlock1, mockStorageBlock2}
+	mockBlockService  *mocks.MockIPFSBlockService
+	mockCIDWrapper    = &eth.CIDWrapper{
+		BlockNumber: big.NewInt(9000),
+		Headers: []eth.HeaderModel{
+			{
+				CID: mockHeaderBlock.Cid().String(),
+			},
+		},
+		Uncles: []eth.HeaderModel{
+			{
+				CID: mockUncleBlock.Cid().String(),
+			},
+		},
+		Transactions: []eth.TxModel{
+			{
+				CID: mockTrxBlock.Cid().String(),
+			},
+		},
+		Receipts: []eth.ReceiptModel{
+			{
+				CID: mockReceiptBlock.Cid().String(),
+			},
+		},
+		StateNodes: []eth.StateNodeModel{{
+			CID:      mockStateBlock.Cid().String(),
+			Leaf:     true,
+			StateKey: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470",
+		}},
+		StorageNodes: []eth.StorageNodeWithStateKeyModel{{
+			CID:        mockStorageBlock1.Cid().String(),
+			Leaf:       true,
+			StateKey:   "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470",
+			StorageKey: "0000000000000000000000000000000000000000000000000000000000000001",
+		},
+			{
+				CID:        mockStorageBlock2.Cid().String(),
+				Leaf:       true,
+				StateKey:   "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470",
+				StorageKey: "0000000000000000000000000000000000000000000000000000000000000002",
+			}},
+	}
+)
+
+var _ = Describe("Fetcher", func() {
+	Describe("FetchCIDs", func() {
+		BeforeEach(func() {
+			mockBlockService = new(mocks.MockIPFSBlockService)
+			err := mockBlockService.AddBlocks(mockBlocks)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(mockBlockService.Blocks)).To(Equal(7))
+		})
+
+		It("Fetches and returns IPLDs for the CIDs provided in the CIDWrapper", func() {
+			fetcher := new(eth.IPLDFetcher)
+			fetcher.BlockService = mockBlockService
+			i, err := fetcher.Fetch(mockCIDWrapper)
+			Expect(err).ToNot(HaveOccurred())
+			iplds, ok := i.(*eth.IPLDWrapper)
+			Expect(ok).To(BeTrue())
+			Expect(iplds.BlockNumber).To(Equal(mockCIDWrapper.BlockNumber))
+			Expect(len(iplds.Headers)).To(Equal(1))
+			Expect(iplds.Headers[0]).To(Equal(mockHeaderBlock))
+			Expect(len(iplds.Uncles)).To(Equal(1))
+			Expect(iplds.Uncles[0]).To(Equal(mockUncleBlock))
+			Expect(len(iplds.Transactions)).To(Equal(1))
+			Expect(iplds.Transactions[0]).To(Equal(mockTrxBlock))
+			Expect(len(iplds.Receipts)).To(Equal(1))
+			Expect(iplds.Receipts[0]).To(Equal(mockReceiptBlock))
+			Expect(len(iplds.StateNodes)).To(Equal(1))
+			stateNode, ok := iplds.StateNodes[common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")]
+			Expect(ok).To(BeTrue())
+			Expect(stateNode).To(Equal(mockStateBlock))
+			Expect(len(iplds.StorageNodes)).To(Equal(1))
+			storageNodes := iplds.StorageNodes[common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")]
+			Expect(len(storageNodes)).To(Equal(2))
+			storageNode1, ok := storageNodes[common.HexToHash("0000000000000000000000000000000000000000000000000000000000000001")]
+			Expect(ok).To(BeTrue())
+			Expect(storageNode1).To(Equal(mockStorageBlock1))
+			storageNode2, ok := storageNodes[common.HexToHash("0000000000000000000000000000000000000000000000000000000000000002")]
+			Expect(storageNode2).To(Equal(mockStorageBlock2))
+			Expect(ok).To(BeTrue())
+		})
+	})
+})

pkg/super_node/eth/mocks/blockservice.go

@@ -0,0 +1,86 @@
+package mocks
+
+import (
+	"context"
+	"errors"
+
+	"github.com/ipfs/go-block-format"
+	"github.com/ipfs/go-cid"
+	"github.com/ipfs/go-ipfs-blockstore"
+	"github.com/ipfs/go-ipfs-exchange-interface"
+)
+
+// MockIPFSBlockService is a mock for testing the ipfs fetcher
+type MockIPFSBlockService struct {
+	Blocks map[cid.Cid]blocks.Block
+}
+
+// GetBlock is used to retrieve a block from the mock BlockService
+func (bs *MockIPFSBlockService) GetBlock(ctx context.Context, c cid.Cid) (blocks.Block, error) {
+	if bs.Blocks == nil {
+		return nil, errors.New("BlockService has not been initialized")
+	}
+	blk, ok := bs.Blocks[c]
+	if ok {
+		return blk, nil
+	}
+	return nil, nil
+}
+
+// GetBlocks is used to retrieve a set of blocks from the mock BlockService
+func (bs *MockIPFSBlockService) GetBlocks(ctx context.Context, cs []cid.Cid) <-chan blocks.Block {
+	if bs.Blocks == nil {
+		panic("BlockService has not been initialized")
+	}
+	blkChan := make(chan blocks.Block)
+	go func() {
+		for _, c := range cs {
+			blk, ok := bs.Blocks[c]
+			if ok {
+				blkChan <- blk
+			}
+		}
+		close(blkChan)
+	}()
+	return blkChan
+}
+
+// AddBlock adds a block to the mock BlockService
+func (bs *MockIPFSBlockService) AddBlock(blk blocks.Block) error {
+	if bs.Blocks == nil {
+		bs.Blocks = make(map[cid.Cid]blocks.Block)
+	}
+	bs.Blocks[blk.Cid()] = blk
+	return nil
+}
+
+// AddBlocks adds a set of blocks to the mock BlockService
+func (bs *MockIPFSBlockService) AddBlocks(blks []blocks.Block) error {
+	if bs.Blocks == nil {
+		bs.Blocks = make(map[cid.Cid]blocks.Block)
+	}
+	for _, block := range blks {
+		bs.Blocks[block.Cid()] = block
+	}
+	return nil
+}
+
+// Close is here to satisfy the interface
+func (*MockIPFSBlockService) Close() error {
+	panic("implement me")
+}
+
+// Blockstore is here to satisfy the interface
+func (*MockIPFSBlockService) Blockstore() blockstore.Blockstore {
+	panic("implement me")
+}
+
+// DeleteBlock is here to satisfy the interface
+func (*MockIPFSBlockService) DeleteBlock(c cid.Cid) error {
+	panic("implement me")
+}
+
+// Exchange is here to satisfy the interface
+func (*MockIPFSBlockService) Exchange() exchange.Interface {
+	panic("implement me")
+}

pkg/super_node/eth/mocks/converter.go

@@ -0,0 +1,65 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package mocks
+
+import (
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/statediff"
+
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
+)
+
+// PayloadConverter is the underlying struct for the Converter interface
+type PayloadConverter struct {
+	PassedStatediffPayload statediff.Payload
+	ReturnIPLDPayload      *eth.IPLDPayload
+	ReturnErr              error
+}
+
+// Convert method is used to convert a geth statediff.Payload to a IPLDPayload
+func (pc *PayloadConverter) Convert(payload interface{}) (interface{}, error) {
+	stateDiffPayload, ok := payload.(statediff.Payload)
+	if !ok {
+		return nil, fmt.Errorf("convert expected payload type %T got %T", statediff.Payload{}, payload)
+	}
+	pc.PassedStatediffPayload = stateDiffPayload
+	return pc.ReturnIPLDPayload, pc.ReturnErr
+}
+
+// IterativePayloadConverter is the underlying struct for the Converter interface
+type IterativePayloadConverter struct {
+	PassedStatediffPayload []statediff.Payload
+	ReturnIPLDPayload      []*eth.IPLDPayload
+	ReturnErr              error
+	iteration              int
+}
+
+// Convert method is used to convert a geth statediff.Payload to a IPLDPayload
+func (pc *IterativePayloadConverter) Convert(payload interface{}) (interface{}, error) {
+	stateDiffPayload, ok := payload.(statediff.Payload)
+	if !ok {
+		return nil, fmt.Errorf("convert expected payload type %T got %T", statediff.Payload{}, payload)
+	}
+	pc.PassedStatediffPayload = append(pc.PassedStatediffPayload, stateDiffPayload)
+	if len(pc.PassedStatediffPayload) < pc.iteration+1 {
+		return nil, fmt.Errorf("IterativePayloadConverter does not have a payload to return at iteration %d", pc.iteration)
+	}
+	returnPayload := pc.ReturnIPLDPayload[pc.iteration]
+	pc.iteration++
+	return returnPayload, pc.ReturnErr
+}

pkg/super_node/eth/mocks/dag_putters.go

@@ -0,0 +1,53 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package mocks
+
+import (
+	"errors"
+
+	"github.com/ethereum/go-ethereum/common"
+)
+
+// DagPutter is a mock for testing the ipfs publisher
+type DagPutter struct {
+	CIDsToReturn []string
+	ErrToReturn  error
+}
+
+// DagPut returns the pre-loaded CIDs or error
+func (dp *DagPutter) DagPut(raw interface{}) ([]string, error) {
+	return dp.CIDsToReturn, dp.ErrToReturn
+}
+
+// MappedDagPutter is a mock for testing the ipfs publisher
+type MappedDagPutter struct {
+	CIDsToReturn map[common.Hash][]string
+	ErrToReturn  error
+}
+
+// DagPut returns the pre-loaded CIDs or error
+func (mdp *MappedDagPutter) DagPut(raw interface{}) ([]string, error) {
+	if mdp.CIDsToReturn == nil {
+		return nil, errors.New("mapped dag putter needs to be initialized with a map of cids to return")
+	}
+	by, ok := raw.([]byte)
+	if !ok {
+		return nil, errors.New("mapped dag putters can only dag put []byte values")
+	}
+	hash := common.BytesToHash(by)
+	return mdp.CIDsToReturn[hash], nil
+}

pkg/super_node/eth/mocks/fetcher.go

@@ -0,0 +1,50 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package mocks
+
+import (
+	"errors"
+	"sync/atomic"
+
+	"github.com/ethereum/go-ethereum/statediff"
+)
+
+// StateDiffFetcher mock for tests
+type StateDiffFetcher struct {
+	PayloadsToReturn     map[uint64]statediff.Payload
+	FetchErrs            map[uint64]error
+	CalledAtBlockHeights [][]uint64
+	CalledTimes          int64
+}
+
+// FetchStateDiffsAt mock method
+func (fetcher *StateDiffFetcher) FetchAt(blockHeights []uint64) ([]interface{}, error) {
+	if fetcher.PayloadsToReturn == nil {
+		return nil, errors.New("mock StateDiffFetcher needs to be initialized with payloads to return")
+	}
+	atomic.AddInt64(&fetcher.CalledTimes, 1) // thread-safe increment
+	fetcher.CalledAtBlockHeights = append(fetcher.CalledAtBlockHeights, blockHeights)
+	results := make([]interface{}, 0, len(blockHeights))
+	for _, height := range blockHeights {
+		results = append(results, fetcher.PayloadsToReturn[height])
+		err, ok := fetcher.FetchErrs[height]
+		if ok && err != nil {
+			return nil, err
+		}
+	}
+	return results, nil
+}

pkg/super_node/eth/mocks/indexer.go

@@ -0,0 +1,39 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package mocks
+
+import (
+	"fmt"
+
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
+)
+
+// CIDIndexer is the underlying struct for the Indexer interface
+type CIDIndexer struct {
+	PassedCIDPayload []*eth.CIDPayload
+	ReturnErr        error
+}
+
+// Index indexes a cidPayload in Postgres
+func (repo *CIDIndexer) Index(cids interface{}) error {
+	cidPayload, ok := cids.(*eth.CIDPayload)
+	if !ok {
+		return fmt.Errorf("index expected cids type %T got %T", &eth.CIDPayload{}, cids)
+	}
+	repo.PassedCIDPayload = append(repo.PassedCIDPayload, cidPayload)
+	return repo.ReturnErr
+}

pkg/super_node/eth/mocks/publisher.go

@@ -0,0 +1,63 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package mocks
+
+import (
+	"fmt"
+
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
+)
+
+// IPLDPublisher is the underlying struct for the Publisher interface
+type IPLDPublisher struct {
+	PassedIPLDPayload *eth.IPLDPayload
+	ReturnCIDPayload  *eth.CIDPayload
+	ReturnErr         error
+}
+
+// Publish publishes an IPLDPayload to IPFS and returns the corresponding CIDPayload
+func (pub *IPLDPublisher) Publish(payload interface{}) (interface{}, error) {
+	ipldPayload, ok := payload.(*eth.IPLDPayload)
+	if !ok {
+		return nil, fmt.Errorf("publish expected payload type %T got %T", &eth.IPLDPayload{}, payload)
+	}
+	pub.PassedIPLDPayload = ipldPayload
+	return pub.ReturnCIDPayload, pub.ReturnErr
+}
+
+// IterativeIPLDPublisher is the underlying struct for the Publisher interface; used in testing
+type IterativeIPLDPublisher struct {
+	PassedIPLDPayload []*eth.IPLDPayload
+	ReturnCIDPayload  []*eth.CIDPayload
+	ReturnErr         error
+	iteration         int
+}
+
+// Publish publishes an IPLDPayload to IPFS and returns the corresponding CIDPayload
+func (pub *IterativeIPLDPublisher) Publish(payload interface{}) (interface{}, error) {
+	ipldPayload, ok := payload.(*eth.IPLDPayload)
+	if !ok {
+		return nil, fmt.Errorf("publish expected payload type %T got %T", &eth.IPLDPayload{}, payload)
+	}
+	pub.PassedIPLDPayload = append(pub.PassedIPLDPayload, ipldPayload)
+	if len(pub.ReturnCIDPayload) < pub.iteration+1 {
+		return nil, fmt.Errorf("IterativeIPLDPublisher does not have a payload to return at iteration %d", pub.iteration)
+	}
+	returnPayload := pub.ReturnCIDPayload[pub.iteration]
+	pub.iteration++
+	return returnPayload, pub.ReturnErr
+}

pkg/super_node/eth/mocks/retriever.go

@@ -0,0 +1,64 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package mocks
+
+import (
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
+)
+
+// MockCIDRetriever is a mock CID retriever for use in tests
+type MockCIDRetriever struct {
+	GapsToRetrieve              []shared.Gap
+	GapsToRetrieveErr           error
+	CalledTimes                 int
+	FirstBlockNumberToReturn    int64
+	RetrieveFirstBlockNumberErr error
+}
+
+// RetrieveCIDs mock method
+func (*MockCIDRetriever) Retrieve(filter interface{}, blockNumber int64) (interface{}, bool, error) {
+	panic("implement me")
+}
+
+// RetrieveLastBlockNumber mock method
+func (*MockCIDRetriever) RetrieveLastBlockNumber() (int64, error) {
+	panic("implement me")
+}
+
+// RetrieveFirstBlockNumber mock method
+func (mcr *MockCIDRetriever) RetrieveFirstBlockNumber() (int64, error) {
+	return mcr.FirstBlockNumberToReturn, mcr.RetrieveFirstBlockNumberErr
+}
+
+// RetrieveGapsInData mock method
+func (mcr *MockCIDRetriever) RetrieveGapsInData() ([]shared.Gap, error) {
+	mcr.CalledTimes++
+	return mcr.GapsToRetrieve, mcr.GapsToRetrieveErr
+}
+
+// SetGapsToRetrieve mock method
+func (mcr *MockCIDRetriever) SetGapsToRetrieve(gaps []shared.Gap) {
+	if mcr.GapsToRetrieve == nil {
+		mcr.GapsToRetrieve = make([]shared.Gap, 0)
+	}
+	mcr.GapsToRetrieve = append(mcr.GapsToRetrieve, gaps...)
+}
+
+func (mcr *MockCIDRetriever) Database() *postgres.DB {
+	panic("implement me")
+}

pkg/super_node/eth/mocks/streamer.go

@@ -0,0 +1,43 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package mocks
+
+import (
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/statediff"
+)
+
+// StateDiffStreamer is the underlying struct for the Streamer interface
+type StateDiffStreamer struct {
+	PassedPayloadChan chan interface{}
+	ReturnSub         *rpc.ClientSubscription
+	ReturnErr         error
+	StreamPayloads    []statediff.Payload
+}
+
+// Stream is the main loop for subscribing to data from the Geth state diff process
+func (sds *StateDiffStreamer) Stream(payloadChan chan interface{}) (*rpc.ClientSubscription, error) {
+	sds.PassedPayloadChan = payloadChan
+
+	go func() {
+		for _, payload := range sds.StreamPayloads {
+			sds.PassedPayloadChan <- payload
+		}
+	}()
+
+	return sds.ReturnSub, sds.ReturnErr
+}

pkg/super_node/eth/mocks/test_data.go

@@ -0,0 +1,412 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package mocks
+
+import (
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
+	"math/big"
+	rand2 "math/rand"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/state"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/statediff"
+	"github.com/ipfs/go-block-format"
+	log "github.com/sirupsen/logrus"
+
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
+	eth2 "github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
+)
+
+// Test variables
+var (
+	// block data
+	BlockNumber = big.NewInt(1)
+	MockHeader  = types.Header{
+		Time:        0,
+		Number:      BlockNumber,
+		Root:        common.HexToHash("0x0"),
+		TxHash:      common.HexToHash("0x0"),
+		ReceiptHash: common.HexToHash("0x0"),
+	}
+	MockTransactions, MockReceipts, senderAddr = createTransactionsAndReceipts()
+	ReceiptsRlp, _                             = rlp.EncodeToBytes(MockReceipts)
+	MockBlock                                  = types.NewBlock(&MockHeader, MockTransactions, nil, MockReceipts)
+	MockBlockRlp, _                            = rlp.EncodeToBytes(MockBlock)
+	MockHeaderRlp, _                           = rlp.EncodeToBytes(MockBlock.Header())
+	MockTrxMeta                                = []eth.TxModel{
+		{
+			CID: "", // This is empty until we go to publish to ipfs
+			Src: senderAddr.Hex(),
+			Dst: "0x0000000000000000000000000000000000000000",
+		},
+		{
+			CID: "",
+			Src: senderAddr.Hex(),
+			Dst: "0x0000000000000000000000000000000000000001",
+		},
+	}
+	MockRctMeta = []eth.ReceiptModel{
+		{
+			CID: "",
+			Topic0s: []string{
+				"0x0000000000000000000000000000000000000000000000000000000000000004",
+			},
+			Contract: "0x0000000000000000000000000000000000000000",
+		},
+		{
+			CID: "",
+			Topic0s: []string{
+				"0x0000000000000000000000000000000000000000000000000000000000000005",
+			},
+			Contract: "0x0000000000000000000000000000000000000001",
+		},
+	}
+
+	// statediff data
+	CodeHash            = common.Hex2Bytes("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")
+	NonceValue          = rand2.Uint64()
+	anotherNonceValue   = rand2.Uint64()
+	BalanceValue        = rand2.Int63()
+	anotherBalanceValue = rand2.Int63()
+	ContractRoot        = common.HexToHash("0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421")
+	StoragePath         = common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes()
+	StorageKey          = common.HexToHash("0000000000000000000000000000000000000000000000000000000000000001").Bytes()
+	StorageValue        = common.Hex2Bytes("0x03")
+	storage             = []statediff.StorageDiff{{
+		Key:   StorageKey,
+		Value: StorageValue,
+		Path:  StoragePath,
+		Proof: [][]byte{},
+		Leaf:  true,
+	}}
+	emptyStorage           = make([]statediff.StorageDiff, 0)
+	Address                = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592")
+	ContractLeafKey        = eth.AddressToKey(Address)
+	AnotherAddress         = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476593")
+	AnotherContractLeafKey = eth.AddressToKey(AnotherAddress)
+	testAccount            = state.Account{
+		Nonce:    NonceValue,
+		Balance:  big.NewInt(BalanceValue),
+		Root:     ContractRoot,
+		CodeHash: CodeHash,
+	}
+	anotherTestAccount = state.Account{
+		Nonce:    anotherNonceValue,
+		Balance:  big.NewInt(anotherBalanceValue),
+		Root:     common.HexToHash("0x"),
+		CodeHash: nil,
+	}
+	ValueBytes, _        = rlp.EncodeToBytes(testAccount)
+	AnotherValueBytes, _ = rlp.EncodeToBytes(anotherTestAccount)
+	CreatedAccountDiffs  = []statediff.AccountDiff{
+		{
+			Key:     ContractLeafKey.Bytes(),
+			Value:   ValueBytes,
+			Storage: storage,
+			Leaf:    true,
+		},
+		{
+			Key:     AnotherContractLeafKey.Bytes(),
+			Value:   AnotherValueBytes,
+			Storage: emptyStorage,
+			Leaf:    true,
+		},
+	}
+
+	MockStateDiff = statediff.StateDiff{
+		BlockNumber:     BlockNumber,
+		BlockHash:       MockBlock.Hash(),
+		CreatedAccounts: CreatedAccountDiffs,
+	}
+	MockStateDiffBytes, _ = rlp.EncodeToBytes(MockStateDiff)
+	MockStateNodes        = []eth.TrieNode{
+		{
+			Key:   ContractLeafKey,
+			Value: ValueBytes,
+			Leaf:  true,
+		},
+		{
+			Key:   AnotherContractLeafKey,
+			Value: AnotherValueBytes,
+			Leaf:  true,
+		},
+	}
+	MockStorageNodes = map[common.Hash][]eth.TrieNode{
+		ContractLeafKey: {
+			{
+				Key:   common.BytesToHash(StorageKey),
+				Value: StorageValue,
+				Leaf:  true,
+			},
+		},
+	}
+
+	// aggregate payloads
+	MockStateDiffPayload = statediff.Payload{
+		BlockRlp:        MockBlockRlp,
+		StateDiffRlp:    MockStateDiffBytes,
+		ReceiptsRlp:     ReceiptsRlp,
+		TotalDifficulty: big.NewInt(1337),
+	}
+
+	MockIPLDPayload = &eth.IPLDPayload{
+		TotalDifficulty: big.NewInt(1337),
+		Block:           MockBlock,
+		Receipts:        MockReceipts,
+		HeaderRLP:       MockHeaderRlp,
+		TrxMetaData: []eth.TxModel{
+			{
+				CID: "",
+				Src: senderAddr.Hex(),
+				Dst: "0x0000000000000000000000000000000000000000",
+			},
+			{
+				CID: "",
+				Src: senderAddr.Hex(),
+				Dst: "0x0000000000000000000000000000000000000001",
+			},
+		},
+		ReceiptMetaData: []eth.ReceiptModel{
+			{
+				CID: "",
+				Topic0s: []string{
+					"0x0000000000000000000000000000000000000000000000000000000000000004",
+				},
+				Contract: "0x0000000000000000000000000000000000000000",
+			},
+			{
+				CID: "",
+				Topic0s: []string{
+					"0x0000000000000000000000000000000000000000000000000000000000000005",
+				},
+				Contract: "0x0000000000000000000000000000000000000001",
+			},
+		},
+		StorageNodes: MockStorageNodes,
+		StateNodes:   MockStateNodes,
+	}
+
+	MockCIDPayload = &eth.CIDPayload{
+		HeaderCID: eth2.HeaderModel{
+			BlockHash:       MockBlock.Hash().String(),
+			BlockNumber:     MockBlock.Number().String(),
+			CID:             "mockHeaderCID",
+			Uncle:           false,
+			ParentHash:      MockBlock.ParentHash().String(),
+			TotalDifficulty: "1337",
+		},
+		UncleCIDs: []eth2.HeaderModel{},
+		TransactionCIDs: []eth.TxModel{
+			{
+				TxHash: MockTransactions[0].Hash().String(),
+				CID:    "mockTrxCID1",
+				Dst:    "0x0000000000000000000000000000000000000000",
+				Src:    senderAddr.Hex(),
+			},
+			{
+				TxHash: MockTransactions[1].Hash().String(),
+				CID:    "mockTrxCID2",
+				Dst:    "0x0000000000000000000000000000000000000001",
+				Src:    senderAddr.Hex(),
+			},
+		},
+		ReceiptCIDs: map[common.Hash]eth.ReceiptModel{
+			MockTransactions[0].Hash(): {
+				CID:      "mockRctCID1",
+				Topic0s:  []string{"0x0000000000000000000000000000000000000000000000000000000000000004"},
+				Contract: "0x0000000000000000000000000000000000000000",
+			},
+			MockTransactions[1].Hash(): {
+				CID:      "mockRctCID2",
+				Topic0s:  []string{"0x0000000000000000000000000000000000000000000000000000000000000005"},
+				Contract: "0x0000000000000000000000000000000000000001",
+			},
+		},
+		StateNodeCIDs: []eth.StateNodeModel{
+			{
+				CID:      "mockStateCID1",
+				Leaf:     true,
+				StateKey: ContractLeafKey.String(),
+			},
+			{
+				CID:      "mockStateCID2",
+				Leaf:     true,
+				StateKey: AnotherContractLeafKey.String(),
+			},
+		},
+		StorageNodeCIDs: map[common.Hash][]eth.StorageNodeModel{
+			ContractLeafKey: {
+				{
+					CID:        "mockStorageCID",
+					StorageKey: "0x0000000000000000000000000000000000000000000000000000000000000001",
+					Leaf:       true,
+				},
+			},
+		},
+	}
+
+	MockCIDWrapper = &eth.CIDWrapper{
+		BlockNumber: big.NewInt(1),
+		Headers: []eth2.HeaderModel{
+			{
+				BlockNumber:     "1",
+				BlockHash:       MockBlock.Hash().String(),
+				ParentHash:      "0x0000000000000000000000000000000000000000000000000000000000000000",
+				CID:             "mockHeaderCID",
+				Uncle:           false,
+				TotalDifficulty: "1337",
+			},
+		},
+		Transactions: []eth2.TxModel{
+			{
+				CID: "mockTrxCID1",
+			},
+			{
+				TxHash: MockTransactions[1].Hash().String(),
+				CID:    "mockTrxCID2",
+				Dst:    "0x0000000000000000000000000000000000000001",
+				Src:    senderAddr.String(),
+			},
+		},
+		Receipts: []eth2.ReceiptModel{
+			{
+				CID:      "mockRctCID1",
+				Contract: "0x0000000000000000000000000000000000000000",
+				Topic0s: []string{
+					"0x0000000000000000000000000000000000000000000000000000000000000004",
+				},
+			},
+			{
+				CID:      "mockRctCID2",
+				Contract: "0x0000000000000000000000000000000000000001",
+				Topic0s: []string{
+					"0x0000000000000000000000000000000000000000000000000000000000000005",
+				},
+			},
+		},
+		Uncles: []eth2.HeaderModel{},
+		StateNodes: []eth.StateNodeModel{
+			{
+				CID:      "mockStateCID1",
+				Leaf:     true,
+				StateKey: ContractLeafKey.Hex(),
+			},
+			{
+				CID:      "mockStateCID2",
+				Leaf:     true,
+				StateKey: AnotherContractLeafKey.Hex(),
+			},
+		},
+		StorageNodes: []eth.StorageNodeWithStateKeyModel{
+			{
+				CID:        "mockStorageCID",
+				Leaf:       true,
+				StateKey:   ContractLeafKey.Hex(),
+				StorageKey: "0x0000000000000000000000000000000000000000000000000000000000000001",
+			},
+		},
+	}
+
+	MockIPLDWrapper = &eth.IPLDWrapper{
+		BlockNumber: big.NewInt(1),
+		Headers: []blocks.Block{
+			blocks.NewBlock(MockHeaderRlp),
+		},
+		Transactions: []blocks.Block{
+			blocks.NewBlock(MockTransactions.GetRlp(0)),
+			blocks.NewBlock(MockTransactions.GetRlp(1)),
+		},
+		Receipts: []blocks.Block{
+			blocks.NewBlock(MockReceipts.GetRlp(0)),
+			blocks.NewBlock(MockReceipts.GetRlp(1)),
+		},
+		StateNodes: map[common.Hash]blocks.Block{
+			ContractLeafKey:        blocks.NewBlock(ValueBytes),
+			AnotherContractLeafKey: blocks.NewBlock(AnotherValueBytes),
+		},
+		StorageNodes: map[common.Hash]map[common.Hash]blocks.Block{
+			ContractLeafKey: {
+				common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000001"): blocks.NewBlock(StorageValue),
+			},
+		},
+	}
+
+	MockSeedNodePayload = eth2.StreamPayload{
+		BlockNumber:     big.NewInt(1),
+		HeadersRlp:      [][]byte{MockHeaderRlp},
+		UnclesRlp:       [][]byte{},
+		TransactionsRlp: [][]byte{MockTransactions.GetRlp(0), MockTransactions.GetRlp(1)},
+		ReceiptsRlp:     [][]byte{MockTransactions.GetRlp(0), MockTransactions.GetRlp(1)},
+		StateNodesRlp: map[common.Hash][]byte{
+			ContractLeafKey:        ValueBytes,
+			AnotherContractLeafKey: AnotherValueBytes,
+		},
+		StorageNodesRlp: map[common.Hash]map[common.Hash][]byte{
+			ContractLeafKey: {
+				common.HexToHash("0x0000000000000000000000000000000000000000000000000000000000000001"): StorageValue,
+			},
+		},
+	}
+)
+
+// createTransactionsAndReceipts is a helper function to generate signed mock transactions and mock receipts with mock logs
+func createTransactionsAndReceipts() (types.Transactions, types.Receipts, common.Address) {
+	// make transactions
+	trx1 := types.NewTransaction(0, common.HexToAddress("0x0"), big.NewInt(1000), 50, big.NewInt(100), nil)
+	trx2 := types.NewTransaction(1, common.HexToAddress("0x1"), big.NewInt(2000), 100, big.NewInt(200), nil)
+	transactionSigner := types.MakeSigner(params.MainnetChainConfig, BlockNumber)
+	mockCurve := elliptic.P256()
+	mockPrvKey, err := ecdsa.GenerateKey(mockCurve, rand.Reader)
+	if err != nil {
+		log.Fatal(err)
+	}
+	signedTrx1, err := types.SignTx(trx1, transactionSigner, mockPrvKey)
+	if err != nil {
+		log.Fatal(err)
+	}
+	signedTrx2, err := types.SignTx(trx2, transactionSigner, mockPrvKey)
+	if err != nil {
+		log.Fatal(err)
+	}
+	senderAddr, err := types.Sender(transactionSigner, signedTrx1) // same for both trx
+	if err != nil {
+		log.Fatal(err)
+	}
+	// make receipts
+	mockTopic1 := common.HexToHash("0x04")
+	mockReceipt1 := types.NewReceipt(common.HexToHash("0x0").Bytes(), false, 50)
+	mockReceipt1.ContractAddress = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592")
+	mockLog1 := &types.Log{
+		Topics: []common.Hash{mockTopic1},
+	}
+	mockReceipt1.Logs = []*types.Log{mockLog1}
+	mockReceipt1.TxHash = signedTrx1.Hash()
+	mockTopic2 := common.HexToHash("0x05")
+	mockReceipt2 := types.NewReceipt(common.HexToHash("0x1").Bytes(), false, 100)
+	mockReceipt2.ContractAddress = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476593")
+	mockLog2 := &types.Log{
+		Topics: []common.Hash{mockTopic2},
+	}
+	mockReceipt2.Logs = []*types.Log{mockLog2}
+	mockReceipt2.TxHash = signedTrx2.Hash()
+	return types.Transactions{signedTrx1, signedTrx2}, types.Receipts{mockReceipt1, mockReceipt2}, senderAddr
+}

pkg/super_node/eth/models.go

@@ -0,0 +1,71 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package eth
+
+import "github.com/lib/pq"
+
+type HeaderModel struct {
+	ID              int64  `db:"id"`
+	BlockNumber     string `db:"block_number"`
+	BlockHash       string `db:"block_hash"`
+	ParentHash      string `db:"parent_hash"`
+	CID             string `db:"cid"`
+	Uncle           bool   `db:"uncle"`
+	TotalDifficulty string `db:"td"`
+}
+
+type TxModel struct {
+	ID       int64  `db:"id"`
+	HeaderID int64  `db:"header_id"`
+	TxHash   string `db:"tx_hash"`
+	CID      string `db:"cid"`
+	Dst      string `db:"dst"`
+	Src      string `db:"src"`
+}
+
+type ReceiptModel struct {
+	ID       int64          `db:"id"`
+	TxID     int64          `db:"tx_id"`
+	CID      string         `db:"cid"`
+	Contract string         `db:"contract"`
+	Topic0s  pq.StringArray `db:"topic0s"`
+}
+
+type StateNodeModel struct {
+	ID       int64  `db:"id"`
+	HeaderID int64  `db:"header_id"`
+	StateKey string `db:"state_key"`
+	Leaf     bool   `db:"leaf"`
+	CID      string `db:"cid"`
+}
+
+type StorageNodeModel struct {
+	ID         int64  `db:"id"`
+	StateID    int64  `db:"state_id"`
+	StorageKey string `db:"storage_key"`
+	Leaf       bool   `db:"leaf"`
+	CID        string `db:"cid"`
+}
+
+type StorageNodeWithStateKeyModel struct {
+	ID         int64  `db:"id"`
+	StateID    int64  `db:"state_id"`
+	StateKey   string `db:"state_key"`
+	StorageKey string `db:"storage_key"`
+	Leaf       bool   `db:"leaf"`
+	CID        string `db:"cid"`
+}

pkg/super_node/eth/payload_fetcher.go

@@ -0,0 +1,74 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package eth
+
+import (
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/statediff"
+
+	"github.com/vulcanize/vulcanizedb/pkg/eth/client"
+)
+
+// BatchClient is an interface to a batch-fetching geth rpc client; created to allow mock insertion
+type BatchClient interface {
+	BatchCall(batch []client.BatchElem) error
+}
+
+// PayloadFetcher satisfies the PayloadFetcher interface for ethereum
+type PayloadFetcher struct {
+	// PayloadFetcher is thread-safe as long as the underlying client is thread-safe, since it has/modifies no other state
+	// http.Client is thread-safe
+	client BatchClient
+}
+
+const method = "statediff_stateDiffAt"
+
+// NewStateDiffFetcher returns a PayloadFetcher
+func NewPayloadFetcher(bc BatchClient) *PayloadFetcher {
+	return &PayloadFetcher{
+		client: bc,
+	}
+}
+
+// FetchAt fetches the statediff payloads at the given block heights
+// Calls StateDiffAt(ctx context.Context, blockNumber uint64) (*Payload, error)
+func (fetcher *PayloadFetcher) FetchAt(blockHeights []uint64) ([]interface{}, error) {
+	batch := make([]client.BatchElem, 0)
+	for _, height := range blockHeights {
+		batch = append(batch, client.BatchElem{
+			Method: method,
+			Args:   []interface{}{height},
+			Result: new(statediff.Payload),
+		})
+	}
+	batchErr := fetcher.client.BatchCall(batch)
+	if batchErr != nil {
+		return nil, fmt.Errorf("PayloadFetcher err: %s", batchErr.Error())
+	}
+	results := make([]interface{}, 0, len(blockHeights))
+	for _, batchElem := range batch {
+		if batchElem.Error != nil {
+			return nil, fmt.Errorf("PayloadFetcher err: %s", batchElem.Error.Error())
+		}
+		payload, ok := batchElem.Result.(*statediff.Payload)
+		if ok {
+			results = append(results, *payload)
+		}
+	}
+	return results, nil
+}

pkg/super_node/eth/payload_fetcher_test.go

@@ -0,0 +1,59 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package eth_test
+
+import (
+	"github.com/ethereum/go-ethereum/statediff"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/libraries/shared/mocks"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/test_data"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
+)
+
+var _ = Describe("StateDiffFetcher", func() {
+	Describe("FetchStateDiffsAt", func() {
+		var (
+			mc               *mocks.BackFillerClient
+			stateDiffFetcher *eth.PayloadFetcher
+		)
+		BeforeEach(func() {
+			mc = new(mocks.BackFillerClient)
+			setDiffAtErr1 := mc.SetReturnDiffAt(test_data.BlockNumber.Uint64(), test_data.MockStatediffPayload)
+			Expect(setDiffAtErr1).ToNot(HaveOccurred())
+			setDiffAtErr2 := mc.SetReturnDiffAt(test_data.BlockNumber2.Uint64(), test_data.MockStatediffPayload2)
+			Expect(setDiffAtErr2).ToNot(HaveOccurred())
+			stateDiffFetcher = eth.NewPayloadFetcher(mc)
+		})
+		It("Batch calls statediff_stateDiffAt", func() {
+			blockHeights := []uint64{
+				test_data.BlockNumber.Uint64(),
+				test_data.BlockNumber2.Uint64(),
+			}
+			stateDiffPayloads, fetchErr := stateDiffFetcher.FetchAt(blockHeights)
+			Expect(fetchErr).ToNot(HaveOccurred())
+			Expect(len(stateDiffPayloads)).To(Equal(2))
+			payload1, ok := stateDiffPayloads[0].(statediff.Payload)
+			Expect(ok).To(BeTrue())
+			payload2, ok := stateDiffPayloads[1].(statediff.Payload)
+			Expect(ok).To(BeTrue())
+			Expect(payload1).To(Equal(test_data.MockStatediffPayload))
+			Expect(payload2).To(Equal(test_data.MockStatediffPayload2))
+		})
+	})
+})

pkg/super_node/eth/publisher.go

@@ -0,0 +1,226 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package eth
+
+import (
+	"errors"
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/rlp"
+
+	"github.com/vulcanize/eth-block-extractor/pkg/ipfs"
+	"github.com/vulcanize/eth-block-extractor/pkg/ipfs/eth_block_header"
+	"github.com/vulcanize/eth-block-extractor/pkg/ipfs/eth_block_receipts"
+	"github.com/vulcanize/eth-block-extractor/pkg/ipfs/eth_block_transactions"
+	"github.com/vulcanize/eth-block-extractor/pkg/ipfs/eth_state_trie"
+	"github.com/vulcanize/eth-block-extractor/pkg/ipfs/eth_storage_trie"
+	rlp2 "github.com/vulcanize/eth-block-extractor/pkg/wrappers/rlp"
+)
+
+// IPLDPublisher satisfies the IPLDPublisher for ethereum
+type IPLDPublisher struct {
+	HeaderPutter      ipfs.DagPutter
+	TransactionPutter ipfs.DagPutter
+	ReceiptPutter     ipfs.DagPutter
+	StatePutter       ipfs.DagPutter
+	StoragePutter     ipfs.DagPutter
+}
+
+// NewIPLDPublisher creates a pointer to a new Publisher which satisfies the IPLDPublisher interface
+func NewIPLDPublisher(ipfsPath string) (*IPLDPublisher, error) {
+	node, err := ipfs.InitIPFSNode(ipfsPath)
+	if err != nil {
+		return nil, err
+	}
+	return &IPLDPublisher{
+		HeaderPutter:      eth_block_header.NewBlockHeaderDagPutter(node, rlp2.RlpDecoder{}),
+		TransactionPutter: eth_block_transactions.NewBlockTransactionsDagPutter(node),
+		ReceiptPutter:     eth_block_receipts.NewEthBlockReceiptDagPutter(node),
+		StatePutter:       eth_state_trie.NewStateTrieDagPutter(node),
+		StoragePutter:     eth_storage_trie.NewStorageTrieDagPutter(node),
+	}, nil
+}
+
+// Publish publishes an IPLDPayload to IPFS and returns the corresponding CIDPayload
+func (pub *IPLDPublisher) Publish(payload interface{}) (interface{}, error) {
+	ipldPayload, ok := payload.(*IPLDPayload)
+	if !ok {
+		return nil, fmt.Errorf("eth publisher expected payload type %T got %T", &IPLDPayload{}, payload)
+	}
+	// Process and publish headers
+	headerCid, err := pub.publishHeader(ipldPayload.HeaderRLP)
+	if err != nil {
+		return nil, err
+	}
+	header := HeaderModel{
+		CID:             headerCid,
+		ParentHash:      ipldPayload.Block.ParentHash().String(),
+		BlockNumber:     ipldPayload.Block.Number().String(),
+		Uncle:           false,
+		BlockHash:       ipldPayload.Block.Hash().String(),
+		TotalDifficulty: ipldPayload.TotalDifficulty.String(),
+	}
+
+	// Process and publish uncles
+	uncleCids := make([]HeaderModel, 0, len(ipldPayload.Block.Uncles()))
+	for _, uncle := range ipldPayload.Block.Uncles() {
+		uncleRlp, err := rlp.EncodeToBytes(uncle)
+		if err != nil {
+			return nil, err
+		}
+		uncleCid, err := pub.publishHeader(uncleRlp)
+		if err != nil {
+			return nil, err
+		}
+		uncleCids = append(uncleCids, HeaderModel{
+			CID:         uncleCid,
+			ParentHash:  uncle.ParentHash.String(),
+			Uncle:       true,
+			BlockHash:   uncle.Hash().String(),
+			BlockNumber: uncle.Number.String(),
+		})
+	}
+
+	// Process and publish transactions
+	transactionCids, err := pub.publishTransactions(ipldPayload.Block.Body(), ipldPayload.TrxMetaData)
+	if err != nil {
+		return nil, err
+	}
+
+	// Process and publish receipts
+	receiptsCids, err := pub.publishReceipts(ipldPayload.Receipts, ipldPayload.ReceiptMetaData)
+	if err != nil {
+		return nil, err
+	}
+
+	// Process and publish state leafs
+	stateNodeCids, err := pub.publishStateNodes(ipldPayload.StateNodes)
+	if err != nil {
+		return nil, err
+	}
+
+	// Process and publish storage leafs
+	storageNodeCids, err := pub.publishStorageNodes(ipldPayload.StorageNodes)
+	if err != nil {
+		return nil, err
+	}
+
+	// Package CIDs and their metadata into a single struct
+	return &CIDPayload{
+		HeaderCID:       header,
+		UncleCIDs:       uncleCids,
+		TransactionCIDs: transactionCids,
+		ReceiptCIDs:     receiptsCids,
+		StateNodeCIDs:   stateNodeCids,
+		StorageNodeCIDs: storageNodeCids,
+	}, nil
+}
+
+func (pub *IPLDPublisher) publishHeader(headerRLP []byte) (string, error) {
+	headerCids, err := pub.HeaderPutter.DagPut(headerRLP)
+	if err != nil {
+		return "", err
+	}
+	if len(headerCids) != 1 {
+		return "", errors.New("single CID expected to be returned for header")
+	}
+	return headerCids[0], nil
+}
+
+func (pub *IPLDPublisher) publishTransactions(blockBody *types.Body, trxMeta []TxModel) ([]TxModel, error) {
+	transactionCids, err := pub.TransactionPutter.DagPut(blockBody)
+	if err != nil {
+		return nil, err
+	}
+	if len(transactionCids) != len(blockBody.Transactions) {
+		return nil, errors.New("expected one CID for each transaction")
+	}
+	mappedTrxCids := make([]TxModel, len(transactionCids))
+	for i, trx := range blockBody.Transactions {
+		mappedTrxCids[i] = TxModel{
+			CID:    transactionCids[i],
+			TxHash: trx.Hash().Hex(),
+			Src:    trxMeta[i].Src,
+			Dst:    trxMeta[i].Dst,
+		}
+	}
+	return mappedTrxCids, nil
+}
+
+func (pub *IPLDPublisher) publishReceipts(receipts types.Receipts, receiptMeta []ReceiptModel) (map[common.Hash]ReceiptModel, error) {
+	receiptsCids, err := pub.ReceiptPutter.DagPut(receipts)
+	if err != nil {
+		return nil, err
+	}
+	if len(receiptsCids) != len(receipts) {
+		return nil, errors.New("expected one CID for each receipt")
+	}
+	// Map receipt cids to their transaction hashes
+	mappedRctCids := make(map[common.Hash]ReceiptModel, len(receiptsCids))
+	for i, rct := range receipts {
+		mappedRctCids[rct.TxHash] = ReceiptModel{
+			CID:      receiptsCids[i],
+			Contract: receiptMeta[i].Contract,
+			Topic0s:  receiptMeta[i].Topic0s,
+		}
+	}
+	return mappedRctCids, nil
+}
+
+func (pub *IPLDPublisher) publishStateNodes(stateNodes []TrieNode) ([]StateNodeModel, error) {
+	stateNodeCids := make([]StateNodeModel, 0, len(stateNodes))
+	for _, node := range stateNodes {
+		stateNodeCid, err := pub.StatePutter.DagPut(node.Value)
+		if err != nil {
+			return nil, err
+		}
+		if len(stateNodeCid) != 1 {
+			return nil, errors.New("single CID expected to be returned for state leaf")
+		}
+		stateNodeCids = append(stateNodeCids, StateNodeModel{
+			StateKey: node.Key.String(),
+			CID:      stateNodeCid[0],
+			Leaf:     node.Leaf,
+		})
+	}
+	return stateNodeCids, nil
+}
+
+func (pub *IPLDPublisher) publishStorageNodes(storageNodes map[common.Hash][]TrieNode) (map[common.Hash][]StorageNodeModel, error) {
+	storageLeafCids := make(map[common.Hash][]StorageNodeModel)
+	for addrKey, storageTrie := range storageNodes {
+		storageLeafCids[addrKey] = make([]StorageNodeModel, 0, len(storageTrie))
+		for _, node := range storageTrie {
+			storageNodeCid, err := pub.StoragePutter.DagPut(node.Value)
+			if err != nil {
+				return nil, err
+			}
+			if len(storageNodeCid) != 1 {
+				return nil, errors.New("single CID expected to be returned for storage leaf")
+			}
+			// Map storage node cids to their state key hashes
+			storageLeafCids[addrKey] = append(storageLeafCids[addrKey], StorageNodeModel{
+				StorageKey: node.Key.Hex(),
+				CID:        storageNodeCid[0],
+				Leaf:       node.Leaf,
+			})
+		}
+	}
+	return storageLeafCids, nil
+}

pkg/super_node/eth/publisher_test.go

@@ -0,0 +1,85 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package eth_test
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth/mocks"
+)
+
+var (
+	mockHeaderDagPutter  *mocks.DagPutter
+	mockTrxDagPutter     *mocks.DagPutter
+	mockRctDagPutter     *mocks.DagPutter
+	mockStateDagPutter   *mocks.MappedDagPutter
+	mockStorageDagPutter *mocks.DagPutter
+)
+
+var _ = Describe("Publisher", func() {
+	BeforeEach(func() {
+		mockHeaderDagPutter = new(mocks.DagPutter)
+		mockTrxDagPutter = new(mocks.DagPutter)
+		mockRctDagPutter = new(mocks.DagPutter)
+		mockStateDagPutter = new(mocks.MappedDagPutter)
+		mockStorageDagPutter = new(mocks.DagPutter)
+	})
+
+	Describe("Publish", func() {
+		It("Publishes the passed IPLDPayload objects to IPFS and returns a CIDPayload for indexing", func() {
+			mockHeaderDagPutter.CIDsToReturn = []string{"mockHeaderCID"}
+			mockTrxDagPutter.CIDsToReturn = []string{"mockTrxCID1", "mockTrxCID2"}
+			mockRctDagPutter.CIDsToReturn = []string{"mockRctCID1", "mockRctCID2"}
+			val1 := common.BytesToHash(mocks.MockIPLDPayload.StateNodes[0].Value)
+			val2 := common.BytesToHash(mocks.MockIPLDPayload.StateNodes[1].Value)
+			mockStateDagPutter.CIDsToReturn = map[common.Hash][]string{
+				val1: {"mockStateCID1"},
+				val2: {"mockStateCID2"},
+			}
+			mockStorageDagPutter.CIDsToReturn = []string{"mockStorageCID"}
+			publisher := eth.IPLDPublisher{
+				HeaderPutter:      mockHeaderDagPutter,
+				TransactionPutter: mockTrxDagPutter,
+				ReceiptPutter:     mockRctDagPutter,
+				StatePutter:       mockStateDagPutter,
+				StoragePutter:     mockStorageDagPutter,
+			}
+			payload, err := publisher.Publish(mocks.MockIPLDPayload)
+			Expect(err).ToNot(HaveOccurred())
+			cidPayload, ok := payload.(*eth.CIDPayload)
+			Expect(ok).To(BeTrue())
+			Expect(cidPayload.HeaderCID.TotalDifficulty).To(Equal(mocks.MockIPLDPayload.TotalDifficulty.String()))
+			Expect(cidPayload.HeaderCID.BlockNumber).To(Equal(mocks.MockCIDPayload.HeaderCID.BlockNumber))
+			Expect(cidPayload.HeaderCID.BlockHash).To(Equal(mocks.MockCIDPayload.HeaderCID.BlockHash))
+			Expect(cidPayload.UncleCIDs).To(Equal(mocks.MockCIDPayload.UncleCIDs))
+			Expect(cidPayload.HeaderCID).To(Equal(mocks.MockCIDPayload.HeaderCID))
+			Expect(len(cidPayload.TransactionCIDs)).To(Equal(2))
+			Expect(cidPayload.TransactionCIDs[0]).To(Equal(mocks.MockCIDPayload.TransactionCIDs[0]))
+			Expect(cidPayload.TransactionCIDs[1]).To(Equal(mocks.MockCIDPayload.TransactionCIDs[1]))
+			Expect(len(cidPayload.ReceiptCIDs)).To(Equal(2))
+			Expect(cidPayload.ReceiptCIDs[mocks.MockTransactions[0].Hash()]).To(Equal(mocks.MockCIDPayload.ReceiptCIDs[mocks.MockTransactions[0].Hash()]))
+			Expect(cidPayload.ReceiptCIDs[mocks.MockTransactions[1].Hash()]).To(Equal(mocks.MockCIDPayload.ReceiptCIDs[mocks.MockTransactions[1].Hash()]))
+			Expect(len(cidPayload.StateNodeCIDs)).To(Equal(2))
+			Expect(cidPayload.StateNodeCIDs[0]).To(Equal(mocks.MockCIDPayload.StateNodeCIDs[0]))
+			Expect(cidPayload.StateNodeCIDs[1]).To(Equal(mocks.MockCIDPayload.StateNodeCIDs[1]))
+			Expect(cidPayload.StorageNodeCIDs).To(Equal(mocks.MockCIDPayload.StorageNodeCIDs))
+		})
+	})
+})

pkg/super_node/eth/resolver.go

@@ -0,0 +1,100 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package eth
+
+import (
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ipfs/go-block-format"
+)
+
+// IPLDResolver satisfies the IPLDResolver interface for ethereum
+type IPLDResolver struct{}
+
+// NewIPLDResolver returns a pointer to an IPLDResolver which satisfies the IPLDResolver interface
+func NewIPLDResolver() *IPLDResolver {
+	return &IPLDResolver{}
+}
+
+// Resolve is the exported method for resolving all of the ETH IPLDs packaged in an IpfsBlockWrapper
+func (eir *IPLDResolver) Resolve(iplds interface{}) (interface{}, error) {
+	ipfsBlocks, ok := iplds.(*IPLDWrapper)
+	if !ok {
+		return StreamPayload{}, fmt.Errorf("eth resolver expected iplds type %T got %T", &IPLDWrapper{}, iplds)
+	}
+	return StreamPayload{
+		BlockNumber:     ipfsBlocks.BlockNumber,
+		HeadersRlp:      eir.ResolveHeaders(ipfsBlocks.Headers),
+		UnclesRlp:       eir.ResolveUncles(ipfsBlocks.Uncles),
+		TransactionsRlp: eir.ResolveTransactions(ipfsBlocks.Transactions),
+		ReceiptsRlp:     eir.ResolveReceipts(ipfsBlocks.Receipts),
+		StateNodesRlp:   eir.ResolveState(ipfsBlocks.StateNodes),
+		StorageNodesRlp: eir.ResolveStorage(ipfsBlocks.StorageNodes),
+	}, nil
+}
+
+func (eir *IPLDResolver) ResolveHeaders(iplds []blocks.Block) [][]byte {
+	headerRlps := make([][]byte, 0, len(iplds))
+	for _, ipld := range iplds {
+		headerRlps = append(headerRlps, ipld.RawData())
+	}
+	return headerRlps
+}
+
+func (eir *IPLDResolver) ResolveUncles(iplds []blocks.Block) [][]byte {
+	uncleRlps := make([][]byte, 0, len(iplds))
+	for _, ipld := range iplds {
+		uncleRlps = append(uncleRlps, ipld.RawData())
+	}
+	return uncleRlps
+}
+
+func (eir *IPLDResolver) ResolveTransactions(iplds []blocks.Block) [][]byte {
+	trxs := make([][]byte, 0, len(iplds))
+	for _, ipld := range iplds {
+		trxs = append(trxs, ipld.RawData())
+	}
+	return trxs
+}
+
+func (eir *IPLDResolver) ResolveReceipts(iplds []blocks.Block) [][]byte {
+	rcts := make([][]byte, 0, len(iplds))
+	for _, ipld := range iplds {
+		rcts = append(rcts, ipld.RawData())
+	}
+	return rcts
+}
+
+func (eir *IPLDResolver) ResolveState(iplds map[common.Hash]blocks.Block) map[common.Hash][]byte {
+	stateNodes := make(map[common.Hash][]byte, len(iplds))
+	for key, ipld := range iplds {
+		stateNodes[key] = ipld.RawData()
+	}
+	return stateNodes
+}
+
+func (eir *IPLDResolver) ResolveStorage(iplds map[common.Hash]map[common.Hash]blocks.Block) map[common.Hash]map[common.Hash][]byte {
+	storageNodes := make(map[common.Hash]map[common.Hash][]byte)
+	for stateKey, storageIPLDs := range iplds {
+		storageNodes[stateKey] = make(map[common.Hash][]byte)
+		for storageKey, storageVal := range storageIPLDs {
+			storageNodes[stateKey][storageKey] = storageVal.RawData()
+		}
+	}
+	return storageNodes
+}

pkg/super_node/eth/resolver_test.go

@@ -0,0 +1,55 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package eth_test
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
+
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth/mocks"
+)
+
+var (
+	resolver *eth.IPLDResolver
+)
+
+var _ = Describe("Resolver", func() {
+	Describe("ResolveIPLDs", func() {
+		BeforeEach(func() {
+			resolver = eth.NewIPLDResolver()
+		})
+		It("Resolves IPLD data to their correct geth data types and packages them to send to requesting transformers", func() {
+			payload, err := resolver.Resolve(mocks.MockIPLDWrapper)
+			Expect(err).ToNot(HaveOccurred())
+			superNodePayload, ok := payload.(eth.StreamPayload)
+			Expect(ok).To(BeTrue())
+			Expect(superNodePayload.BlockNumber.Int64()).To(Equal(mocks.MockSeedNodePayload.BlockNumber.Int64()))
+			Expect(superNodePayload.HeadersRlp).To(Equal(mocks.MockSeedNodePayload.HeadersRlp))
+			Expect(superNodePayload.UnclesRlp).To(Equal(mocks.MockSeedNodePayload.UnclesRlp))
+			Expect(len(superNodePayload.TransactionsRlp)).To(Equal(2))
+			Expect(shared.ListContainsBytes(superNodePayload.TransactionsRlp, mocks.MockTransactions.GetRlp(0))).To(BeTrue())
+			Expect(shared.ListContainsBytes(superNodePayload.TransactionsRlp, mocks.MockTransactions.GetRlp(1))).To(BeTrue())
+			Expect(len(superNodePayload.ReceiptsRlp)).To(Equal(2))
+			Expect(shared.ListContainsBytes(superNodePayload.ReceiptsRlp, mocks.MockReceipts.GetRlp(0))).To(BeTrue())
+			Expect(shared.ListContainsBytes(superNodePayload.ReceiptsRlp, mocks.MockReceipts.GetRlp(1))).To(BeTrue())
+			Expect(len(superNodePayload.StateNodesRlp)).To(Equal(2))
+			Expect(superNodePayload.StorageNodesRlp).To(Equal(mocks.MockSeedNodePayload.StorageNodesRlp))
+		})
+	})
+})

pkg/super_node/eth/retriever.go

@@ -0,0 +1,343 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package eth
+
+import (
+	"fmt"
+	"math/big"
+
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
+
+	"github.com/jmoiron/sqlx"
+	"github.com/lib/pq"
+	log "github.com/sirupsen/logrus"
+
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/config"
+)
+
+// CIDRetriever satisfies the CIDRetriever interface for ethereum
+type CIDRetriever struct {
+	db *postgres.DB
+}
+
+// NewCIDRetriever returns a pointer to a new CIDRetriever which supports the CIDRetriever interface
+func NewCIDRetriever(db *postgres.DB) *CIDRetriever {
+	return &CIDRetriever{
+		db: db,
+	}
+}
+
+// RetrieveFirstBlockNumber is used to retrieve the first block number in the db
+func (ecr *CIDRetriever) RetrieveFirstBlockNumber() (int64, error) {
+	var blockNumber int64
+	err := ecr.db.Get(&blockNumber, "SELECT block_number FROM header_cids ORDER BY block_number ASC LIMIT 1")
+	return blockNumber, err
+}
+
+// RetrieveLastBlockNumber is used to retrieve the latest block number in the db
+func (ecr *CIDRetriever) RetrieveLastBlockNumber() (int64, error) {
+	var blockNumber int64
+	err := ecr.db.Get(&blockNumber, "SELECT block_number FROM header_cids ORDER BY block_number DESC LIMIT 1 ")
+	return blockNumber, err
+}
+
+// Retrieve is used to retrieve all of the CIDs which conform to the passed StreamFilters
+func (ecr *CIDRetriever) Retrieve(filter interface{}, blockNumber int64) (interface{}, bool, error) {
+	streamFilter, ok := filter.(*config.EthSubscription)
+	if !ok {
+		return nil, true, fmt.Errorf("eth retriever expected filter type %T got %T", &config.EthSubscription{}, filter)
+	}
+	log.Debug("retrieving cids")
+	tx, err := ecr.db.Beginx()
+	if err != nil {
+		return nil, true, err
+	}
+
+	cw := new(CIDWrapper)
+	cw.BlockNumber = big.NewInt(blockNumber)
+	// Retrieve cached header CIDs
+	if !streamFilter.HeaderFilter.Off {
+		cw.Headers, err = ecr.RetrieveHeaderCIDs(tx, blockNumber)
+		if err != nil {
+			if err := tx.Rollback(); err != nil {
+				log.Error(err)
+			}
+			log.Error("header cid retrieval error")
+			return nil, true, err
+		}
+		if streamFilter.HeaderFilter.Uncles {
+			cw.Uncles, err = ecr.RetrieveUncleCIDs(tx, blockNumber)
+			if err != nil {
+				if err := tx.Rollback(); err != nil {
+					log.Error(err)
+				}
+				log.Error("uncle cid retrieval error")
+				return nil, true, err
+			}
+		}
+	}
+	// Retrieve cached trx CIDs
+	if !streamFilter.TxFilter.Off {
+		cw.Transactions, err = ecr.RetrieveTrxCIDs(tx, streamFilter.TxFilter, blockNumber)
+		if err != nil {
+			if err := tx.Rollback(); err != nil {
+				log.Error(err)
+			}
+			log.Error("transaction cid retrieval error")
+			return nil, true, err
+		}
+	}
+	trxIds := make([]int64, 0, len(cw.Transactions))
+	for _, tx := range cw.Transactions {
+		trxIds = append(trxIds, tx.ID)
+	}
+	// Retrieve cached receipt CIDs
+	if !streamFilter.ReceiptFilter.Off {
+		cw.Receipts, err = ecr.RetrieveRctCIDs(tx, streamFilter.ReceiptFilter, blockNumber, trxIds)
+		if err != nil {
+			if err := tx.Rollback(); err != nil {
+				log.Error(err)
+			}
+			log.Error("receipt cid retrieval error")
+			return nil, true, err
+		}
+	}
+	// Retrieve cached state CIDs
+	if !streamFilter.StateFilter.Off {
+		cw.StateNodes, err = ecr.RetrieveStateCIDs(tx, streamFilter.StateFilter, blockNumber)
+		if err != nil {
+			if err := tx.Rollback(); err != nil {
+				log.Error(err)
+			}
+			log.Error("state cid retrieval error")
+			return nil, true, err
+		}
+	}
+	// Retrieve cached storage CIDs
+	if !streamFilter.StorageFilter.Off {
+		cw.StorageNodes, err = ecr.RetrieveStorageCIDs(tx, streamFilter.StorageFilter, blockNumber)
+		if err != nil {
+			if err := tx.Rollback(); err != nil {
+				log.Error(err)
+			}
+			log.Error("storage cid retrieval error")
+			return nil, true, err
+		}
+	}
+	return cw, empty(cw), tx.Commit()
+}
+
+func empty(cidWrapper *CIDWrapper) bool {
+	if len(cidWrapper.Transactions) > 0 || len(cidWrapper.Headers) > 0 || len(cidWrapper.Uncles) > 0 || len(cidWrapper.Receipts) > 0 || len(cidWrapper.StateNodes) > 0 || len(cidWrapper.StorageNodes) > 0 {
+		return false
+	}
+	return true
+}
+
+// RetrieveHeaderCIDs retrieves and returns all of the header cids at the provided blockheight
+func (ecr *CIDRetriever) RetrieveHeaderCIDs(tx *sqlx.Tx, blockNumber int64) ([]HeaderModel, error) {
+	log.Debug("retrieving header cids for block ", blockNumber)
+	headers := make([]HeaderModel, 0)
+	pgStr := `SELECT * FROM header_cids
+				WHERE block_number = $1 AND uncle IS FALSE`
+	err := tx.Select(&headers, pgStr, blockNumber)
+	return headers, err
+}
+
+// RetrieveUncleCIDs retrieves and returns all of the uncle cids at the provided blockheight
+func (ecr *CIDRetriever) RetrieveUncleCIDs(tx *sqlx.Tx, blockNumber int64) ([]HeaderModel, error) {
+	log.Debug("retrieving header cids for block ", blockNumber)
+	headers := make([]HeaderModel, 0)
+	pgStr := `SELECT * FROM header_cids
+				WHERE block_number = $1 AND uncle IS TRUE`
+	err := tx.Select(&headers, pgStr, blockNumber)
+	return headers, err
+}
+
+// RetrieveTrxCIDs retrieves and returns all of the trx cids at the provided blockheight that conform to the provided filter parameters
+// also returns the ids for the returned transaction cids
+func (ecr *CIDRetriever) RetrieveTrxCIDs(tx *sqlx.Tx, txFilter config.TxFilter, blockNumber int64) ([]TxModel, error) {
+	log.Debug("retrieving transaction cids for block ", blockNumber)
+	args := make([]interface{}, 0, 3)
+	results := make([]TxModel, 0)
+	pgStr := `SELECT transaction_cids.id, transaction_cids.header_id,
+ 			transaction_cids.tx_hash, transaction_cids.cid,
+ 			transaction_cids.dst, transaction_cids.src
+ 			FROM transaction_cids INNER JOIN header_cids ON (transaction_cids.header_id = header_cids.id)
+			WHERE header_cids.block_number = $1`
+	args = append(args, blockNumber)
+	if len(txFilter.Dst) > 0 {
+		pgStr += ` AND transaction_cids.dst = ANY($2::VARCHAR(66)[])`
+		args = append(args, pq.Array(txFilter.Dst))
+	}
+	if len(txFilter.Src) > 0 {
+		pgStr += ` AND transaction_cids.src = ANY($3::VARCHAR(66)[])`
+		args = append(args, pq.Array(txFilter.Src))
+	}
+	err := tx.Select(&results, pgStr, args...)
+	if err != nil {
+		return nil, err
+	}
+	return results, nil
+}
+
+// RetrieveRctCIDs retrieves and returns all of the rct cids at the provided blockheight that conform to the provided
+// filter parameters and correspond to the provided tx ids
+func (ecr *CIDRetriever) RetrieveRctCIDs(tx *sqlx.Tx, rctFilter config.ReceiptFilter, blockNumber int64, trxIds []int64) ([]ReceiptModel, error) {
+	log.Debug("retrieving receipt cids for block ", blockNumber)
+	args := make([]interface{}, 0, 4)
+	pgStr := `SELECT receipt_cids.id, receipt_cids.tx_id, receipt_cids.cid,
+ 			receipt_cids.contract, receipt_cids.topic0s
+ 			FROM receipt_cids, transaction_cids, header_cids
+			WHERE receipt_cids.tx_id = transaction_cids.id 
+			AND transaction_cids.header_id = header_cids.id
+			AND header_cids.block_number = $1`
+	args = append(args, blockNumber)
+	if len(rctFilter.Topic0s) > 0 {
+		pgStr += ` AND ((receipt_cids.topic0s && $2::VARCHAR(66)[]`
+		args = append(args, pq.Array(rctFilter.Topic0s))
+		if len(rctFilter.Contracts) > 0 {
+			pgStr += ` AND receipt_cids.contract = ANY($3::VARCHAR(66)[]))`
+			args = append(args, pq.Array(rctFilter.Contracts))
+			if rctFilter.MatchTxs && len(trxIds) > 0 {
+				pgStr += ` OR receipt_cids.tx_id = ANY($4::INTEGER[]))`
+				args = append(args, pq.Array(trxIds))
+			} else {
+				pgStr += `)`
+			}
+		} else {
+			pgStr += `)`
+			if rctFilter.MatchTxs && len(trxIds) > 0 {
+				pgStr += ` OR receipt_cids.tx_id = ANY($3::INTEGER[]))`
+				args = append(args, pq.Array(trxIds))
+			} else {
+				pgStr += `)`
+			}
+		}
+	} else {
+		if len(rctFilter.Contracts) > 0 {
+			pgStr += ` AND (receipt_cids.contract = ANY($2::VARCHAR(66)[])`
+			args = append(args, pq.Array(rctFilter.Contracts))
+			if rctFilter.MatchTxs && len(trxIds) > 0 {
+				pgStr += ` OR receipt_cids.tx_id = ANY($3::INTEGER[]))`
+				args = append(args, pq.Array(trxIds))
+			} else {
+				pgStr += `)`
+			}
+		} else if rctFilter.MatchTxs && len(trxIds) > 0 {
+			pgStr += ` AND receipt_cids.tx_id = ANY($2::INTEGER[])`
+			args = append(args, pq.Array(trxIds))
+		}
+	}
+	receiptCids := make([]ReceiptModel, 0)
+	err := tx.Select(&receiptCids, pgStr, args...)
+	if err != nil {
+		println(pgStr)
+		println("FUCK YOU\r\n\r\n\r\n")
+	}
+	return receiptCids, err
+}
+
+// RetrieveStateCIDs retrieves and returns all of the state node cids at the provided blockheight that conform to the provided filter parameters
+func (ecr *CIDRetriever) RetrieveStateCIDs(tx *sqlx.Tx, stateFilter config.StateFilter, blockNumber int64) ([]StateNodeModel, error) {
+	log.Debug("retrieving state cids for block ", blockNumber)
+	args := make([]interface{}, 0, 2)
+	pgStr := `SELECT state_cids.id, state_cids.header_id,
+			state_cids.state_key, state_cids.leaf, state_cids.cid
+			FROM state_cids INNER JOIN header_cids ON (state_cids.header_id = header_cids.id)
+			WHERE header_cids.block_number = $1`
+	args = append(args, blockNumber)
+	addrLen := len(stateFilter.Addresses)
+	if addrLen > 0 {
+		keys := make([]string, 0, addrLen)
+		for _, addr := range stateFilter.Addresses {
+			keys = append(keys, HexToKey(addr).Hex())
+		}
+		pgStr += ` AND state_cids.state_key = ANY($2::VARCHAR(66)[])`
+		args = append(args, pq.Array(keys))
+	}
+	if !stateFilter.IntermediateNodes {
+		pgStr += ` AND state_cids.leaf = TRUE`
+	}
+	stateNodeCIDs := make([]StateNodeModel, 0)
+	err := tx.Select(&stateNodeCIDs, pgStr, args...)
+	return stateNodeCIDs, err
+}
+
+// RetrieveStorageCIDs retrieves and returns all of the storage node cids at the provided blockheight that conform to the provided filter parameters
+func (ecr *CIDRetriever) RetrieveStorageCIDs(tx *sqlx.Tx, storageFilter config.StorageFilter, blockNumber int64) ([]StorageNodeWithStateKeyModel, error) {
+	log.Debug("retrieving storage cids for block ", blockNumber)
+	args := make([]interface{}, 0, 3)
+	pgStr := `SELECT storage_cids.id, storage_cids.state_id, storage_cids.storage_key,
+ 			storage_cids.leaf, storage_cids.cid, state_cids.state_key FROM storage_cids, state_cids, header_cids
+			WHERE storage_cids.state_id = state_cids.id 
+			AND state_cids.header_id = header_cids.id
+			AND header_cids.block_number = $1`
+	args = append(args, blockNumber)
+	addrLen := len(storageFilter.Addresses)
+	if addrLen > 0 {
+		keys := make([]string, 0, addrLen)
+		for _, addr := range storageFilter.Addresses {
+			keys = append(keys, HexToKey(addr).Hex())
+		}
+		pgStr += ` AND state_cids.state_key = ANY($2::VARCHAR(66)[])`
+		args = append(args, pq.Array(keys))
+		if len(storageFilter.StorageKeys) > 0 {
+			pgStr += ` AND storage_cids.storage_key = ANY($3::VARCHAR(66)[])`
+			args = append(args, pq.Array(storageFilter.StorageKeys))
+		}
+	} else if len(storageFilter.StorageKeys) > 0 {
+		pgStr += ` AND storage_cids.storage_key = ANY($2::VARCHAR(66)[])`
+		args = append(args, pq.Array(storageFilter.StorageKeys))
+	}
+	if !storageFilter.IntermediateNodes {
+		pgStr += ` AND storage_cids.leaf = TRUE`
+	}
+	storageNodeCIDs := make([]StorageNodeWithStateKeyModel, 0)
+	err := tx.Select(&storageNodeCIDs, pgStr, args...)
+	return storageNodeCIDs, err
+}
+
+// RetrieveGapsInData is used to find the the block numbers at which we are missing data in the db
+func (ecr *CIDRetriever) RetrieveGapsInData() ([]shared.Gap, error) {
+	pgStr := `SELECT header_cids.block_number + 1 AS start, min(fr.block_number) - 1 AS stop FROM header_cids
+				LEFT JOIN header_cids r on header_cids.block_number = r.block_number - 1
+				LEFT JOIN header_cids fr on header_cids.block_number < fr.block_number
+				WHERE r.block_number is NULL and fr.block_number IS NOT NULL
+				GROUP BY header_cids.block_number, r.block_number`
+	results := make([]struct {
+		Start uint64 `db:"start"`
+		Stop  uint64 `db:"stop"`
+	}, 0)
+	err := ecr.db.Select(&results, pgStr)
+	if err != nil {
+		return nil, err
+	}
+	gaps := make([]shared.Gap, len(results))
+	for i, res := range results {
+		gaps[i] = shared.Gap{
+			Start: res.Start,
+			Stop:  res.Stop,
+		}
+	}
+	return gaps, nil
+}
+
+func (ecr *CIDRetriever) Database() *postgres.DB {
+	return ecr.db
+}

pkg/super_node/eth/retriever_test.go

@@ -0,0 +1,506 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package eth_test
+
+import (
+	"math/big"
+
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/config"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
+	eth2 "github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth/mocks"
+)
+
+var (
+	openFilter = &config.EthSubscription{
+		Start:         big.NewInt(0),
+		End:           big.NewInt(1),
+		HeaderFilter:  config.HeaderFilter{},
+		TxFilter:      config.TxFilter{},
+		ReceiptFilter: config.ReceiptFilter{},
+		StateFilter:   config.StateFilter{},
+		StorageFilter: config.StorageFilter{},
+	}
+	rctContractFilter = &config.EthSubscription{
+		Start: big.NewInt(0),
+		End:   big.NewInt(1),
+		HeaderFilter: config.HeaderFilter{
+			Off: true,
+		},
+		TxFilter: config.TxFilter{
+			Off: true,
+		},
+		ReceiptFilter: config.ReceiptFilter{
+			Contracts: []string{"0x0000000000000000000000000000000000000001"},
+		},
+		StateFilter: config.StateFilter{
+			Off: true,
+		},
+		StorageFilter: config.StorageFilter{
+			Off: true,
+		},
+	}
+	rctTopicsFilter = &config.EthSubscription{
+		Start: big.NewInt(0),
+		End:   big.NewInt(1),
+		HeaderFilter: config.HeaderFilter{
+			Off: true,
+		},
+		TxFilter: config.TxFilter{
+			Off: true,
+		},
+		ReceiptFilter: config.ReceiptFilter{
+			Topic0s: []string{"0x0000000000000000000000000000000000000000000000000000000000000004"},
+		},
+		StateFilter: config.StateFilter{
+			Off: true,
+		},
+		StorageFilter: config.StorageFilter{
+			Off: true,
+		},
+	}
+	rctTopicsAndContractFilter = &config.EthSubscription{
+		Start: big.NewInt(0),
+		End:   big.NewInt(1),
+		HeaderFilter: config.HeaderFilter{
+			Off: true,
+		},
+		TxFilter: config.TxFilter{
+			Off: true,
+		},
+		ReceiptFilter: config.ReceiptFilter{
+			Topic0s:   []string{"0x0000000000000000000000000000000000000000000000000000000000000004", "0x0000000000000000000000000000000000000000000000000000000000000005"},
+			Contracts: []string{"0x0000000000000000000000000000000000000000"},
+		},
+		StateFilter: config.StateFilter{
+			Off: true,
+		},
+		StorageFilter: config.StorageFilter{
+			Off: true,
+		},
+	}
+	rctContractsAndTopicFilter = &config.EthSubscription{
+		Start: big.NewInt(0),
+		End:   big.NewInt(1),
+		HeaderFilter: config.HeaderFilter{
+			Off: true,
+		},
+		TxFilter: config.TxFilter{
+			Off: true,
+		},
+		ReceiptFilter: config.ReceiptFilter{
+			Topic0s:   []string{"0x0000000000000000000000000000000000000000000000000000000000000005"},
+			Contracts: []string{"0x0000000000000000000000000000000000000000", "0x0000000000000000000000000000000000000001"},
+		},
+		StateFilter: config.StateFilter{
+			Off: true,
+		},
+		StorageFilter: config.StorageFilter{
+			Off: true,
+		},
+	}
+	rctsForAllCollectedTrxs = &config.EthSubscription{
+		Start: big.NewInt(0),
+		End:   big.NewInt(1),
+		HeaderFilter: config.HeaderFilter{
+			Off: true,
+		},
+		TxFilter: config.TxFilter{}, // Trx filter open so we will collect all trxs, therefore we will also collect all corresponding rcts despite rct filter
+		ReceiptFilter: config.ReceiptFilter{
+			MatchTxs:  true,
+			Topic0s:   []string{"0x0000000000000000000000000000000000000000000000000000000000000006"}, // Topic isn't one of the topics we have
+			Contracts: []string{"0x0000000000000000000000000000000000000002"},                         // Contract isn't one of the contracts we have
+		},
+		StateFilter: config.StateFilter{
+			Off: true,
+		},
+		StorageFilter: config.StorageFilter{
+			Off: true,
+		},
+	}
+	rctsForSelectCollectedTrxs = &config.EthSubscription{
+		Start: big.NewInt(0),
+		End:   big.NewInt(1),
+		HeaderFilter: config.HeaderFilter{
+			Off: true,
+		},
+		TxFilter: config.TxFilter{
+			Dst: []string{"0x0000000000000000000000000000000000000001"}, // We only filter for one of the trxs so we will only get the one corresponding receipt
+		},
+		ReceiptFilter: config.ReceiptFilter{
+			MatchTxs:  true,
+			Topic0s:   []string{"0x0000000000000000000000000000000000000000000000000000000000000006"}, // Topic isn't one of the topics we have
+			Contracts: []string{"0x0000000000000000000000000000000000000002"},                         // Contract isn't one of the contracts we have
+		},
+		StateFilter: config.StateFilter{
+			Off: true,
+		},
+		StorageFilter: config.StorageFilter{
+			Off: true,
+		},
+	}
+	stateFilter = &config.EthSubscription{
+		Start: big.NewInt(0),
+		End:   big.NewInt(1),
+		HeaderFilter: config.HeaderFilter{
+			Off: true,
+		},
+		TxFilter: config.TxFilter{
+			Off: true,
+		},
+		ReceiptFilter: config.ReceiptFilter{
+			Off: true,
+		},
+		StateFilter: config.StateFilter{
+			Addresses: []string{mocks.Address.Hex()},
+		},
+		StorageFilter: config.StorageFilter{
+			Off: true,
+		},
+	}
+)
+
+var _ = Describe("Retriever", func() {
+	var (
+		db        *postgres.DB
+		repo      *eth2.CIDIndexer
+		retriever *eth2.CIDRetriever
+	)
+	BeforeEach(func() {
+		var err error
+		db, err = eth.SetupDB()
+		Expect(err).ToNot(HaveOccurred())
+		repo = eth2.NewCIDIndexer(db)
+		retriever = eth2.NewCIDRetriever(db)
+	})
+	AfterEach(func() {
+		eth.TearDownDB(db)
+	})
+
+	Describe("Retrieve", func() {
+		BeforeEach(func() {
+			err := repo.Index(mocks.MockCIDPayload)
+			Expect(err).ToNot(HaveOccurred())
+		})
+		It("Retrieves all CIDs for the given blocknumber when provided an open filter", func() {
+			cids, empty, err := retriever.Retrieve(openFilter, 1)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(empty).ToNot(BeTrue())
+			cidWrapper, ok := cids.(*eth.CIDWrapper)
+			Expect(ok).To(BeTrue())
+			Expect(cidWrapper.BlockNumber).To(Equal(mocks.MockCIDWrapper.BlockNumber))
+			Expect(len(cidWrapper.Headers)).To(Equal(1))
+			expectedHeaderCIDs := mocks.MockCIDWrapper.Headers
+			expectedHeaderCIDs[0].ID = cidWrapper.Headers[0].ID
+			Expect(cidWrapper.Headers).To(Equal(expectedHeaderCIDs))
+			Expect(len(cidWrapper.Transactions)).To(Equal(2))
+			Expect(eth.TxModelsContainsCID(cidWrapper.Transactions, mocks.MockCIDWrapper.Transactions[0].CID)).To(BeTrue())
+			Expect(eth.TxModelsContainsCID(cidWrapper.Transactions, mocks.MockCIDWrapper.Transactions[1].CID)).To(BeTrue())
+			Expect(len(cidWrapper.Receipts)).To(Equal(2))
+			Expect(eth.ReceiptModelsContainsCID(cidWrapper.Receipts, mocks.MockCIDWrapper.Receipts[0].CID)).To(BeTrue())
+			Expect(eth.ReceiptModelsContainsCID(cidWrapper.Receipts, mocks.MockCIDWrapper.Receipts[1].CID)).To(BeTrue())
+			Expect(len(cidWrapper.StateNodes)).To(Equal(2))
+			for _, stateNode := range cidWrapper.StateNodes {
+				if stateNode.CID == "mockStateCID1" {
+					Expect(stateNode.StateKey).To(Equal(mocks.ContractLeafKey.Hex()))
+					Expect(stateNode.Leaf).To(Equal(true))
+				}
+				if stateNode.CID == "mockStateCID2" {
+					Expect(stateNode.StateKey).To(Equal(mocks.AnotherContractLeafKey.Hex()))
+					Expect(stateNode.Leaf).To(Equal(true))
+				}
+			}
+			Expect(len(cidWrapper.StorageNodes)).To(Equal(1))
+			expectedStorageNodeCIDs := mocks.MockCIDWrapper.StorageNodes
+			expectedStorageNodeCIDs[0].ID = cidWrapper.StorageNodes[0].ID
+			expectedStorageNodeCIDs[0].StateID = cidWrapper.StorageNodes[0].StateID
+			Expect(cidWrapper.StorageNodes).To(Equal(expectedStorageNodeCIDs))
+		})
+
+		It("Applies filters from the provided config.Subscription", func() {
+			cids1, empty, err := retriever.Retrieve(rctContractFilter, 1)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(empty).ToNot(BeTrue())
+			cidWrapper1, ok := cids1.(*eth.CIDWrapper)
+			Expect(ok).To(BeTrue())
+			Expect(cidWrapper1.BlockNumber).To(Equal(mocks.MockCIDWrapper.BlockNumber))
+			Expect(len(cidWrapper1.Headers)).To(Equal(0))
+			Expect(len(cidWrapper1.Transactions)).To(Equal(0))
+			Expect(len(cidWrapper1.StateNodes)).To(Equal(0))
+			Expect(len(cidWrapper1.StorageNodes)).To(Equal(0))
+			Expect(len(cidWrapper1.Receipts)).To(Equal(1))
+			expectedReceiptCID := mocks.MockCIDWrapper.Receipts[1]
+			expectedReceiptCID.ID = cidWrapper1.Receipts[0].ID
+			expectedReceiptCID.TxID = cidWrapper1.Receipts[0].TxID
+			Expect(cidWrapper1.Receipts[0]).To(Equal(expectedReceiptCID))
+
+			cids2, empty, err := retriever.Retrieve(rctTopicsFilter, 1)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(empty).ToNot(BeTrue())
+			cidWrapper2, ok := cids2.(*eth.CIDWrapper)
+			Expect(ok).To(BeTrue())
+			Expect(cidWrapper2.BlockNumber).To(Equal(mocks.MockCIDWrapper.BlockNumber))
+			Expect(len(cidWrapper2.Headers)).To(Equal(0))
+			Expect(len(cidWrapper2.Transactions)).To(Equal(0))
+			Expect(len(cidWrapper2.StateNodes)).To(Equal(0))
+			Expect(len(cidWrapper2.StorageNodes)).To(Equal(0))
+			Expect(len(cidWrapper2.Receipts)).To(Equal(1))
+			expectedReceiptCID = mocks.MockCIDWrapper.Receipts[0]
+			expectedReceiptCID.ID = cidWrapper2.Receipts[0].ID
+			expectedReceiptCID.TxID = cidWrapper2.Receipts[0].TxID
+			Expect(cidWrapper2.Receipts[0]).To(Equal(expectedReceiptCID))
+
+			cids3, empty, err := retriever.Retrieve(rctTopicsAndContractFilter, 1)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(empty).ToNot(BeTrue())
+			cidWrapper3, ok := cids3.(*eth.CIDWrapper)
+			Expect(ok).To(BeTrue())
+			Expect(cidWrapper3.BlockNumber).To(Equal(mocks.MockCIDWrapper.BlockNumber))
+			Expect(len(cidWrapper3.Headers)).To(Equal(0))
+			Expect(len(cidWrapper3.Transactions)).To(Equal(0))
+			Expect(len(cidWrapper3.StateNodes)).To(Equal(0))
+			Expect(len(cidWrapper3.StorageNodes)).To(Equal(0))
+			Expect(len(cidWrapper3.Receipts)).To(Equal(1))
+			expectedReceiptCID = mocks.MockCIDWrapper.Receipts[0]
+			expectedReceiptCID.ID = cidWrapper3.Receipts[0].ID
+			expectedReceiptCID.TxID = cidWrapper3.Receipts[0].TxID
+			Expect(cidWrapper3.Receipts[0]).To(Equal(expectedReceiptCID))
+
+			cids4, empty, err := retriever.Retrieve(rctContractsAndTopicFilter, 1)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(empty).ToNot(BeTrue())
+			cidWrapper4, ok := cids4.(*eth.CIDWrapper)
+			Expect(ok).To(BeTrue())
+			Expect(cidWrapper4.BlockNumber).To(Equal(mocks.MockCIDWrapper.BlockNumber))
+			Expect(len(cidWrapper4.Headers)).To(Equal(0))
+			Expect(len(cidWrapper4.Transactions)).To(Equal(0))
+			Expect(len(cidWrapper4.StateNodes)).To(Equal(0))
+			Expect(len(cidWrapper4.StorageNodes)).To(Equal(0))
+			Expect(len(cidWrapper4.Receipts)).To(Equal(1))
+			expectedReceiptCID = mocks.MockCIDWrapper.Receipts[1]
+			expectedReceiptCID.ID = cidWrapper4.Receipts[0].ID
+			expectedReceiptCID.TxID = cidWrapper4.Receipts[0].TxID
+			Expect(cidWrapper4.Receipts[0]).To(Equal(expectedReceiptCID))
+
+			cids5, empty, err := retriever.Retrieve(rctsForAllCollectedTrxs, 1)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(empty).ToNot(BeTrue())
+			cidWrapper5, ok := cids5.(*eth.CIDWrapper)
+			Expect(ok).To(BeTrue())
+			Expect(cidWrapper5.BlockNumber).To(Equal(mocks.MockCIDWrapper.BlockNumber))
+			Expect(len(cidWrapper5.Headers)).To(Equal(0))
+			Expect(len(cidWrapper5.Transactions)).To(Equal(2))
+			Expect(eth.TxModelsContainsCID(cidWrapper5.Transactions, "mockTrxCID1")).To(BeTrue())
+			Expect(eth.TxModelsContainsCID(cidWrapper5.Transactions, "mockTrxCID2")).To(BeTrue())
+			Expect(len(cidWrapper5.StateNodes)).To(Equal(0))
+			Expect(len(cidWrapper5.StorageNodes)).To(Equal(0))
+			Expect(len(cidWrapper5.Receipts)).To(Equal(2))
+			Expect(eth.ReceiptModelsContainsCID(cidWrapper5.Receipts, "mockRctCID1")).To(BeTrue())
+			Expect(eth.ReceiptModelsContainsCID(cidWrapper5.Receipts, "mockRctCID2")).To(BeTrue())
+
+			cids6, empty, err := retriever.Retrieve(rctsForSelectCollectedTrxs, 1)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(empty).ToNot(BeTrue())
+			cidWrapper6, ok := cids6.(*eth.CIDWrapper)
+			Expect(ok).To(BeTrue())
+			Expect(cidWrapper6.BlockNumber).To(Equal(mocks.MockCIDWrapper.BlockNumber))
+			Expect(len(cidWrapper6.Headers)).To(Equal(0))
+			Expect(len(cidWrapper6.Transactions)).To(Equal(1))
+			expectedTxCID := mocks.MockCIDWrapper.Transactions[1]
+			expectedTxCID.ID = cidWrapper6.Transactions[0].ID
+			expectedTxCID.HeaderID = cidWrapper6.Transactions[0].HeaderID
+			Expect(cidWrapper6.Transactions[0]).To(Equal(expectedTxCID))
+			Expect(len(cidWrapper6.StateNodes)).To(Equal(0))
+			Expect(len(cidWrapper6.StorageNodes)).To(Equal(0))
+			Expect(len(cidWrapper6.Receipts)).To(Equal(1))
+			expectedReceiptCID = mocks.MockCIDWrapper.Receipts[1]
+			expectedReceiptCID.ID = cidWrapper6.Receipts[0].ID
+			expectedReceiptCID.TxID = cidWrapper6.Receipts[0].TxID
+			Expect(cidWrapper6.Receipts[0]).To(Equal(expectedReceiptCID))
+
+			cids7, empty, err := retriever.Retrieve(stateFilter, 1)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(empty).ToNot(BeTrue())
+			cidWrapper7, ok := cids7.(*eth.CIDWrapper)
+			Expect(ok).To(BeTrue())
+			Expect(cidWrapper7.BlockNumber).To(Equal(mocks.MockCIDWrapper.BlockNumber))
+			Expect(len(cidWrapper7.Headers)).To(Equal(0))
+			Expect(len(cidWrapper7.Transactions)).To(Equal(0))
+			Expect(len(cidWrapper7.Receipts)).To(Equal(0))
+			Expect(len(cidWrapper7.StorageNodes)).To(Equal(0))
+			Expect(len(cidWrapper7.StateNodes)).To(Equal(1))
+			Expect(cidWrapper7.StateNodes[0]).To(Equal(eth.StateNodeModel{
+				ID:       cidWrapper7.StateNodes[0].ID,
+				HeaderID: cidWrapper7.StateNodes[0].HeaderID,
+				Leaf:     true,
+				StateKey: mocks.ContractLeafKey.Hex(),
+				CID:      "mockStateCID1",
+			}))
+		})
+	})
+
+	Describe("RetrieveFirstBlockNumber", func() {
+		It("Gets the number of the first block that has data in the database", func() {
+			err := repo.Index(mocks.MockCIDPayload)
+			Expect(err).ToNot(HaveOccurred())
+			num, err := retriever.RetrieveFirstBlockNumber()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(num).To(Equal(int64(1)))
+		})
+
+		It("Gets the number of the first block that has data in the database", func() {
+			payload := *mocks.MockCIDPayload
+			payload.HeaderCID.BlockNumber = "1010101"
+			err := repo.Index(&payload)
+			Expect(err).ToNot(HaveOccurred())
+			num, err := retriever.RetrieveFirstBlockNumber()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(num).To(Equal(int64(1010101)))
+		})
+
+		It("Gets the number of the first block that has data in the database", func() {
+			payload1 := *mocks.MockCIDPayload
+			payload1.HeaderCID.BlockNumber = "1010101"
+			payload2 := payload1
+			payload2.HeaderCID.BlockNumber = "5"
+			err := repo.Index(&payload1)
+			Expect(err).ToNot(HaveOccurred())
+			err = repo.Index(&payload2)
+			Expect(err).ToNot(HaveOccurred())
+			num, err := retriever.RetrieveFirstBlockNumber()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(num).To(Equal(int64(5)))
+		})
+	})
+
+	Describe("RetrieveLastBlockNumber", func() {
+		It("Gets the number of the latest block that has data in the database", func() {
+			err := repo.Index(mocks.MockCIDPayload)
+			Expect(err).ToNot(HaveOccurred())
+			num, err := retriever.RetrieveLastBlockNumber()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(num).To(Equal(int64(1)))
+		})
+
+		It("Gets the number of the latest block that has data in the database", func() {
+			payload := *mocks.MockCIDPayload
+			payload.HeaderCID.BlockNumber = "1010101"
+			err := repo.Index(&payload)
+			Expect(err).ToNot(HaveOccurred())
+			num, err := retriever.RetrieveLastBlockNumber()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(num).To(Equal(int64(1010101)))
+		})
+
+		It("Gets the number of the latest block that has data in the database", func() {
+			payload1 := *mocks.MockCIDPayload
+			payload1.HeaderCID.BlockNumber = "1010101"
+			payload2 := payload1
+			payload2.HeaderCID.BlockNumber = "5"
+			err := repo.Index(&payload1)
+			Expect(err).ToNot(HaveOccurred())
+			err = repo.Index(&payload2)
+			Expect(err).ToNot(HaveOccurred())
+			num, err := retriever.RetrieveLastBlockNumber()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(num).To(Equal(int64(1010101)))
+		})
+	})
+
+	Describe("RetrieveGapsInData", func() {
+		It("Doesn't return gaps if there are none", func() {
+			payload1 := *mocks.MockCIDPayload
+			payload1.HeaderCID.BlockNumber = "2"
+			payload2 := payload1
+			payload2.HeaderCID.BlockNumber = "3"
+			err := repo.Index(mocks.MockCIDPayload)
+			Expect(err).ToNot(HaveOccurred())
+			err = repo.Index(&payload1)
+			Expect(err).ToNot(HaveOccurred())
+			err = repo.Index(&payload2)
+			Expect(err).ToNot(HaveOccurred())
+			gaps, err := retriever.RetrieveGapsInData()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(gaps)).To(Equal(0))
+		})
+
+		It("Doesn't return the gap from 0 to the earliest block", func() {
+			payload := *mocks.MockCIDPayload
+			payload.HeaderCID.BlockNumber = "5"
+			err := repo.Index(&payload)
+			Expect(err).ToNot(HaveOccurred())
+			gaps, err := retriever.RetrieveGapsInData()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(gaps)).To(Equal(0))
+		})
+
+		It("Finds gap between two entries", func() {
+			payload1 := *mocks.MockCIDPayload
+			payload1.HeaderCID.BlockNumber = "1010101"
+			payload2 := payload1
+			payload2.HeaderCID.BlockNumber = "5"
+			err := repo.Index(&payload1)
+			Expect(err).ToNot(HaveOccurred())
+			err = repo.Index(&payload2)
+			Expect(err).ToNot(HaveOccurred())
+			gaps, err := retriever.RetrieveGapsInData()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(gaps)).To(Equal(1))
+			Expect(gaps[0].Start).To(Equal(uint64(6)))
+			Expect(gaps[0].Stop).To(Equal(uint64(1010100)))
+		})
+
+		It("Finds gaps between multiple entries", func() {
+			payload1 := *mocks.MockCIDPayload
+			payload1.HeaderCID.BlockNumber = "1010101"
+			payload2 := payload1
+			payload2.HeaderCID.BlockNumber = "5"
+			payload3 := payload2
+			payload3.HeaderCID.BlockNumber = "100"
+			payload4 := payload3
+			payload4.HeaderCID.BlockNumber = "101"
+			payload5 := payload4
+			payload5.HeaderCID.BlockNumber = "102"
+			payload6 := payload5
+			payload6.HeaderCID.BlockNumber = "1000"
+			err := repo.Index(&payload1)
+			Expect(err).ToNot(HaveOccurred())
+			err = repo.Index(&payload2)
+			Expect(err).ToNot(HaveOccurred())
+			err = repo.Index(&payload3)
+			Expect(err).ToNot(HaveOccurred())
+			err = repo.Index(&payload4)
+			Expect(err).ToNot(HaveOccurred())
+			err = repo.Index(&payload5)
+			Expect(err).ToNot(HaveOccurred())
+			err = repo.Index(&payload6)
+			Expect(err).ToNot(HaveOccurred())
+			gaps, err := retriever.RetrieveGapsInData()
+			Expect(err).ToNot(HaveOccurred())
+			Expect(len(gaps)).To(Equal(3))
+			Expect(shared.ListContainsGap(gaps, shared.Gap{Start: 6, Stop: 99})).To(BeTrue())
+			Expect(shared.ListContainsGap(gaps, shared.Gap{Start: 103, Stop: 999})).To(BeTrue())
+			Expect(shared.ListContainsGap(gaps, shared.Gap{Start: 1001, Stop: 1010100})).To(BeTrue())
+		})
+	})
+})

pkg/super_node/eth/streamer.go

@@ -0,0 +1,46 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package eth
+
+import (
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/sirupsen/logrus"
+
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+)
+
+const (
+	PayloadChanBufferSize = 20000 // the max eth sub buffer size
+)
+
+// PayloadStreamer satisfies the PayloadStreamer interface for ethereum
+type PayloadStreamer struct {
+	Client core.RPCClient
+}
+
+// NewPayloadStreamer creates a pointer to a new StateDiffStreamer which satisfies the PayloadStreamer interface
+func NewPayloadStreamer(client core.RPCClient) *PayloadStreamer {
+	return &PayloadStreamer{
+		Client: client,
+	}
+}
+
+// Stream is the main loop for subscribing to data from the Geth state diff process
+func (sds *PayloadStreamer) Stream(payloadChan chan interface{}) (*rpc.ClientSubscription, error) {
+	logrus.Info("streaming diffs from geth")
+	return sds.Client.Subscribe("statediff", payloadChan, "stream")
+}

pkg/super_node/eth/streamer_test.go

@@ -0,0 +1,34 @@
+// Copyright 2019 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package eth_test
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
+)
+
+var _ = Describe("StateDiff Streamer", func() {
+	It("subscribes to the geth statediff service", func() {
+		client := &fakes.MockRPCClient{}
+		streamer := eth.NewPayloadStreamer(client)
+		payloadChan := make(chan interface{})
+		_, err := streamer.Stream(payloadChan)
+		Expect(err).NotTo(HaveOccurred())
+		client.AssertSubscribeCalledWith("statediff", payloadChan, []interface{}{"stream"})
+	})
+})

pkg/super_node/eth/test_helpers.go

@@ -0,0 +1,86 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package eth
+
+import (
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/pkg/config"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+)
+
+// SetupDB is use to setup a db for super node tests
+func SetupDB() (*postgres.DB, error) {
+	return postgres.NewDB(config.Database{
+		Hostname: "localhost",
+		Name:     "vulcanize_testing",
+		Port:     5432,
+	}, core.Node{})
+}
+
+// TearDownDB is used to tear down the super node dbs after tests
+func TearDownDB(db *postgres.DB) {
+	tx, err := db.Beginx()
+	Expect(err).NotTo(HaveOccurred())
+
+	_, err = tx.Exec(`DELETE FROM header_cids`)
+	Expect(err).NotTo(HaveOccurred())
+	_, err = tx.Exec(`DELETE FROM transaction_cids`)
+	Expect(err).NotTo(HaveOccurred())
+	_, err = tx.Exec(`DELETE FROM receipt_cids`)
+	Expect(err).NotTo(HaveOccurred())
+	_, err = tx.Exec(`DELETE FROM state_cids`)
+	Expect(err).NotTo(HaveOccurred())
+	_, err = tx.Exec(`DELETE FROM storage_cids`)
+	Expect(err).NotTo(HaveOccurred())
+	_, err = tx.Exec(`DELETE FROM blocks`)
+	Expect(err).NotTo(HaveOccurred())
+
+	err = tx.Commit()
+	Expect(err).NotTo(HaveOccurred())
+}
+
+// TxModelsContainsCID used to check if a list of TxModels contains a specific cid string
+func TxModelsContainsCID(txs []TxModel, cid string) bool {
+	for _, tx := range txs {
+		if tx.CID == cid {
+			return true
+		}
+	}
+	return false
+}
+
+// ListContainsBytes used to check if a list of byte arrays contains a particular byte array
+func ReceiptModelsContainsCID(rcts []ReceiptModel, cid string) bool {
+	for _, rct := range rcts {
+		if rct.CID == cid {
+			return true
+		}
+	}
+	return false
+}
+
+// ListContainsRange used to check if a list of [2]uint64 contains a particula [2]uint64
+func ListContainsRange(rangeList [][2]uint64, rng [2]uint64) bool {
+	for _, rangeInList := range rangeList {
+		if rangeInList == rng {
+			return true
+		}
+	}
+	return false
+}

pkg/super_node/eth/types.go

@@ -0,0 +1,119 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package eth
+
+import (
+	"encoding/json"
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ipfs/go-block-format"
+)
+
+// IPLDPayload is a custom type which packages raw ETH data for publishing to IPFS and filtering to subscribers
+// Returned by PayloadConverter
+// Passed to IPLDPublisher and ResponseFilterer
+type IPLDPayload struct {
+	TotalDifficulty *big.Int
+	Block           *types.Block
+	HeaderRLP       []byte
+	TrxMetaData     []TxModel
+	Receipts        types.Receipts
+	ReceiptMetaData []ReceiptModel
+	StateNodes      []TrieNode
+	StorageNodes    map[common.Hash][]TrieNode
+}
+
+// Trie struct used to flag node as leaf or not
+type TrieNode struct {
+	Key   common.Hash
+	Value []byte
+	Leaf  bool
+}
+
+// CIDPayload is a struct to hold all the CIDs and their associated meta data for indexing in Postgres
+// Returned by IPLDPublisher
+// Passed to CIDIndexer
+type CIDPayload struct {
+	HeaderCID       HeaderModel
+	UncleCIDs       []HeaderModel
+	TransactionCIDs []TxModel
+	ReceiptCIDs     map[common.Hash]ReceiptModel
+	StateNodeCIDs   []StateNodeModel
+	StorageNodeCIDs map[common.Hash][]StorageNodeModel
+}
+
+// CIDWrapper is used to direct fetching of IPLDs from IPFS
+// Returned by CIDRetriever
+// Passed to IPLDFetcher
+type CIDWrapper struct {
+	BlockNumber  *big.Int
+	Headers      []HeaderModel
+	Uncles       []HeaderModel
+	Transactions []TxModel
+	Receipts     []ReceiptModel
+	StateNodes   []StateNodeModel
+	StorageNodes []StorageNodeWithStateKeyModel
+}
+
+// IPLDWrapper is used to package raw IPLD block data fetched from IPFS
+// Returned by IPLDFetcher
+// Passed to IPLDResolver
+type IPLDWrapper struct {
+	BlockNumber  *big.Int
+	Headers      []blocks.Block
+	Uncles       []blocks.Block
+	Transactions []blocks.Block
+	Receipts     []blocks.Block
+	StateNodes   map[common.Hash]blocks.Block
+	StorageNodes map[common.Hash]map[common.Hash]blocks.Block
+}
+
+// StreamPayload holds the data streamed from the super node eth service to the requesting clients
+// Returned by IPLDResolver and ResponseFilterer
+// Passed to client subscriptions
+type StreamPayload struct {
+	BlockNumber     *big.Int                               `json:"blockNumber"`
+	HeadersRlp      [][]byte                               `json:"headersRlp"`
+	UnclesRlp       [][]byte                               `json:"unclesRlp"`
+	TransactionsRlp [][]byte                               `json:"transactionsRlp"`
+	ReceiptsRlp     [][]byte                               `json:"receiptsRlp"`
+	StateNodesRlp   map[common.Hash][]byte                 `json:"stateNodesRlp"`
+	StorageNodesRlp map[common.Hash]map[common.Hash][]byte `json:"storageNodesRlp"`
+
+	encoded []byte
+	err     error
+}
+
+func (sd *StreamPayload) ensureEncoded() {
+	if sd.encoded == nil && sd.err == nil {
+		sd.encoded, sd.err = json.Marshal(sd)
+	}
+}
+
+// Length to implement Encoder interface for StateDiff
+func (sd *StreamPayload) Length() int {
+	sd.ensureEncoded()
+	return len(sd.encoded)
+}
+
+// Encode to implement Encoder interface for StateDiff
+func (sd *StreamPayload) Encode() ([]byte, error) {
+	sd.ensureEncoded()
+	return sd.encoded, sd.err
+}