major refactoring of super_node to make it easier to support other chains

cmd/root.go

@@ -34,13 +34,14 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/eth/client"
 	vRpc "github.com/vulcanize/vulcanizedb/pkg/eth/converters/rpc"
 	"github.com/vulcanize/vulcanizedb/pkg/eth/node"
+	config2 "github.com/vulcanize/vulcanizedb/pkg/super_node/config"
 )
 
 var (
 	cfgFile              string
 	databaseConfig       config.Database
 	genConfig            config.Plugin
-	subscriptionConfig   config.Subscription
+	subscriptionConfig   *config2.EthSubscription
 	ipc                  string
 	levelDbPath          string
 	queueRecheckInterval time.Duration

cmd/screenAndServe.go

@@ -26,10 +26,10 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
-	"github.com/vulcanize/vulcanizedb/pkg/config"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
 	"github.com/vulcanize/vulcanizedb/pkg/super_node"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/config"
 	"github.com/vulcanize/vulcanizedb/utils"
 )
 
@@ -52,18 +52,17 @@ func init() {
 }
 
 func screenAndServe() {
-	superNode, newNodeErr := newSuperNodeWithoutPairedGethNode()
+	superNode, err := newSuperNodeWithoutPairedGethNode()
-	if newNodeErr != nil {
+	if err != nil {
-		logWithCommand.Fatal(newNodeErr)
+		logWithCommand.Fatal(err)
 	}
 	wg := &syn.WaitGroup{}
 	quitChan := make(chan bool, 1)
-	emptyPayloadChan := make(chan ipfs.IPLDPayload)
+	emptyPayloadChan := make(chan interface{})
 	superNode.ScreenAndServe(wg, emptyPayloadChan, quitChan)
 
-	serverErr := startServers(superNode)
+	if err := startServers(superNode); err != nil {
-	if serverErr != nil {
+		logWithCommand.Fatal(err)
-		logWithCommand.Fatal(serverErr)
 	}
 	wg.Wait()
 }
@@ -72,15 +71,15 @@ func startServers(superNode super_node.NodeInterface) error {
 	var ipcPath string
 	ipcPath = viper.GetString("server.ipcPath")
 	if ipcPath == "" {
-		home, homeDirErr := os.UserHomeDir()
+		home, err := os.UserHomeDir()
-		if homeDirErr != nil {
+		if err != nil {
-			return homeDirErr
+			return err
 		}
 		ipcPath = filepath.Join(home, ".vulcanize/vulcanize.ipc")
 	}
-	_, _, ipcErr := rpc.StartIPCEndpoint(ipcPath, superNode.APIs())
+	_, _, err := rpc.StartIPCEndpoint(ipcPath, superNode.APIs())
-	if ipcErr != nil {
+	if err != nil {
-		return ipcErr
+		return err
 	}
 
 	var wsEndpoint string
@@ -90,9 +89,9 @@ func startServers(superNode super_node.NodeInterface) error {
 	}
 	var exposeAll = true
 	var wsOrigins []string
-	_, _, wsErr := rpc.StartWSEndpoint(wsEndpoint, superNode.APIs(), []string{"vdb"}, wsOrigins, exposeAll)
+	_, _, err = rpc.StartWSEndpoint(wsEndpoint, superNode.APIs(), []string{"vdb"}, wsOrigins, exposeAll)
-	if wsErr != nil {
+	if err != nil {
-		return wsErr
+		return err
 	}
 	return nil
 }
@@ -100,27 +99,34 @@ func startServers(superNode super_node.NodeInterface) error {
 func newSuperNodeWithoutPairedGethNode() (super_node.NodeInterface, error) {
 	ipfsPath = viper.GetString("client.ipfsPath")
 	if ipfsPath == "" {
-		home, homeDirErr := os.UserHomeDir()
+		home, err := os.UserHomeDir()
-		if homeDirErr != nil {
+		if err != nil {
-			return nil, homeDirErr
+			return nil, err
 		}
 		ipfsPath = filepath.Join(home, ".ipfs")
 	}
-	ipfsInitErr := ipfs.InitIPFSPlugins()
+	if err := ipfs.InitIPFSPlugins(); err != nil {
-	if ipfsInitErr != nil {
+		return nil, err
-		return nil, ipfsInitErr
 	}
-	ipldFetcher, newFetcherErr := ipfs.NewIPLDFetcher(ipfsPath)
+	ipldFetcher, err := super_node.NewIPLDFetcher(config.Ethereum, ipfsPath)
-	if newFetcherErr != nil {
+	if err != nil {
-		return nil, newFetcherErr
+		return nil, err
 	}
 	db := utils.LoadPostgres(databaseConfig, core.Node{})
+	retriever, err := super_node.NewCIDRetriever(config.Ethereum, &db)
+	if err != nil {
+		return nil, err
+	}
+	resolver, err := super_node.NewIPLDResolver(config.Ethereum)
+	if err != nil {
+		return nil, err
+	}
 	return &super_node.Service{
 		IPLDFetcher:       ipldFetcher,
-		Retriever:         super_node.NewCIDRetriever(&db),
+		Retriever:         retriever,
-		Resolver:          ipfs.NewIPLDResolver(),
+		Resolver:          resolver,
 		Subscriptions:     make(map[common.Hash]map[rpc.ID]super_node.Subscription),
-		SubscriptionTypes: make(map[common.Hash]config.Subscription),
+		SubscriptionTypes: make(map[common.Hash]super_node.SubscriptionSettings),
-		GethNode:          core.Node{},
+		NodeInfo:          core.Node{},
 	}, nil
 }

cmd/streamSubscribe.go

@@ -30,9 +30,11 @@ import (
 	"github.com/spf13/viper"
 
 	"github.com/vulcanize/vulcanizedb/libraries/shared/streamer"
-	"github.com/vulcanize/vulcanizedb/pkg/config"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/eth/client"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/config"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
 )
 
 // streamSubscribeCmd represents the streamSubscribe command
@@ -61,10 +63,10 @@ func streamSubscribe() {
 	str := streamer.NewSuperNodeStreamer(rpcClient)
 
 	// Buffered channel for reading subscription payloads
-	payloadChan := make(chan streamer.SuperNodePayload, 20000)
+	payloadChan := make(chan super_node.Payload, 20000)
 
 	// Subscribe to the super node service with the given config/filter parameters
-	sub, err := str.Stream(payloadChan, subscriptionConfig)
+	sub, err := str.StreamETH(payloadChan, subscriptionConfig)
 	if err != nil {
 		logWithCommand.Fatal(err)
 	}
@@ -73,11 +75,16 @@ func streamSubscribe() {
 	for {
 		select {
 		case payload := <-payloadChan:
-			if payload.ErrMsg != "" {
+			if payload.Err != "" {
-				logWithCommand.Error(payload.ErrMsg)
+				logWithCommand.Error(payload.Err)
 				continue
 			}
-			for _, headerRlp := range payload.HeadersRlp {
+			data, ok := payload.Data.(eth.StreamPayload)
+			if !ok {
+				logWithCommand.Warnf("payload data expected type %T got %T", eth.StreamPayload{}, payload.Data)
+				continue
+			}
+			for _, headerRlp := range data.HeadersRlp {
 				var header types.Header
 				err = rlp.Decode(bytes.NewBuffer(headerRlp), &header)
 				if err != nil {
@@ -87,7 +94,7 @@ func streamSubscribe() {
 				fmt.Printf("Header number %d, hash %s\n", header.Number.Int64(), header.Hash().Hex())
 				fmt.Printf("header: %v\n", header)
 			}
-			for _, trxRlp := range payload.TransactionsRlp {
+			for _, trxRlp := range data.TransactionsRlp {
 				var trx types.Transaction
 				buff := bytes.NewBuffer(trxRlp)
 				stream := rlp.NewStream(buff, 0)
@@ -99,7 +106,7 @@ func streamSubscribe() {
 				fmt.Printf("Transaction with hash %s\n", trx.Hash().Hex())
 				fmt.Printf("trx: %v\n", trx)
 			}
-			for _, rctRlp := range payload.ReceiptsRlp {
+			for _, rctRlp := range data.ReceiptsRlp {
 				var rct types.ReceiptForStorage
 				buff := bytes.NewBuffer(rctRlp)
 				stream := rlp.NewStream(buff, 0)
@@ -121,7 +128,7 @@ func streamSubscribe() {
 				}
 			}
 			// This assumes leafs only
-			for key, stateRlp := range payload.StateNodesRlp {
+			for key, stateRlp := range data.StateNodesRlp {
 				var acct state.Account
 				err = rlp.Decode(bytes.NewBuffer(stateRlp), &acct)
 				if err != nil {
@@ -132,7 +139,7 @@ func streamSubscribe() {
 					key.Hex(), acct.Root.Hex(), acct.Balance.Int64())
 				fmt.Printf("state account: %v\n", acct)
 			}
-			for stateKey, mappedRlp := range payload.StorageNodesRlp {
+			for stateKey, mappedRlp := range data.StorageNodesRlp {
 				fmt.Printf("Storage for state key %s ", stateKey.Hex())
 				for storageKey, storageRlp := range mappedRlp {
 					fmt.Printf("with storage key %s\n", storageKey.Hex())
@@ -165,15 +172,15 @@ func streamSubscribe() {
 
 func configureSubscription() {
 	logWithCommand.Info("loading subscription config")
-	subscriptionConfig = config.Subscription{
+	subscriptionConfig = &config.EthSubscription{
 		// Below default to false, which means we do not backfill by default
 		BackFill:     viper.GetBool("subscription.backfill"),
 		BackFillOnly: viper.GetBool("subscription.backfillOnly"),
 
 		// Below default to 0
 		// 0 start means we start at the beginning and 0 end means we continue indefinitely
-		StartingBlock: big.NewInt(viper.GetInt64("subscription.startingBlock")),
+		Start: big.NewInt(viper.GetInt64("subscription.startingBlock")),
-		EndingBlock:   big.NewInt(viper.GetInt64("subscription.endingBlock")),
+		End:   big.NewInt(viper.GetInt64("subscription.endingBlock")),
 
 		// Below default to false, which means we get all headers by default
 		HeaderFilter: config.HeaderFilter{

cmd/syncAndPublish.go

@@ -33,6 +33,7 @@ import (
 	vRpc "github.com/vulcanize/vulcanizedb/pkg/eth/converters/rpc"
 	"github.com/vulcanize/vulcanizedb/pkg/eth/node"
 	"github.com/vulcanize/vulcanizedb/pkg/super_node"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/config"
 	"github.com/vulcanize/vulcanizedb/utils"
 )
 
@@ -107,7 +108,7 @@ func newSuperNode() (super_node.NodeInterface, error) {
 	if workers < 1 {
 		workers = 1
 	}
-	return super_node.NewSuperNode(ipfsPath, &db, rpcClient, quitChan, workers, blockChain.Node())
+	return super_node.NewSuperNode(config.Ethereum, ipfsPath, &db, rpcClient, quitChan, workers, blockChain.Node())
 }
 
 func newBackFiller() (super_node.BackFillInterface, error) {
@@ -120,5 +121,5 @@ func newBackFiller() (super_node.BackFillInterface, error) {
 	} else {
 		frequency = time.Duration(freq)
 	}
-	return super_node.NewBackFillService(ipfsPath, &db, archivalRPCClient, time.Minute*frequency, super_node.DefaultMaxBatchSize)
+	return super_node.NewBackFillService(config.Ethereum, ipfsPath, &db, archivalRPCClient, time.Minute*frequency, super_node.DefaultMaxBatchSize)
 }

cmd/syncPublishScreenAndServe.go

@@ -21,8 +21,6 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
-
-	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
 )
 
 // syncPublishScreenAndServeCmd represents the syncPublishScreenAndServe command
@@ -52,7 +50,7 @@ func syncPublishScreenAndServe() {
 	}
 
 	wg := &syn.WaitGroup{}
-	forwardPayloadChan := make(chan ipfs.IPLDPayload, 20000)
+	forwardPayloadChan := make(chan interface{}, 20000)
 	forwardQuitChan := make(chan bool, 1)
 	syncAndPubErr := superNode.SyncAndPublish(wg, forwardPayloadChan, forwardQuitChan)
 	if syncAndPubErr != nil {

db/migrations/00032_create_header_cids_table.sql

@@ -3,9 +3,10 @@ CREATE TABLE public.header_cids (
   id                    SERIAL PRIMARY KEY,
   block_number          BIGINT NOT NULL,
   block_hash            VARCHAR(66) NOT NULL,
+  parent_hash           VARCHAR(66) NOT NULL,
   cid                   TEXT NOT NULL,
   uncle                 BOOLEAN NOT NULL,
-  td                    BIGINT NOT NULL,
+  td                    BIGINT,
   UNIQUE (block_number, block_hash)
 );
 

db/schema.sql

@@ -313,9 +313,10 @@ CREATE TABLE public.header_cids (
     id integer NOT NULL,
     block_number bigint NOT NULL,
     block_hash character varying(66) NOT NULL,
+    parent_hash character varying(66) NOT NULL,
     cid text NOT NULL,
     uncle boolean NOT NULL,
-    td bigint NOT NULL
+    td bigint
 );
 
 

environments/superNode.toml

@@ -0,0 +1,24 @@
+[superNode]
+    chain = "ethereum"
+    ipfsPath = "/root/.ipfs"
+
+    [superNode.database]
+        name     = "vulcanize_public"
+        hostname = "localhost"
+        port     = 5432
+        user     = "ec2-user"
+
+    [superNode.sync]
+        on = true
+        wsPath  = "ws://127.0.0.1:8546"
+        workers = 1
+
+    [superNode.server]
+        on = true
+        ipcPath = "/root/.vulcanize/vulcanize.ipc"
+        wsPath = "127.0.0.1:8080"
+
+    [superNode.backFill]
+        on = false
+        httpPath = ""
+        frequency = 5

environments/superNodeSubscription.toml

@@ -1,35 +1,36 @@
-[subscription]
+[superNode]
-    path = "ws://127.0.0.1:8080"
+    [superNode.ethSubscription]
-    backfill = true
+        historicalData = true
-    backfillOnly = false
+        historicalDataOnly = false
-    startingBlock = 0
+        startingBlock = 0
-    endingBlock = 0
+        endingBlock = 0
-    [subscription.headerFilter]
+        wsPath = "ws://127.0.0.1:8080"
-        off = false
+        [superNode.ethSubscription.headerFilter]
-        uncles = false
+            off = false
-    [subscription.trxFilter]
+            uncles = false
-        off = false
+        [superNode.ethSubscription.trxFilter]
-        src = [
+            off = false
-            "0xde0B295669a9FD93d5F28D9Ec85E40f4cb697BAe",
+            src = [
-        ]
+                "0xde0B295669a9FD93d5F28D9Ec85E40f4cb697BAe",
-        dst = [
+            ]
-            "0xde0B295669a9FD93d5F28D9Ec85E40f4cb697BAe",
+            dst = [
-        ]
+                "0xde0B295669a9FD93d5F28D9Ec85E40f4cb697BAe",
-    [subscription.receiptFilter]
+            ]
-        off = false
+        [superNode.ethSubscription.receiptFilter]
-        contracts = []
+            off = false
-        topic0s = [
+            contracts = []
-            "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef",
+            topic0s = [
-            "0x930a61a57a70a73c2a503615b87e2e54fe5b9cdeacda518270b852296ab1a377"
+                "0xddf252ad1be2c89b69c2b068fc378daa952ba7f163c4a11628f55a4df523b3ef",
-        ]
+                "0x930a61a57a70a73c2a503615b87e2e54fe5b9cdeacda518270b852296ab1a377"
-    [subscription.stateFilter]
+            ]
-        off = false
+        [superNode.ethSubscription.stateFilter]
-        addresses = [
+            off = false
-           "0xde0B295669a9FD93d5F28D9Ec85E40f4cb697BAe"
+            addresses = [
-       ]
+               "0xde0B295669a9FD93d5F28D9Ec85E40f4cb697BAe"
-       intermediateNodes = false
+           ]
-    [subscription.storageFilter]
+           intermediateNodes = false
-        off = true
+        [superNode.ethSubscription.storageFilter]
-        addresses = []
+            off = true
-        storageKeys = []
+            addresses = []
-        intermediateNodes = false
+            storageKeys = []
+            intermediateNodes = false

environments/syncPublishScreenAndServe.toml

@@ -1,18 +0,0 @@
-[database]
-    name     = "vulcanize_public"
-    hostname = "localhost"
-    port     = 5432
-    user     = "ec2-user"
-
-[client]
-    ipcPath  = "ws://127.0.0.1:8546"
-    ipfsPath = "/root/.ipfs"
-
-[server]
-    ipcPath = "/root/.vulcanize/vulcanize.ipc"
-    wsEndpoint = "127.0.0.1:8080"
-
-[superNodeBackFill]
-    on = false
-    rpcPath = ""
-    frequency = 5

libraries/shared/streamer/statediff_streamer_test.go

@@ -18,6 +18,7 @@ import (
 	"github.com/ethereum/go-ethereum/statediff"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+
 	"github.com/vulcanize/vulcanizedb/libraries/shared/streamer"
 	"github.com/vulcanize/vulcanizedb/pkg/fakes"
 )

libraries/shared/streamer/super_node_streamer.go

@@ -18,19 +18,15 @@
 package streamer
 
 import (
-	"encoding/json"
-	"math/big"
-
-	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node"
 
-	"github.com/vulcanize/vulcanizedb/pkg/config"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 )
 
 // ISuperNodeStreamer is the interface for streaming SuperNodePayloads from a vulcanizeDB super node
 type ISuperNodeStreamer interface {
-	Stream(payloadChan chan SuperNodePayload, streamFilters config.Subscription) (*rpc.ClientSubscription, error)
+	Stream(payloadChan chan super_node.Payload, params super_node.SubscriptionSettings) (*rpc.ClientSubscription, error)
 }
 
 // SuperNodeStreamer is the underlying struct for the ISuperNodeStreamer interface
@@ -46,39 +42,6 @@ func NewSuperNodeStreamer(client core.RPCClient) *SuperNodeStreamer {
 }
 
 // Stream is the main loop for subscribing to data from a vulcanizedb super node
-func (sds *SuperNodeStreamer) Stream(payloadChan chan SuperNodePayload, streamFilters config.Subscription) (*rpc.ClientSubscription, error) {
+func (sds *SuperNodeStreamer) Stream(payloadChan chan super_node.Payload, params super_node.SubscriptionSettings) (*rpc.ClientSubscription, error) {
-	return sds.Client.Subscribe("vdb", payloadChan, "stream", streamFilters)
+	return sds.Client.Subscribe("vdb", payloadChan, "stream", params)
-}
-
-// Payload holds the data returned from the super node to the requesting client
-type SuperNodePayload struct {
-	BlockNumber     *big.Int                               `json:"blockNumber"`
-	HeadersRlp      [][]byte                               `json:"headersRlp"`
-	UnclesRlp       [][]byte                               `json:"unclesRlp"`
-	TransactionsRlp [][]byte                               `json:"transactionsRlp"`
-	ReceiptsRlp     [][]byte                               `json:"receiptsRlp"`
-	StateNodesRlp   map[common.Hash][]byte                 `json:"stateNodesRlp"`
-	StorageNodesRlp map[common.Hash]map[common.Hash][]byte `json:"storageNodesRlp"`
-	ErrMsg          string                                 `json:"errMsg"`
-
-	encoded []byte
-	err     error
-}
-
-func (sd *SuperNodePayload) ensureEncoded() {
-	if sd.encoded == nil && sd.err == nil {
-		sd.encoded, sd.err = json.Marshal(sd)
-	}
-}
-
-// Length to implement Encoder interface for StateDiff
-func (sd *SuperNodePayload) Length() int {
-	sd.ensureEncoded()
-	return len(sd.encoded)
-}
-
-// Encode to implement Encoder interface for StateDiff
-func (sd *SuperNodePayload) Encode() ([]byte, error) {
-	sd.ensureEncoded()
-	return sd.encoded, sd.err
 }

libraries/shared/transformer/super_node_transformer.go

@@ -17,15 +17,15 @@
 package transformer
 
 import (
-	"github.com/vulcanize/vulcanizedb/pkg/config"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node"
 )
 
 type SuperNodeTransformer interface {
 	Init() error
 	Execute() error
-	GetConfig() config.Subscription
+	GetConfig() super_node.SubscriptionSettings
 }
 
-type SuperNodeTransformerInitializer func(db *postgres.DB, subCon config.Subscription, client core.RPCClient) SuperNodeTransformer
+type SuperNodeTransformerInitializer func(db *postgres.DB, subCon super_node.SubscriptionSettings, client core.RPCClient) SuperNodeTransformer

libraries/shared/watcher/contract_watcher.go

@@ -22,6 +22,7 @@ package watcher
 
 import (
 	"fmt"
+
 	"github.com/sirupsen/logrus"
 
 	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"

pkg/config/subscription.go

@@ -1,63 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package config
-
-import "math/big"
-
-// Subscription config is used by a subscribing transformer to specifiy which data to receive from the super node
-type Subscription struct {
-	BackFill      bool
-	BackFillOnly  bool
-	StartingBlock *big.Int
-	EndingBlock   *big.Int // set to 0 or a negative value to have no ending block
-	HeaderFilter  HeaderFilter
-	TrxFilter     TrxFilter
-	ReceiptFilter ReceiptFilter
-	StateFilter   StateFilter
-	StorageFilter StorageFilter
-}
-
-type HeaderFilter struct {
-	Off    bool
-	Uncles bool
-}
-
-type TrxFilter struct {
-	Off bool
-	Src []string
-	Dst []string
-}
-
-type ReceiptFilter struct {
-	Off       bool
-	MatchTxs  bool // turn on to retrieve receipts that pair with retrieved transactions
-	Contracts []string
-	Topic0s   []string
-}
-
-type StateFilter struct {
-	Off               bool
-	Addresses         []string // is converted to state key by taking its keccak256 hash
-	IntermediateNodes bool
-}
-
-type StorageFilter struct {
-	Off               bool
-	Addresses         []string
-	StorageKeys       []string
-	IntermediateNodes bool
-}

pkg/fakes/mock_rpc_client.go

@@ -18,13 +18,11 @@ package fakes
 
 import (
 	"context"
-	"errors"
 	"math/big"
 
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/p2p"
 	"github.com/ethereum/go-ethereum/rpc"
-	"github.com/ethereum/go-ethereum/statediff"
 	. "github.com/onsi/gomega"
 
 	"github.com/vulcanize/vulcanizedb/pkg/core"
@@ -40,7 +38,7 @@ type MockRPCClient struct {
 	passedResult        interface{}
 	passedBatch         []client.BatchElem
 	passedNamespace     string
-	passedPayloadChan   chan statediff.Payload
+	passedPayloadChan   interface{}
 	passedSubscribeArgs []interface{}
 	lengthOfBatch       int
 	returnPOAHeader     core.POAHeader
@@ -51,12 +49,7 @@ type MockRPCClient struct {
 
 func (client *MockRPCClient) Subscribe(namespace string, payloadChan interface{}, args ...interface{}) (*rpc.ClientSubscription, error) {
 	client.passedNamespace = namespace
-
+	client.passedPayloadChan = payloadChan
-	passedPayloadChan, ok := payloadChan.(chan statediff.Payload)
-	if !ok {
-		return nil, errors.New("passed in channel is not of the correct type")
-	}
-	client.passedPayloadChan = passedPayloadChan
 
 	for _, arg := range args {
 		client.passedSubscribeArgs = append(client.passedSubscribeArgs, arg)
@@ -66,7 +59,7 @@ func (client *MockRPCClient) Subscribe(namespace string, payloadChan interface{}
 	return &subscription, nil
 }
 
-func (client *MockRPCClient) AssertSubscribeCalledWith(namespace string, payloadChan chan statediff.Payload, args []interface{}) {
+func (client *MockRPCClient) AssertSubscribeCalledWith(namespace string, payloadChan interface{}, args []interface{}) {
 	Expect(client.passedNamespace).To(Equal(namespace))
 	Expect(client.passedPayloadChan).To(Equal(payloadChan))
 	Expect(client.passedSubscribeArgs).To(Equal(args))

pkg/ipfs/helpers.go

@@ -19,8 +19,6 @@ package ipfs
 import (
 	"context"
 
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ipfs/go-blockservice"
 	"github.com/ipfs/go-ipfs/core"
 	"github.com/ipfs/go-ipfs/plugin/loader"
@@ -58,22 +56,3 @@ func InitIPFSBlockService(ipfsPath string) (blockservice.BlockService, error) {
 	}
 	return ipfsNode.Blocks, nil
 }
-
-// AddressToKey hashes an address
-func AddressToKey(address common.Address) common.Hash {
-	return crypto.Keccak256Hash(address[:])
-}
-
-// HexToKey hashes a hex (0x leading or not) string
-func HexToKey(hex string) common.Hash {
-	addr := common.FromHex(hex)
-	return crypto.Keccak256Hash(addr[:])
-}
-
-// EmptyCIDWrapper returns whether or not the provided CIDWrapper has any Cids we need to process
-func EmptyCIDWrapper(cids CIDWrapper) bool {
-	if len(cids.Transactions) > 0 || len(cids.Headers) > 0 || len(cids.Uncles) > 0 || len(cids.Receipts) > 0 || len(cids.StateNodes) > 0 || len(cids.StorageNodes) > 0 {
-		return false
-	}
-	return true
-}

pkg/ipfs/models.go

@@ -0,0 +1,22 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package ipfs
+
+type IPLDModel struct {
+	Key  string `db:"key"`
+	Data []byte `db:"data"`
+}

pkg/ipfs/types.go

@@ -1,116 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package ipfs
-
-import (
-	"math/big"
-
-	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core/types"
-	"github.com/ipfs/go-block-format"
-)
-
-// CIDWrapper is used to package CIDs retrieved from the local Postgres cache and direct fetching of IPLDs
-type CIDWrapper struct {
-	BlockNumber  *big.Int
-	Headers      []string
-	Uncles       []string
-	Transactions []string
-	Receipts     []string
-	StateNodes   []StateNodeCID
-	StorageNodes []StorageNodeCID
-}
-
-// IPLDWrapper is used to package raw IPLD block data fetched from IPFS
-type IPLDWrapper struct {
-	BlockNumber  *big.Int
-	Headers      []blocks.Block
-	Uncles       []blocks.Block
-	Transactions []blocks.Block
-	Receipts     []blocks.Block
-	StateNodes   map[common.Hash]blocks.Block
-	StorageNodes map[common.Hash]map[common.Hash]blocks.Block
-}
-
-// IPLDPayload is a custom type which packages raw ETH data for the IPFS publisher
-type IPLDPayload struct {
-	HeaderRLP       []byte
-	TotalDifficulty *big.Int
-	BlockNumber     *big.Int
-	BlockHash       common.Hash
-	BlockBody       *types.Body
-	TrxMetaData     []*TrxMetaData
-	Receipts        types.Receipts
-	ReceiptMetaData []*ReceiptMetaData
-	StateNodes      map[common.Hash]StateNode
-	StorageNodes    map[common.Hash][]StorageNode
-}
-
-// StateNode struct used to flag node as leaf or not
-type StateNode struct {
-	Value []byte
-	Leaf  bool
-}
-
-// StorageNode struct used to flag node as leaf or not
-type StorageNode struct {
-	Key   common.Hash
-	Value []byte
-	Leaf  bool
-}
-
-// CIDPayload is a struct to hold all the CIDs and their meta data
-type CIDPayload struct {
-	BlockNumber     string
-	BlockHash       common.Hash
-	TotalDifficulty string
-	HeaderCID       string
-	UncleCIDs       map[common.Hash]string
-	TransactionCIDs map[common.Hash]*TrxMetaData
-	ReceiptCIDs     map[common.Hash]*ReceiptMetaData
-	StateNodeCIDs   map[common.Hash]StateNodeCID
-	StorageNodeCIDs map[common.Hash][]StorageNodeCID
-}
-
-// StateNodeCID is used to associate a leaf flag with a state node cid
-type StateNodeCID struct {
-	CID  string
-	Leaf bool
-	Key  string `db:"state_key"`
-}
-
-// StorageNodeCID is used to associate a leaf flag with a storage node cid
-type StorageNodeCID struct {
-	Key      string `db:"storage_key"`
-	CID      string
-	Leaf     bool
-	StateKey string `db:"state_key"`
-}
-
-// ReceiptMetaData wraps some additional data around our receipt CIDs for indexing
-type ReceiptMetaData struct {
-	CID             string
-	Topic0s         []string
-	ContractAddress string
-}
-
-// TrxMetaData wraps some additional data around our transaction CID for indexing
-type TrxMetaData struct {
-	CID string
-	Src string
-	Dst string
-}

pkg/super_node/api.go

@@ -22,8 +22,6 @@ import (
 	"github.com/ethereum/go-ethereum/rpc"
 	log "github.com/sirupsen/logrus"
 
-	"github.com/vulcanize/vulcanizedb/libraries/shared/streamer"
-	"github.com/vulcanize/vulcanizedb/pkg/config"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 )
 
@@ -35,47 +33,47 @@ const APIVersion = "0.0.1"
 
 // PublicSuperNodeAPI is the public api for the super node
 type PublicSuperNodeAPI struct {
-	sni NodeInterface
+	sn SuperNode
 }
 
 // NewPublicSuperNodeAPI creates a new PublicSuperNodeAPI with the provided underlying SyncPublishScreenAndServe process
-func NewPublicSuperNodeAPI(superNodeInterface NodeInterface) *PublicSuperNodeAPI {
+func NewPublicSuperNodeAPI(superNodeInterface SuperNode) *PublicSuperNodeAPI {
 	return &PublicSuperNodeAPI{
-		sni: superNodeInterface,
+		sn: superNodeInterface,
 	}
 }
 
-// Stream is the public method to setup a subscription that fires off SyncPublishScreenAndServe payloads as they are created
+// Stream is the public method to setup a subscription that fires off super node payloads as they are processed
-func (api *PublicSuperNodeAPI) Stream(ctx context.Context, streamFilters config.Subscription) (*rpc.Subscription, error) {
+func (api *PublicSuperNodeAPI) Stream(ctx context.Context, params SubscriptionSettings) (*rpc.Subscription, error) {
 	// ensure that the RPC connection supports subscriptions
 	notifier, supported := rpc.NotifierFromContext(ctx)
 	if !supported {
 		return nil, rpc.ErrNotificationsUnsupported
 	}
 
-	// create subscription and start waiting for statediff events
+	// create subscription and start waiting for stream events
 	rpcSub := notifier.CreateSubscription()
 
 	go func() {
 		// subscribe to events from the SyncPublishScreenAndServe service
-		payloadChannel := make(chan streamer.SuperNodePayload, payloadChanBufferSize)
+		payloadChannel := make(chan Payload, PayloadChanBufferSize)
 		quitChan := make(chan bool, 1)
-		go api.sni.Subscribe(rpcSub.ID, payloadChannel, quitChan, streamFilters)
+		go api.sn.Subscribe(rpcSub.ID, payloadChannel, quitChan, params)
 
-		// loop and await state diff payloads and relay them to the subscriber with then notifier
+		// loop and await payloads and relay them to the subscriber using notifier
 		for {
 			select {
 			case packet := <-payloadChannel:
-				if notifyErr := notifier.Notify(rpcSub.ID, packet); notifyErr != nil {
+				if err := notifier.Notify(rpcSub.ID, packet); err != nil {
-					log.Error("Failed to send state diff packet", "err", notifyErr)
+					log.Error("Failed to send super node packet", "err", err)
-					api.sni.Unsubscribe(rpcSub.ID)
+					api.sn.Unsubscribe(rpcSub.ID)
 					return
 				}
 			case <-rpcSub.Err():
-				api.sni.Unsubscribe(rpcSub.ID)
+				api.sn.Unsubscribe(rpcSub.ID)
 				return
 			case <-quitChan:
-				// don't need to unsubscribe, SyncPublishScreenAndServe service does so before sending the quit signal
+				// don't need to unsubscribe to super node, the service does so before sending the quit signal this way
 				return
 			}
 		}
@@ -84,7 +82,7 @@ func (api *PublicSuperNodeAPI) Stream(ctx context.Context, streamFilters config.
 	return rpcSub, nil
 }
 
-// Node is a public rpc method to allow transformers to fetch the Geth node info for the super node
+// Node is a public rpc method to allow transformers to fetch the node info for the super node
 func (api *PublicSuperNodeAPI) Node() core.Node {
-	return api.sni.Node()
+	return api.sn.Node()
 }

pkg/super_node/backfiller.go

@@ -22,14 +22,14 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
+
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/config"
+
 	"github.com/ethereum/go-ethereum/params"
 	log "github.com/sirupsen/logrus"
 
-	"github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
-	"github.com/vulcanize/vulcanizedb/pkg/core"
-	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
-	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
 )
 
 const (
@@ -45,36 +45,52 @@ type BackFillInterface interface {
 
 // BackFillService for filling in gaps in the super node
 type BackFillService struct {
-	// Interface for converting statediff payloads into ETH-IPLD object payloads
+	// Interface for converting payloads into IPLD object payloads
-	Converter ipfs.PayloadConverter
+	Converter shared.PayloadConverter
-	// Interface for publishing the ETH-IPLD payloads to IPFS
+	// Interface for publishing the IPLD payloads to IPFS
-	Publisher ipfs.IPLDPublisher
+	Publisher shared.IPLDPublisher
-	// Interface for indexing the CIDs of the published ETH-IPLDs in Postgres
+	// Interface for indexing the CIDs of the published IPLDs in Postgres
-	Repository CIDRepository
+	Indexer shared.CIDIndexer
 	// Interface for searching and retrieving CIDs from Postgres index
-	Retriever CIDRetriever
+	Retriever shared.CIDRetriever
-	// State-diff fetcher; needs to be configured with an archival core.RpcClient
+	// Interface for fetching payloads over at historical blocks; over http
-	Fetcher fetcher.StateDiffFetcher
+	Fetcher shared.PayloadFetcher
 	// Check frequency
 	GapCheckFrequency time.Duration
-	// size of batch fetches
+	// Size of batch fetches
 	BatchSize uint64
 }
 
 // NewBackFillService returns a new BackFillInterface
-func NewBackFillService(ipfsPath string, db *postgres.DB, archivalNodeRPCClient core.RPCClient, freq time.Duration, batchSize uint64) (BackFillInterface, error) {
+func NewBackFillService(settings *config.BackFill) (BackFillInterface, error) {
-	publisher, err := ipfs.NewIPLDPublisher(ipfsPath)
+	publisher, err := NewIPLDPublisher(settings.Chain, settings.IPFSPath)
+	if err != nil {
+		return nil, err
+	}
+	indexer, err := NewCIDIndexer(settings.Chain, settings.DB)
+	if err != nil {
+		return nil, err
+	}
+	converter, err := NewPayloadConverter(settings.Chain, params.MainnetChainConfig)
+	if err != nil {
+		return nil, err
+	}
+	retriever, err := NewCIDRetriever(settings.Chain, settings.DB)
+	if err != nil {
+		return nil, err
+	}
+	fetcher, err := NewPaylaodFetcher(settings.Chain, settings.HTTPClient)
 	if err != nil {
 		return nil, err
 	}
 	return &BackFillService{
-		Repository:        NewCIDRepository(db),
+		Indexer:           indexer,
-		Converter:         ipfs.NewPayloadConverter(params.MainnetChainConfig),
+		Converter:         converter,
 		Publisher:         publisher,
-		Retriever:         NewCIDRetriever(db),
+		Retriever:         retriever,
-		Fetcher:           fetcher.NewStateDiffFetcher(archivalNodeRPCClient),
+		Fetcher:           fetcher,
-		GapCheckFrequency: freq,
+		GapCheckFrequency: settings.Frequency,
-		BatchSize:         batchSize,
+		BatchSize:         settings.BatchSize,
 	}, nil
 }
 
@@ -93,23 +109,24 @@ func (bfs *BackFillService) FillGaps(wg *sync.WaitGroup, quitChan <-chan bool) {
 				return
 			case <-ticker.C:
 				log.Info("searching for gaps in the super node database")
-				startingBlock, firstBlockErr := bfs.Retriever.RetrieveFirstBlockNumber()
+				startingBlock, err := bfs.Retriever.RetrieveFirstBlockNumber()
-				if firstBlockErr != nil {
+				if err != nil {
-					log.Error(firstBlockErr)
+					log.Error(err)
 					continue
 				}
 				if startingBlock != 1 {
 					log.Info("found gap at the beginning of the sync")
 					bfs.fillGaps(1, uint64(startingBlock-1))
 				}
-
+				gaps, err := bfs.Retriever.RetrieveGapsInData()
-				gaps, gapErr := bfs.Retriever.RetrieveGapsInData()
+				if err != nil {
-				if gapErr != nil {
+					log.Error(err)
-					log.Error(gapErr)
 					continue
 				}
 				for _, gap := range gaps {
-					bfs.fillGaps(gap[0], gap[1])
+					if err := bfs.fillGaps(gap.Start, gap.Stop); err != nil {
+						log.Error(err)
+					}
 				}
 			}
 		}
@@ -117,14 +134,13 @@ func (bfs *BackFillService) FillGaps(wg *sync.WaitGroup, quitChan <-chan bool) {
 	log.Info("fillGaps goroutine successfully spun up")
 }
 
-func (bfs *BackFillService) fillGaps(startingBlock, endingBlock uint64) {
+func (bfs *BackFillService) fillGaps(startingBlock, endingBlock uint64) error {
 	log.Infof("going to fill in gap from %d to %d", startingBlock, endingBlock)
 	errChan := make(chan error)
 	done := make(chan bool)
-	backFillInitErr := bfs.backFill(startingBlock, endingBlock, errChan, done)
+	err := bfs.backFill(startingBlock, endingBlock, errChan, done)
-	if backFillInitErr != nil {
+	if err != nil {
-		log.Error(backFillInitErr)
+		return err
-		return
 	}
 	for {
 		select {
@@ -132,7 +148,7 @@ func (bfs *BackFillService) fillGaps(startingBlock, endingBlock uint64) {
 			log.Error(err)
 		case <-done:
 			log.Infof("finished filling in gap from %d to %d", startingBlock, endingBlock)
-			return
+			return nil
 		}
 	}
 }
@@ -165,24 +181,26 @@ func (bfs *BackFillService) backFill(startingBlock, endingBlock uint64, errChan
 				<-forwardDone
 			}
 			go func(blockHeights []uint64) {
-				payloads, fetchErr := bfs.Fetcher.FetchStateDiffsAt(blockHeights)
+				payloads, err := bfs.Fetcher.FetchAt(blockHeights)
-				if fetchErr != nil {
+				if err != nil {
-					errChan <- fetchErr
+					errChan <- err
 				}
 				for _, payload := range payloads {
-					ipldPayload, convertErr := bfs.Converter.Convert(payload)
+					ipldPayload, err := bfs.Converter.Convert(payload)
-					if convertErr != nil {
+					if err != nil {
-						errChan <- convertErr
+						errChan <- err
 						continue
 					}
-					cidPayload, publishErr := bfs.Publisher.Publish(ipldPayload)
+					// make backfiller a part of super_node service and forward these
-					if publishErr != nil {
+					// ipldPayload the the regular publishAndIndex and screenAndServe channels
-						errChan <- publishErr
+					// this would allow us to stream backfilled data to subscribers
+					cidPayload, err := bfs.Publisher.Publish(ipldPayload)
+					if err != nil {
+						errChan <- err
 						continue
 					}
-					indexErr := bfs.Repository.Index(cidPayload)
+					if err := bfs.Indexer.Index(cidPayload); err != nil {
-					if indexErr != nil {
+						errChan <- err
-						errChan <- indexErr
 					}
 				}
 				// when this goroutine is done, send out a signal

pkg/super_node/backfiller_test.go

@@ -24,43 +24,42 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	mocks2 "github.com/vulcanize/vulcanizedb/libraries/shared/mocks"
-	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
-	"github.com/vulcanize/vulcanizedb/pkg/ipfs/mocks"
 	"github.com/vulcanize/vulcanizedb/pkg/super_node"
-	mocks3 "github.com/vulcanize/vulcanizedb/pkg/super_node/mocks"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth/mocks"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
 )
 
 var _ = Describe("BackFiller", func() {
 	Describe("FillGaps", func() {
 		It("Periodically checks for and fills in gaps in the super node's data", func() {
-			mockCidRepo := &mocks3.CIDRepository{
+			mockCidRepo := &mocks.CIDIndexer{
 				ReturnErr: nil,
 			}
 			mockPublisher := &mocks.IterativeIPLDPublisher{
-				ReturnCIDPayload: []*ipfs.CIDPayload{mocks.MockCIDPayload, mocks.MockCIDPayload},
+				ReturnCIDPayload: []*eth.CIDPayload{mocks.MockCIDPayload, mocks.MockCIDPayload},
 				ReturnErr:        nil,
 			}
 			mockConverter := &mocks.IterativePayloadConverter{
-				ReturnIPLDPayload: []*ipfs.IPLDPayload{mocks.MockIPLDPayload, mocks.MockIPLDPayload},
+				ReturnIPLDPayload: []*eth.IPLDPayload{mocks.MockIPLDPayload, mocks.MockIPLDPayload},
 				ReturnErr:         nil,
 			}
-			mockRetriever := &mocks3.MockCIDRetriever{
+			mockRetriever := &mocks.MockCIDRetriever{
 				FirstBlockNumberToReturn: 1,
-				GapsToRetrieve: [][2]uint64{
+				GapsToRetrieve: []shared.Gap{
 					{
-						100, 101,
+						Start: 100, Stop: 101,
 					},
 				},
 			}
-			mockFetcher := &mocks2.StateDiffFetcher{
+			mockFetcher := &mocks.StateDiffFetcher{
 				PayloadsToReturn: map[uint64]statediff.Payload{
 					100: mocks.MockStateDiffPayload,
 					101: mocks.MockStateDiffPayload,
 				},
 			}
 			backfiller := &super_node.BackFillService{
-				Repository:        mockCidRepo,
+				Indexer:           mockCidRepo,
 				Publisher:         mockPublisher,
 				Converter:         mockConverter,
 				Fetcher:           mockFetcher,
@@ -88,32 +87,32 @@ var _ = Describe("BackFiller", func() {
 		})
 
 		It("Works for single block `ranges`", func() {
-			mockCidRepo := &mocks3.CIDRepository{
+			mockCidRepo := &mocks.CIDIndexer{
 				ReturnErr: nil,
 			}
 			mockPublisher := &mocks.IterativeIPLDPublisher{
-				ReturnCIDPayload: []*ipfs.CIDPayload{mocks.MockCIDPayload},
+				ReturnCIDPayload: []*eth.CIDPayload{mocks.MockCIDPayload},
 				ReturnErr:        nil,
 			}
 			mockConverter := &mocks.IterativePayloadConverter{
-				ReturnIPLDPayload: []*ipfs.IPLDPayload{mocks.MockIPLDPayload},
+				ReturnIPLDPayload: []*eth.IPLDPayload{mocks.MockIPLDPayload},
 				ReturnErr:         nil,
 			}
-			mockRetriever := &mocks3.MockCIDRetriever{
+			mockRetriever := &mocks.MockCIDRetriever{
 				FirstBlockNumberToReturn: 1,
-				GapsToRetrieve: [][2]uint64{
+				GapsToRetrieve: []shared.Gap{
 					{
-						100, 100,
+						Start: 100, Stop: 100,
 					},
 				},
 			}
-			mockFetcher := &mocks2.StateDiffFetcher{
+			mockFetcher := &mocks.StateDiffFetcher{
 				PayloadsToReturn: map[uint64]statediff.Payload{
 					100: mocks.MockStateDiffPayload,
 				},
 			}
 			backfiller := &super_node.BackFillService{
-				Repository:        mockCidRepo,
+				Indexer:           mockCidRepo,
 				Publisher:         mockPublisher,
 				Converter:         mockConverter,
 				Fetcher:           mockFetcher,
@@ -138,29 +137,29 @@ var _ = Describe("BackFiller", func() {
 		})
 
 		It("Finds beginning gap", func() {
-			mockCidRepo := &mocks3.CIDRepository{
+			mockCidRepo := &mocks.CIDIndexer{
 				ReturnErr: nil,
 			}
 			mockPublisher := &mocks.IterativeIPLDPublisher{
-				ReturnCIDPayload: []*ipfs.CIDPayload{mocks.MockCIDPayload, mocks.MockCIDPayload},
+				ReturnCIDPayload: []*eth.CIDPayload{mocks.MockCIDPayload, mocks.MockCIDPayload},
 				ReturnErr:        nil,
 			}
 			mockConverter := &mocks.IterativePayloadConverter{
-				ReturnIPLDPayload: []*ipfs.IPLDPayload{mocks.MockIPLDPayload, mocks.MockIPLDPayload},
+				ReturnIPLDPayload: []*eth.IPLDPayload{mocks.MockIPLDPayload, mocks.MockIPLDPayload},
 				ReturnErr:         nil,
 			}
-			mockRetriever := &mocks3.MockCIDRetriever{
+			mockRetriever := &mocks.MockCIDRetriever{
 				FirstBlockNumberToReturn: 3,
-				GapsToRetrieve:           [][2]uint64{},
+				GapsToRetrieve:           []shared.Gap{},
 			}
-			mockFetcher := &mocks2.StateDiffFetcher{
+			mockFetcher := &mocks.StateDiffFetcher{
 				PayloadsToReturn: map[uint64]statediff.Payload{
 					1: mocks.MockStateDiffPayload,
 					2: mocks.MockStateDiffPayload,
 				},
 			}
 			backfiller := &super_node.BackFillService{
-				Repository:        mockCidRepo,
+				Indexer:           mockCidRepo,
 				Publisher:         mockPublisher,
 				Converter:         mockConverter,
 				Fetcher:           mockFetcher,

pkg/super_node/config/chain_type.go

@@ -0,0 +1,58 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package config
+
+import (
+	"errors"
+	"strings"
+)
+
+// ChainType enum for specifying blockchain
+type ChainType int
+
+const (
+	Unknown ChainType = iota
+	Ethereum
+	Bitcoin
+	Omni
+)
+
+func (c ChainType) String() string {
+	switch c {
+	case Ethereum:
+		return "Ethereum"
+	case Bitcoin:
+		return "Bitcoin"
+	case Omni:
+		return "Omni"
+	default:
+		return ""
+	}
+}
+
+func NewChainType(name string) (ChainType, error) {
+	switch strings.ToLower(name) {
+	case "ethereum", "eth":
+		return Ethereum, nil
+	case "bitcoin", "btc", "xbt":
+		return Bitcoin, nil
+	case "omni":
+		return Omni, nil
+	default:
+		return Unknown, errors.New("invalid name for chain")
+	}
+}

pkg/super_node/config/config.go

@@ -0,0 +1,179 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package config
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/ethereum/go-ethereum/ethclient"
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/spf13/viper"
+
+	"github.com/vulcanize/vulcanizedb/pkg/config"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/eth"
+	"github.com/vulcanize/vulcanizedb/pkg/eth/client"
+	vRpc "github.com/vulcanize/vulcanizedb/pkg/eth/converters/rpc"
+	"github.com/vulcanize/vulcanizedb/pkg/eth/node"
+	"github.com/vulcanize/vulcanizedb/utils"
+)
+
+// SuperNode config struct
+type SuperNode struct {
+	// Ubiquitous fields
+	Chain    ChainType
+	IPFSPath string
+	DB       *postgres.DB
+	Quit     chan bool
+	// Server fields
+	Serve       bool
+	WSEndpoint  string
+	IPCEndpoint string
+	// Sync params
+	Sync     bool
+	Workers  int
+	WSClient core.RPCClient
+	NodeInfo core.Node
+	// Backfiller params
+	BackFill         bool
+	BackFillSettings *BackFill
+}
+
+// NewSuperNodeConfig is used to initialize a SuperNode config
+func NewSuperNodeConfig() (*SuperNode, error) {
+	dbConfig := config.Database{
+		Name:     viper.GetString("superNode.database.name"),
+		Hostname: viper.GetString("superNode.database.hostname"),
+		Port:     viper.GetInt("superNode.database.port"),
+		User:     viper.GetString("superNode.database.user"),
+		Password: viper.GetString("superNode.database.password"),
+	}
+	sn := new(SuperNode)
+	var err error
+	sn.Chain, err = NewChainType(viper.GetString("superNode.chain"))
+	if err != nil {
+		return nil, err
+	}
+	ipfsPath := viper.GetString("superNode.ipfsPath")
+	if ipfsPath == "" {
+		home, err := os.UserHomeDir()
+		if err != nil {
+			return nil, err
+		}
+		ipfsPath = filepath.Join(home, ".ipfs")
+	}
+	sn.IPFSPath = ipfsPath
+	sn.Serve = viper.GetBool("superNode.server.on")
+	sn.Sync = viper.GetBool("superNode.sync.on")
+	if sn.Sync {
+		workers := viper.GetInt("superNode.sync.workers")
+		if workers < 1 {
+			workers = 1
+		}
+		sn.Workers = workers
+		sn.NodeInfo, sn.WSClient, err = getNodeAndClient(sn.Chain, viper.GetString("superNode.sync.wsPath"))
+	}
+	if sn.Serve {
+		wsPath := viper.GetString("superNode.server.wsPath")
+		if wsPath == "" {
+			wsPath = "127.0.0.1:8080"
+		}
+		sn.WSEndpoint = wsPath
+		ipcPath := viper.GetString("superNode.server.ipcPath")
+		if ipcPath == "" {
+			home, err := os.UserHomeDir()
+			if err != nil {
+				return nil, err
+			}
+			ipcPath = filepath.Join(home, ".vulcanize/vulcanize.ipc")
+		}
+		sn.IPCEndpoint = ipcPath
+	}
+	db := utils.LoadPostgres(dbConfig, sn.NodeInfo)
+	sn.DB = &db
+	sn.Quit = make(chan bool)
+	if viper.GetBool("superNode.backFill.on") {
+		sn.BackFill = true
+		sn.BackFillSettings, err = NewBackFillerConfig(dbConfig)
+	}
+	return sn, err
+}
+
+// BackFill config struct
+type BackFill struct {
+	Chain      ChainType
+	IPFSPath   string
+	DB         *postgres.DB
+	HTTPClient core.RPCClient
+	Frequency  time.Duration
+	BatchSize  uint64
+}
+
+// newBackFillerConfig is used to initialize a backfiller config
+func NewBackFillerConfig(dbConfig config.Database) (*BackFill, error) {
+	bf := new(BackFill)
+	var err error
+	bf.Chain, err = NewChainType(viper.GetString("superNode.chain"))
+	if err != nil {
+		return nil, err
+	}
+	ipfsPath := viper.GetString("superNode.ipfsPath")
+	if ipfsPath == "" {
+		home, homeDirErr := os.UserHomeDir()
+		if homeDirErr != nil {
+			return nil, err
+		}
+		ipfsPath = filepath.Join(home, ".ipfs")
+	}
+	bf.IPFSPath = ipfsPath
+	node, httpClient, err := getNodeAndClient(bf.Chain, viper.GetString("superNode.backFill.httpPath"))
+	db := utils.LoadPostgres(dbConfig, node)
+	bf.DB = &db
+	bf.HTTPClient = httpClient
+	freq := viper.GetInt("superNode.backFill.frequency")
+	var frequency time.Duration
+	if freq <= 0 {
+		frequency = time.Minute * 5
+	} else {
+		frequency = time.Duration(freq)
+	}
+	bf.Frequency = frequency
+	return bf, nil
+}
+
+func getNodeAndClient(chain ChainType, path string) (core.Node, core.RPCClient, error) {
+	switch chain {
+	case Ethereum:
+		rawRPCClient, err := rpc.Dial(path)
+		if err != nil {
+			return core.Node{}, nil, err
+		}
+		rpcClient := client.NewRPCClient(rawRPCClient, path)
+		ethClient := ethclient.NewClient(rawRPCClient)
+		vdbEthClient := client.NewEthClient(ethClient)
+		vdbNode := node.MakeNode(rpcClient)
+		transactionConverter := vRpc.NewRPCTransactionConverter(ethClient)
+		blockChain := eth.NewBlockChain(vdbEthClient, rpcClient, vdbNode, transactionConverter)
+		return blockChain.Node(), rpcClient, nil
+	default:
+		return core.Node{}, nil, fmt.Errorf("unrecognized chain type %s", chain.String())
+	}
+}

pkg/super_node/config/eth_subscription.go

@@ -0,0 +1,144 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package config
+
+import (
+	"math/big"
+
+	"github.com/spf13/viper"
+)
+
+// EthSubscription config is used by a subscriber to specify what eth data to stream from the super node
+type EthSubscription struct {
+	BackFill      bool
+	BackFillOnly  bool
+	Start         *big.Int
+	End           *big.Int // set to 0 or a negative value to have no ending block
+	HeaderFilter  HeaderFilter
+	TxFilter      TxFilter
+	ReceiptFilter ReceiptFilter
+	StateFilter   StateFilter
+	StorageFilter StorageFilter
+}
+
+// HeaderFilter contains filter settings for headers
+type HeaderFilter struct {
+	Off    bool
+	Uncles bool
+}
+
+// TxFilter contains filter settings for txs
+type TxFilter struct {
+	Off bool
+	Src []string
+	Dst []string
+}
+
+// ReceiptFilter contains filter settings for receipts
+type ReceiptFilter struct {
+	Off       bool
+	MatchTxs  bool // turn on to retrieve receipts that pair with retrieved transactions
+	Contracts []string
+	Topic0s   []string
+}
+
+// StateFilter contains filter settings for state
+type StateFilter struct {
+	Off               bool
+	Addresses         []string // is converted to state key by taking its keccak256 hash
+	IntermediateNodes bool
+}
+
+// StorageFilter contains filter settings for storage
+type StorageFilter struct {
+	Off               bool
+	Addresses         []string
+	StorageKeys       []string
+	IntermediateNodes bool
+}
+
+// Init is used to initialize a EthSubscription struct with env variables
+func NewEthSubscriptionConfig() *EthSubscription {
+	sc := new(EthSubscription)
+	// Below default to false, which means we do not backfill by default
+	sc.BackFill = viper.GetBool("superNode.ethSubscription.historicalData")
+	sc.BackFillOnly = viper.GetBool("superNode.ethSubscription.historicalDataOnly")
+	// Below default to 0
+	// 0 start means we start at the beginning and 0 end means we continue indefinitely
+	sc.Start = big.NewInt(viper.GetInt64("superNode.ethSubscription.startingBlock"))
+	sc.End = big.NewInt(viper.GetInt64("superNode.ethSubscription.endingBlock"))
+	// Below default to false, which means we get all headers and no uncles by default
+	sc.HeaderFilter = HeaderFilter{
+		Off:    viper.GetBool("superNode.ethSubscription.off"),
+		Uncles: viper.GetBool("superNode.ethSubscription.uncles"),
+	}
+	// Below defaults to false and two slices of length 0
+	// Which means we get all transactions by default
+	sc.TxFilter = TxFilter{
+		Off: viper.GetBool("superNode.ethSubscription.trxFilter.off"),
+		Src: viper.GetStringSlice("superNode.ethSubscription.trxFilter.src"),
+		Dst: viper.GetStringSlice("superNode.ethSubscription.trxFilter.dst"),
+	}
+	// Below defaults to false and one slice of length 0
+	// Which means we get all receipts by default
+	sc.ReceiptFilter = ReceiptFilter{
+		Off:       viper.GetBool("superNode.ethSubscription.receiptFilter.off"),
+		Contracts: viper.GetStringSlice("superNode.ethSubscription.receiptFilter.contracts"),
+		Topic0s:   viper.GetStringSlice("superNode.ethSubscription.receiptFilter.topic0s"),
+	}
+	// Below defaults to two false, and a slice of length 0
+	// Which means we get all state leafs by default, but no intermediate nodes
+	sc.StateFilter = StateFilter{
+		Off:               viper.GetBool("superNode.ethSubscription.stateFilter.off"),
+		IntermediateNodes: viper.GetBool("superNode.ethSubscription.stateFilter.intermediateNodes"),
+		Addresses:         viper.GetStringSlice("superNode.ethSubscription.stateFilter.addresses"),
+	}
+	// Below defaults to two false, and two slices of length 0
+	// Which means we get all storage leafs by default, but no intermediate nodes
+	sc.StorageFilter = StorageFilter{
+		Off:               viper.GetBool("superNode.ethSubscription.storageFilter.off"),
+		IntermediateNodes: viper.GetBool("superNode.ethSubscription.storageFilter.intermediateNodes"),
+		Addresses:         viper.GetStringSlice("superNode.ethSubscription.storageFilter.addresses"),
+		StorageKeys:       viper.GetStringSlice("superNode.ethSubscription.storageFilter.storageKeys"),
+	}
+	return sc
+}
+
+// StartingBlock satisfies the SubscriptionSettings() interface
+func (sc *EthSubscription) StartingBlock() *big.Int {
+	return sc.Start
+}
+
+// EndingBlock satisfies the SubscriptionSettings() interface
+func (sc *EthSubscription) EndingBlock() *big.Int {
+	return sc.End
+}
+
+// HistoricalData satisfies the SubscriptionSettings() interface
+func (sc *EthSubscription) HistoricalData() bool {
+	return sc.BackFill
+}
+
+// HistoricalDataOnly satisfies the SubscriptionSettings() interface
+func (sc *EthSubscription) HistoricalDataOnly() bool {
+	return sc.BackFillOnly
+}
+
+// ChainType satisfies the SubscriptionSettings() interface
+func (sc *EthSubscription) ChainType() ChainType {
+	return Ethereum
+}

pkg/super_node/constructors.go

@@ -0,0 +1,155 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package super_node
+
+import (
+	"fmt"
+
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
+
+	"github.com/ethereum/go-ethereum/params"
+	"github.com/ethereum/go-ethereum/rpc"
+
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/config"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
+)
+
+// NewResponseFilterer constructs a ResponseFilterer for the provided chain type
+func NewResponseFilterer(chain config.ChainType) (shared.ResponseFilterer, error) {
+	switch chain {
+	case config.Ethereum:
+		return eth.NewResponseFilterer(), nil
+	default:
+		return nil, fmt.Errorf("invalid chain %T for filterer constructor", chain)
+	}
+}
+
+// NewCIDIndexer constructs a CIDIndexer for the provided chain type
+func NewCIDIndexer(chain config.ChainType, db *postgres.DB) (shared.CIDIndexer, error) {
+	switch chain {
+	case config.Ethereum:
+		return eth.NewCIDIndexer(db), nil
+	default:
+		return nil, fmt.Errorf("invalid chain %T for indexer constructor", chain)
+	}
+}
+
+// NewCIDRetriever constructs a CIDRetriever for the provided chain type
+func NewCIDRetriever(chain config.ChainType, db *postgres.DB) (shared.CIDRetriever, error) {
+	switch chain {
+	case config.Ethereum:
+		return eth.NewCIDRetriever(db), nil
+	default:
+		return nil, fmt.Errorf("invalid chain %T for retriever constructor", chain)
+	}
+}
+
+// NewPayloadStreamer constructs a PayloadStreamer for the provided chain type
+func NewPayloadStreamer(chain config.ChainType, client interface{}) (shared.PayloadStreamer, chan interface{}, error) {
+	switch chain {
+	case config.Ethereum:
+		ethClient, ok := client.(core.RPCClient)
+		if !ok {
+			var expectedClientType core.RPCClient
+			return nil, nil, fmt.Errorf("ethereum payload constructor expected client type %T got %T", expectedClientType, client)
+		}
+		streamChan := make(chan interface{}, eth.PayloadChanBufferSize)
+		return eth.NewPayloadStreamer(ethClient), streamChan, nil
+	default:
+		return nil, nil, fmt.Errorf("invalid chain %T for streamer constructor", chain)
+	}
+}
+
+// NewPaylaodFetcher constructs a PayloadFetcher for the provided chain type
+func NewPaylaodFetcher(chain config.ChainType, client interface{}) (shared.PayloadFetcher, error) {
+	switch chain {
+	case config.Ethereum:
+		batchClient, ok := client.(eth.BatchClient)
+		if !ok {
+			var expectedClient eth.BatchClient
+			return nil, fmt.Errorf("ethereum fetcher constructor expected client type %T got %T", expectedClient, client)
+		}
+		return eth.NewPayloadFetcher(batchClient), nil
+	default:
+		return nil, fmt.Errorf("invalid chain %T for fetcher constructor", chain)
+	}
+}
+
+// NewPayloadConverter constructs a PayloadConverter for the provided chain type
+func NewPayloadConverter(chain config.ChainType, settings interface{}) (shared.PayloadConverter, error) {
+	switch chain {
+	case config.Ethereum:
+		ethConfig, ok := settings.(*params.ChainConfig)
+		if !ok {
+			return nil, fmt.Errorf("ethereum converter constructor expected config type %T got %T", &params.ChainConfig{}, settings)
+		}
+		return eth.NewPayloadConverter(ethConfig), nil
+	default:
+		return nil, fmt.Errorf("invalid chain %T for converter constructor", chain)
+	}
+}
+
+// NewIPLDFetcher constructs an IPLDFetcher for the provided chain type
+func NewIPLDFetcher(chain config.ChainType, ipfsPath string) (shared.IPLDFetcher, error) {
+	switch chain {
+	case config.Ethereum:
+		return eth.NewIPLDFetcher(ipfsPath)
+	default:
+		return nil, fmt.Errorf("invalid chain %T for fetcher constructor", chain)
+	}
+}
+
+// NewIPLDPublisher constructs an IPLDPublisher for the provided chain type
+func NewIPLDPublisher(chain config.ChainType, ipfsPath string) (shared.IPLDPublisher, error) {
+	switch chain {
+	case config.Ethereum:
+		return eth.NewIPLDPublisher(ipfsPath)
+	default:
+		return nil, fmt.Errorf("invalid chain %T for publisher constructor", chain)
+	}
+}
+
+// NewIPLDResolver constructs an IPLDResolver for the provided chain type
+func NewIPLDResolver(chain config.ChainType) (shared.IPLDResolver, error) {
+	switch chain {
+	case config.Ethereum:
+		return eth.NewIPLDResolver(), nil
+	default:
+		return nil, fmt.Errorf("invalid chain %T for resolver constructor", chain)
+	}
+}
+
+// NewPublicAPI constructs a PublicAPI for the provided chain type
+func NewPublicAPI(chain config.ChainType, db *postgres.DB, ipfsPath string) (rpc.API, error) {
+	switch chain {
+	case config.Ethereum:
+		backend, err := eth.NewEthBackend(db, ipfsPath)
+		if err != nil {
+			return rpc.API{}, err
+		}
+		return rpc.API{
+			Namespace: eth.APIName,
+			Version:   eth.APIVersion,
+			Service:   eth.NewPublicEthAPI(backend),
+			Public:    true,
+		}, nil
+	default:
+		return rpc.API{}, fmt.Errorf("invalid chain %T for public api constructor", chain)
+	}
+}

pkg/super_node/eth/api.go

@@ -1,3 +1,19 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
 package eth
 
 import (
@@ -9,18 +25,18 @@ import (
 	"github.com/ethereum/go-ethereum/rpc"
 )
 
-// APIName is the namespace used for the state diffing service API
+// APIName is the namespace for the super node's eth api
 const APIName = "eth"
 
-// APIVersion is the version of the state diffing service API
+// APIVersion is the version of the super node's eth api
 const APIVersion = "0.0.1"
 
 type PublicEthAPI struct {
-	b Backend
+	b *Backend
 }
 
 // NewPublicEthAPI creates a new PublicEthAPI with the provided underlying Backend
-func NewPublicEthAPI(b Backend) *PublicEthAPI {
+func NewPublicEthAPI(b *Backend) *PublicEthAPI {
 	return &PublicEthAPI{
 		b: b,
 	}

pkg/super_node/eth/backend.go

@@ -1,3 +1,19 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
 package eth
 
 import (
@@ -6,16 +22,13 @@ import (
 	"fmt"
 	"math/big"
 
-	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
-
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/hashicorp/golang-lru"
 
-	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
-	"github.com/vulcanize/vulcanizedb/pkg/super_node"
 )
 
 var (
@@ -23,8 +36,8 @@ var (
 )
 
 type Backend struct {
-	retriever super_node.CIDRetriever
+	retriever *CIDRetriever
-	fetcher   ipfs.IPLDFetcher
+	fetcher   *IPLDFetcher
 	db        *postgres.DB
 
 	headerCache *lru.Cache // Cache for the most recent block headers
@@ -32,12 +45,17 @@ type Backend struct {
 	numberCache *lru.Cache // Cache for the most recent block numbers
 }
 
-func NewEthBackend(r super_node.CIDRetriever, f ipfs.IPLDFetcher) *Backend {
+func NewEthBackend(db *postgres.DB, ipfsPath string) (*Backend, error) {
+	r := NewCIDRetriever(db)
+	f, err := NewIPLDFetcher(ipfsPath)
+	if err != nil {
+		return nil, err
+	}
 	return &Backend{
 		retriever: r,
 		fetcher:   f,
 		db:        r.Database(),
-	}
+	}, nil
 }
 
 func (b *Backend) HeaderByNumber(ctx context.Context, blockNumber rpc.BlockNumber) (*types.Header, error) {

pkg/super_node/eth/converter.go

@@ -14,9 +14,11 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package ipfs
+package eth
 
 import (
+	"fmt"
+
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/params"
@@ -24,31 +26,30 @@ import (
 	"github.com/ethereum/go-ethereum/statediff"
 )
 
-// PayloadConverter interface is used to convert a geth statediff.Payload to our IPLDPayload type
+// PayloadConverter satisfies the PayloadConverter interface for ethereum
-type PayloadConverter interface {
+type PayloadConverter struct {
-	Convert(payload statediff.Payload) (*IPLDPayload, error)
-}
-
-// Converter is the underlying struct for the PayloadConverter interface
-type Converter struct {
 	chainConfig *params.ChainConfig
 }
 
 // NewPayloadConverter creates a pointer to a new Converter which satisfies the PayloadConverter interface
-func NewPayloadConverter(chainConfig *params.ChainConfig) *Converter {
+func NewPayloadConverter(chainConfig *params.ChainConfig) *PayloadConverter {
-	return &Converter{
+	return &PayloadConverter{
 		chainConfig: chainConfig,
 	}
 }
 
 // Convert method is used to convert a geth statediff.Payload to a IPLDPayload
-func (pc *Converter) Convert(payload statediff.Payload) (*IPLDPayload, error) {
+func (pc *PayloadConverter) Convert(payload interface{}) (interface{}, error) {
+	stateDiffPayload, ok := payload.(statediff.Payload)
+	if !ok {
+		return nil, fmt.Errorf("eth converter: expected payload type %T got %T", statediff.Payload{}, payload)
+	}
 	// Unpack block rlp to access fields
 	block := new(types.Block)
-	err := rlp.DecodeBytes(payload.BlockRlp, block)
+	if err := rlp.DecodeBytes(stateDiffPayload.BlockRlp, block); err != nil {
-	if err != nil {
 		return nil, err
 	}
+	// Process and publish headers
 	header := block.Header()
 	headerRlp, err := rlp.EncodeToBytes(header)
 	if err != nil {
@@ -56,16 +57,14 @@ func (pc *Converter) Convert(payload statediff.Payload) (*IPLDPayload, error) {
 	}
 	trxLen := len(block.Transactions())
 	convertedPayload := &IPLDPayload{
-		TotalDifficulty: payload.TotalDifficulty,
+		TotalDifficulty: stateDiffPayload.TotalDifficulty,
-		BlockHash:       block.Hash(),
+		Block:           block,
-		BlockNumber:     block.Number(),
 		HeaderRLP:       headerRlp,
-		BlockBody:       block.Body(),
+		TrxMetaData:     make([]TxModel, 0, trxLen),
-		TrxMetaData:     make([]*TrxMetaData, 0, trxLen),
 		Receipts:        make(types.Receipts, 0, trxLen),
-		ReceiptMetaData: make([]*ReceiptMetaData, 0, trxLen),
+		ReceiptMetaData: make([]ReceiptModel, 0, trxLen),
-		StateNodes:      make(map[common.Hash]StateNode),
+		StateNodes:      make([]TrieNode, 0),
-		StorageNodes:    make(map[common.Hash][]StorageNode),
+		StorageNodes:    make(map[common.Hash][]TrieNode),
 	}
 	signer := types.MakeSigner(pc.chainConfig, block.Number())
 	transactions := block.Transactions()
@@ -75,7 +74,7 @@ func (pc *Converter) Convert(payload statediff.Payload) (*IPLDPayload, error) {
 		if err != nil {
 			return nil, err
 		}
-		txMeta := &TrxMetaData{
+		txMeta := TxModel{
 			Dst: handleNullAddr(trx.To()),
 			Src: handleNullAddr(&from),
 		}
@@ -85,7 +84,7 @@ func (pc *Converter) Convert(payload statediff.Payload) (*IPLDPayload, error) {
 
 	// Decode receipts for this block
 	receipts := make(types.Receipts, 0)
-	if err := rlp.DecodeBytes(payload.ReceiptsRlp, &receipts); err != nil {
+	if err := rlp.DecodeBytes(stateDiffPayload.ReceiptsRlp, &receipts); err != nil {
 		return nil, err
 	}
 	// Derive any missing fields
@@ -100,9 +99,9 @@ func (pc *Converter) Convert(payload statediff.Payload) (*IPLDPayload, error) {
 			receipt.ContractAddress = *transactions[i].To()
 		}
 		// Extract topic0 data from the receipt's logs for indexing
-		rctMeta := &ReceiptMetaData{
+		rctMeta := ReceiptModel{
-			Topic0s:         make([]string, 0, len(receipt.Logs)),
+			Topic0s:  make([]string, 0, len(receipt.Logs)),
-			ContractAddress: receipt.ContractAddress.Hex(),
+			Contract: receipt.ContractAddress.Hex(),
 		}
 		for _, log := range receipt.Logs {
 			if len(log.Topics) < 1 {
@@ -117,17 +116,18 @@ func (pc *Converter) Convert(payload statediff.Payload) (*IPLDPayload, error) {
 
 	// Unpack state diff rlp to access fields
 	stateDiff := new(statediff.StateDiff)
-	if err = rlp.DecodeBytes(payload.StateDiffRlp, stateDiff); err != nil {
+	if err := rlp.DecodeBytes(stateDiffPayload.StateDiffRlp, stateDiff); err != nil {
 		return nil, err
 	}
 	for _, createdAccount := range stateDiff.CreatedAccounts {
 		hashKey := common.BytesToHash(createdAccount.Key)
-		convertedPayload.StateNodes[hashKey] = StateNode{
+		convertedPayload.StateNodes = append(convertedPayload.StateNodes, TrieNode{
+			Key:   hashKey,
 			Value: createdAccount.Value,
 			Leaf:  createdAccount.Leaf,
-		}
+		})
 		for _, storageDiff := range createdAccount.Storage {
-			convertedPayload.StorageNodes[hashKey] = append(convertedPayload.StorageNodes[hashKey], StorageNode{
+			convertedPayload.StorageNodes[hashKey] = append(convertedPayload.StorageNodes[hashKey], TrieNode{
 				Key:   common.BytesToHash(storageDiff.Key),
 				Value: storageDiff.Value,
 				Leaf:  storageDiff.Leaf,
@@ -136,12 +136,13 @@ func (pc *Converter) Convert(payload statediff.Payload) (*IPLDPayload, error) {
 	}
 	for _, deletedAccount := range stateDiff.DeletedAccounts {
 		hashKey := common.BytesToHash(deletedAccount.Key)
-		convertedPayload.StateNodes[hashKey] = StateNode{
+		convertedPayload.StateNodes = append(convertedPayload.StateNodes, TrieNode{
+			Key:   hashKey,
 			Value: deletedAccount.Value,
 			Leaf:  deletedAccount.Leaf,
-		}
+		})
 		for _, storageDiff := range deletedAccount.Storage {
-			convertedPayload.StorageNodes[hashKey] = append(convertedPayload.StorageNodes[hashKey], StorageNode{
+			convertedPayload.StorageNodes[hashKey] = append(convertedPayload.StorageNodes[hashKey], TrieNode{
 				Key:   common.BytesToHash(storageDiff.Key),
 				Value: storageDiff.Value,
 				Leaf:  storageDiff.Leaf,
@@ -150,12 +151,13 @@ func (pc *Converter) Convert(payload statediff.Payload) (*IPLDPayload, error) {
 	}
 	for _, updatedAccount := range stateDiff.UpdatedAccounts {
 		hashKey := common.BytesToHash(updatedAccount.Key)
-		convertedPayload.StateNodes[hashKey] = StateNode{
+		convertedPayload.StateNodes = append(convertedPayload.StateNodes, TrieNode{
+			Key:   hashKey,
 			Value: updatedAccount.Value,
 			Leaf:  updatedAccount.Leaf,
-		}
+		})
 		for _, storageDiff := range updatedAccount.Storage {
-			convertedPayload.StorageNodes[hashKey] = append(convertedPayload.StorageNodes[hashKey], StorageNode{
+			convertedPayload.StorageNodes[hashKey] = append(convertedPayload.StorageNodes[hashKey], TrieNode{
 				Key:   common.BytesToHash(storageDiff.Key),
 				Value: storageDiff.Value,
 				Leaf:  storageDiff.Leaf,

pkg/super_node/eth/converter_test.go

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package ipfs_test
+package eth_test
 
 import (
 	"github.com/ethereum/go-ethereum/params"
@@ -22,35 +22,31 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
-	"github.com/vulcanize/vulcanizedb/pkg/ipfs/mocks"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth/mocks"
 )
 
 var _ = Describe("Converter", func() {
 	Describe("Convert", func() {
 		It("Converts mock statediff.Payloads into the expected IPLDPayloads", func() {
-			converter := ipfs.NewPayloadConverter(params.MainnetChainConfig)
+			converter := eth.NewPayloadConverter(params.MainnetChainConfig)
-			converterPayload, err := converter.Convert(mocks.MockStateDiffPayload)
+			payload, err := converter.Convert(mocks.MockStateDiffPayload)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(converterPayload.BlockNumber).To(Equal(mocks.BlockNumber))
+			convertedPayload, ok := payload.(*eth.IPLDPayload)
-			Expect(converterPayload.BlockHash).To(Equal(mocks.MockBlock.Hash()))
+			Expect(ok).To(BeTrue())
-			Expect(converterPayload.StateNodes).To(Equal(mocks.MockStateNodes))
+			Expect(convertedPayload.Block.Number().String()).To(Equal(mocks.BlockNumber.String()))
-			Expect(converterPayload.StorageNodes).To(Equal(mocks.MockStorageNodes))
+			Expect(convertedPayload.Block.Hash().String()).To(Equal(mocks.MockBlock.Hash().String()))
-			Expect(converterPayload.TotalDifficulty.Int64()).To(Equal(mocks.MockStateDiffPayload.TotalDifficulty.Int64()))
+			Expect(convertedPayload.StateNodes).To(Equal(mocks.MockStateNodes))
-			gotBody, err := rlp.EncodeToBytes(converterPayload.BlockBody)
+			Expect(convertedPayload.StorageNodes).To(Equal(mocks.MockStorageNodes))
+			Expect(convertedPayload.TotalDifficulty.Int64()).To(Equal(mocks.MockStateDiffPayload.TotalDifficulty.Int64()))
+			gotBody, err := rlp.EncodeToBytes(convertedPayload.Block.Body())
 			Expect(err).ToNot(HaveOccurred())
 			expectedBody, err := rlp.EncodeToBytes(mocks.MockBlock.Body())
 			Expect(err).ToNot(HaveOccurred())
 			Expect(gotBody).To(Equal(expectedBody))
-			Expect(converterPayload.HeaderRLP).To(Equal(mocks.MockHeaderRlp))
+			Expect(convertedPayload.HeaderRLP).To(Equal(mocks.MockHeaderRlp))
-			Expect(converterPayload.TrxMetaData).To(Equal(mocks.MockTrxMeta))
+			Expect(convertedPayload.TrxMetaData).To(Equal(mocks.MockTrxMeta))
-			Expect(converterPayload.ReceiptMetaData).To(Equal(mocks.MockRctMeta))
+			Expect(convertedPayload.ReceiptMetaData).To(Equal(mocks.MockRctMeta))
-		})
-
-		It(" Throws an error if the wrong chain config is used", func() {
-			converter := ipfs.NewPayloadConverter(params.TestnetChainConfig)
-			_, err := converter.Convert(mocks.MockStateDiffPayload)
-			Expect(err).To(HaveOccurred())
 		})
 	})
 })

pkg/super_node/eth/eth_suite_test.go

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package ipfs_test
+package eth_test
 
 import (
 	"io/ioutil"
@@ -25,9 +25,9 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-func TestIPFS(t *testing.T) {
+func TestETHSuperNode(t *testing.T) {
 	RegisterFailHandler(Fail)
-	RunSpecs(t, "IPFS Suite Test")
+	RunSpecs(t, "Super Node ETH Suite Test")
 }
 
 var _ = BeforeSuite(func() {

pkg/super_node/eth/filterer.go

@@ -14,65 +14,67 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package super_node
+package eth
 
 import (
 	"bytes"
+	"fmt"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/rlp"
 
-	"github.com/vulcanize/vulcanizedb/libraries/shared/streamer"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/config"
-	"github.com/vulcanize/vulcanizedb/pkg/config"
-	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
 )
 
-// ResponseFilterer is the inteface used to screen eth data and package appropriate data into a response payload
+// ResponseFilterer satisfies the ResponseFilterer interface for ethereum
-type ResponseFilterer interface {
+type ResponseFilterer struct{}
-	FilterResponse(streamFilters config.Subscription, payload ipfs.IPLDPayload) (streamer.SuperNodePayload, error)
-}
-
-// Filterer is the underlying struct for the ResponseFilterer interface
-type Filterer struct{}
 
 // NewResponseFilterer creates a new Filterer satisfying the ResponseFilterer interface
-func NewResponseFilterer() *Filterer {
+func NewResponseFilterer() *ResponseFilterer {
-	return &Filterer{}
+	return &ResponseFilterer{}
 }
 
-// FilterResponse is used to filter through eth data to extract and package requested data into a Payload
+// Filter is used to filter through eth data to extract and package requested data into a Payload
-func (s *Filterer) FilterResponse(streamFilters config.Subscription, payload ipfs.IPLDPayload) (streamer.SuperNodePayload, error) {
+func (s *ResponseFilterer) Filter(filter, payload interface{}) (interface{}, error) {
-	if checkRange(streamFilters.StartingBlock.Int64(), streamFilters.EndingBlock.Int64(), payload.BlockNumber.Int64()) {
+	ethFilters, ok := filter.(*config.EthSubscription)
-		response := new(streamer.SuperNodePayload)
+	if !ok {
-		if err := s.filterHeaders(streamFilters.HeaderFilter, response, payload); err != nil {
+		return StreamPayload{}, fmt.Errorf("eth filterer expected filter type %T got %T", &config.EthSubscription{}, filter)
-			return streamer.SuperNodePayload{}, err
+	}
+	ethPayload, ok := payload.(*IPLDPayload)
+	if !ok {
+		return StreamPayload{}, fmt.Errorf("eth filterer expected payload type %T got %T", &IPLDPayload{}, payload)
+	}
+	if checkRange(ethFilters.Start.Int64(), ethFilters.End.Int64(), ethPayload.Block.Number().Int64()) {
+		response := new(StreamPayload)
+		if err := s.filterHeaders(ethFilters.HeaderFilter, response, ethPayload); err != nil {
+			return StreamPayload{}, err
 		}
-		txHashes, err := s.filterTransactions(streamFilters.TrxFilter, response, payload)
+		txHashes, err := s.filterTransactions(ethFilters.TxFilter, response, ethPayload)
 		if err != nil {
-			return streamer.SuperNodePayload{}, err
+			return StreamPayload{}, err
 		}
-		if err := s.filerReceipts(streamFilters.ReceiptFilter, response, payload, txHashes); err != nil {
+		if err := s.filerReceipts(ethFilters.ReceiptFilter, response, ethPayload, txHashes); err != nil {
-			return streamer.SuperNodePayload{}, err
+			return StreamPayload{}, err
 		}
-		if err := s.filterState(streamFilters.StateFilter, response, payload); err != nil {
+		if err := s.filterState(ethFilters.StateFilter, response, ethPayload); err != nil {
-			return streamer.SuperNodePayload{}, err
+			return StreamPayload{}, err
 		}
-		if err := s.filterStorage(streamFilters.StorageFilter, response, payload); err != nil {
+		if err := s.filterStorage(ethFilters.StorageFilter, response, ethPayload); err != nil {
-			return streamer.SuperNodePayload{}, err
+			return StreamPayload{}, err
 		}
-		response.BlockNumber = payload.BlockNumber
+		response.BlockNumber = ethPayload.Block.Number()
 		return *response, nil
 	}
-	return streamer.SuperNodePayload{}, nil
+	return StreamPayload{}, nil
 }
 
-func (s *Filterer) filterHeaders(headerFilter config.HeaderFilter, response *streamer.SuperNodePayload, payload ipfs.IPLDPayload) error {
+func (s *ResponseFilterer) filterHeaders(headerFilter config.HeaderFilter, response *StreamPayload, payload *IPLDPayload) error {
 	if !headerFilter.Off {
 		response.HeadersRlp = append(response.HeadersRlp, payload.HeaderRLP)
 		if headerFilter.Uncles {
-			response.UnclesRlp = make([][]byte, 0, len(payload.BlockBody.Uncles))
+			response.UnclesRlp = make([][]byte, 0, len(payload.Block.Body().Uncles))
-			for _, uncle := range payload.BlockBody.Uncles {
+			for _, uncle := range payload.Block.Body().Uncles {
 				uncleRlp, err := rlp.EncodeToBytes(uncle)
 				if err != nil {
 					return err
@@ -91,10 +93,10 @@ func checkRange(start, end, actual int64) bool {
 	return false
 }
 
-func (s *Filterer) filterTransactions(trxFilter config.TrxFilter, response *streamer.SuperNodePayload, payload ipfs.IPLDPayload) ([]common.Hash, error) {
+func (s *ResponseFilterer) filterTransactions(trxFilter config.TxFilter, response *StreamPayload, payload *IPLDPayload) ([]common.Hash, error) {
-	trxHashes := make([]common.Hash, 0, len(payload.BlockBody.Transactions))
+	trxHashes := make([]common.Hash, 0, len(payload.Block.Body().Transactions))
 	if !trxFilter.Off {
-		for i, trx := range payload.BlockBody.Transactions {
+		for i, trx := range payload.Block.Body().Transactions {
 			if checkTransactions(trxFilter.Src, trxFilter.Dst, payload.TrxMetaData[i].Src, payload.TrxMetaData[i].Dst) {
 				trxBuffer := new(bytes.Buffer)
 				err := trx.EncodeRLP(trxBuffer)
@@ -127,10 +129,10 @@ func checkTransactions(wantedSrc, wantedDst []string, actualSrc, actualDst strin
 	return false
 }
 
-func (s *Filterer) filerReceipts(receiptFilter config.ReceiptFilter, response *streamer.SuperNodePayload, payload ipfs.IPLDPayload, trxHashes []common.Hash) error {
+func (s *ResponseFilterer) filerReceipts(receiptFilter config.ReceiptFilter, response *StreamPayload, payload *IPLDPayload, trxHashes []common.Hash) error {
 	if !receiptFilter.Off {
 		for i, receipt := range payload.Receipts {
-			if checkReceipts(receipt, receiptFilter.Topic0s, payload.ReceiptMetaData[i].Topic0s, receiptFilter.Contracts, payload.ReceiptMetaData[i].ContractAddress, trxHashes, receiptFilter.MatchTxs) {
+			if checkReceipts(receipt, receiptFilter.Topic0s, payload.ReceiptMetaData[i].Topic0s, receiptFilter.Contracts, payload.ReceiptMetaData[i].Contract, trxHashes, receiptFilter.MatchTxs) {
 				receiptForStorage := (*types.ReceiptForStorage)(receipt)
 				receiptBuffer := new(bytes.Buffer)
 				err := receiptForStorage.EncodeRLP(receiptBuffer)
@@ -188,18 +190,18 @@ func checkReceipts(rct *types.Receipt, wantedTopics, actualTopics, wantedContrac
 	return false
 }
 
-func (s *Filterer) filterState(stateFilter config.StateFilter, response *streamer.SuperNodePayload, payload ipfs.IPLDPayload) error {
+func (s *ResponseFilterer) filterState(stateFilter config.StateFilter, response *StreamPayload, payload *IPLDPayload) error {
 	if !stateFilter.Off {
 		response.StateNodesRlp = make(map[common.Hash][]byte)
 		keyFilters := make([]common.Hash, 0, len(stateFilter.Addresses))
 		for _, addr := range stateFilter.Addresses {
-			keyFilter := ipfs.AddressToKey(common.HexToAddress(addr))
+			keyFilter := AddressToKey(common.HexToAddress(addr))
 			keyFilters = append(keyFilters, keyFilter)
 		}
-		for key, stateNode := range payload.StateNodes {
+		for _, stateNode := range payload.StateNodes {
-			if checkNodeKeys(keyFilters, key) {
+			if checkNodeKeys(keyFilters, stateNode.Key) {
 				if stateNode.Leaf || stateFilter.IntermediateNodes {
-					response.StateNodesRlp[key] = stateNode.Value
+					response.StateNodesRlp[stateNode.Key] = stateNode.Value
 				}
 			}
 		}
@@ -220,17 +222,17 @@ func checkNodeKeys(wantedKeys []common.Hash, actualKey common.Hash) bool {
 	return false
 }
 
-func (s *Filterer) filterStorage(storageFilter config.StorageFilter, response *streamer.SuperNodePayload, payload ipfs.IPLDPayload) error {
+func (s *ResponseFilterer) filterStorage(storageFilter config.StorageFilter, response *StreamPayload, payload *IPLDPayload) error {
 	if !storageFilter.Off {
 		response.StorageNodesRlp = make(map[common.Hash]map[common.Hash][]byte)
 		stateKeyFilters := make([]common.Hash, 0, len(storageFilter.Addresses))
 		for _, addr := range storageFilter.Addresses {
-			keyFilter := ipfs.AddressToKey(common.HexToAddress(addr))
+			keyFilter := AddressToKey(common.HexToAddress(addr))
 			stateKeyFilters = append(stateKeyFilters, keyFilter)
 		}
 		storageKeyFilters := make([]common.Hash, 0, len(storageFilter.StorageKeys))
 		for _, store := range storageFilter.StorageKeys {
-			keyFilter := ipfs.HexToKey(store)
+			keyFilter := HexToKey(store)
 			storageKeyFilters = append(storageKeyFilters, keyFilter)
 		}
 		for stateKey, storageNodes := range payload.StorageNodes {

pkg/super_node/eth/filterer_test.go

@@ -14,21 +14,23 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package super_node_test
+package eth_test
 
 import (
 	"bytes"
 
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
+
 	"github.com/ethereum/go-ethereum/core/types"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	"github.com/vulcanize/vulcanizedb/pkg/ipfs/mocks"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
-	"github.com/vulcanize/vulcanizedb/pkg/super_node"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth/mocks"
 )
 
 var (
-	filterer                  super_node.ResponseFilterer
+	filterer                  *eth.ResponseFilterer
 	expectedRctForStorageRLP1 []byte
 	expectedRctForStorageRLP2 []byte
 )
@@ -36,34 +38,38 @@ var (
 var _ = Describe("Filterer", func() {
 	Describe("FilterResponse", func() {
 		BeforeEach(func() {
-			filterer = super_node.NewResponseFilterer()
+			filterer = eth.NewResponseFilterer()
 			expectedRctForStorageRLP1 = getReceiptForStorageRLP(mocks.MockReceipts, 0)
 			expectedRctForStorageRLP2 = getReceiptForStorageRLP(mocks.MockReceipts, 1)
 		})
 
-		It("Transcribes all the data from the IPLDPayload into the SuperNodePayload if given an open filter", func() {
+		It("Transcribes all the data from the IPLDPayload into the StreamPayload if given an open filter", func() {
-			superNodePayload, err := filterer.FilterResponse(openFilter, *mocks.MockIPLDPayload)
+			payload, err := filterer.Filter(openFilter, mocks.MockIPLDPayload)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(superNodePayload.BlockNumber.Int64()).To(Equal(mocks.MockSeeNodePayload.BlockNumber.Int64()))
+			superNodePayload, ok := payload.(eth.StreamPayload)
-			Expect(superNodePayload.HeadersRlp).To(Equal(mocks.MockSeeNodePayload.HeadersRlp))
+			Expect(ok).To(BeTrue())
+			Expect(superNodePayload.BlockNumber.Int64()).To(Equal(mocks.MockSeedNodePayload.BlockNumber.Int64()))
+			Expect(superNodePayload.HeadersRlp).To(Equal(mocks.MockSeedNodePayload.HeadersRlp))
 			var unclesRlp [][]byte
 			Expect(superNodePayload.UnclesRlp).To(Equal(unclesRlp))
 			Expect(len(superNodePayload.TransactionsRlp)).To(Equal(2))
-			Expect(super_node.ListContainsBytes(superNodePayload.TransactionsRlp, mocks.MockTransactions.GetRlp(0))).To(BeTrue())
+			Expect(shared.ListContainsBytes(superNodePayload.TransactionsRlp, mocks.MockTransactions.GetRlp(0))).To(BeTrue())
-			Expect(super_node.ListContainsBytes(superNodePayload.TransactionsRlp, mocks.MockTransactions.GetRlp(1))).To(BeTrue())
+			Expect(shared.ListContainsBytes(superNodePayload.TransactionsRlp, mocks.MockTransactions.GetRlp(1))).To(BeTrue())
 			Expect(len(superNodePayload.ReceiptsRlp)).To(Equal(2))
-			Expect(super_node.ListContainsBytes(superNodePayload.ReceiptsRlp, expectedRctForStorageRLP1)).To(BeTrue())
+			Expect(shared.ListContainsBytes(superNodePayload.ReceiptsRlp, expectedRctForStorageRLP1)).To(BeTrue())
-			Expect(super_node.ListContainsBytes(superNodePayload.ReceiptsRlp, expectedRctForStorageRLP2)).To(BeTrue())
+			Expect(shared.ListContainsBytes(superNodePayload.ReceiptsRlp, expectedRctForStorageRLP2)).To(BeTrue())
 			Expect(len(superNodePayload.StateNodesRlp)).To(Equal(2))
 			Expect(superNodePayload.StateNodesRlp[mocks.ContractLeafKey]).To(Equal(mocks.ValueBytes))
 			Expect(superNodePayload.StateNodesRlp[mocks.AnotherContractLeafKey]).To(Equal(mocks.AnotherValueBytes))
-			Expect(superNodePayload.StorageNodesRlp).To(Equal(mocks.MockSeeNodePayload.StorageNodesRlp))
+			Expect(superNodePayload.StorageNodesRlp).To(Equal(mocks.MockSeedNodePayload.StorageNodesRlp))
 		})
 
 		It("Applies filters from the provided config.Subscription", func() {
-			superNodePayload1, err := filterer.FilterResponse(rctContractFilter, *mocks.MockIPLDPayload)
+			payload1, err := filterer.Filter(rctContractFilter, mocks.MockIPLDPayload)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(superNodePayload1.BlockNumber.Int64()).To(Equal(mocks.MockSeeNodePayload.BlockNumber.Int64()))
+			superNodePayload1, ok := payload1.(eth.StreamPayload)
+			Expect(ok).To(BeTrue())
+			Expect(superNodePayload1.BlockNumber.Int64()).To(Equal(mocks.MockSeedNodePayload.BlockNumber.Int64()))
 			Expect(len(superNodePayload1.HeadersRlp)).To(Equal(0))
 			Expect(len(superNodePayload1.UnclesRlp)).To(Equal(0))
 			Expect(len(superNodePayload1.TransactionsRlp)).To(Equal(0))
@@ -72,9 +78,11 @@ var _ = Describe("Filterer", func() {
 			Expect(len(superNodePayload1.ReceiptsRlp)).To(Equal(1))
 			Expect(superNodePayload1.ReceiptsRlp[0]).To(Equal(expectedRctForStorageRLP2))
 
-			superNodePayload2, err := filterer.FilterResponse(rctTopicsFilter, *mocks.MockIPLDPayload)
+			payload2, err := filterer.Filter(rctTopicsFilter, mocks.MockIPLDPayload)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(superNodePayload2.BlockNumber.Int64()).To(Equal(mocks.MockSeeNodePayload.BlockNumber.Int64()))
+			superNodePayload2, ok := payload2.(eth.StreamPayload)
+			Expect(ok).To(BeTrue())
+			Expect(superNodePayload2.BlockNumber.Int64()).To(Equal(mocks.MockSeedNodePayload.BlockNumber.Int64()))
 			Expect(len(superNodePayload2.HeadersRlp)).To(Equal(0))
 			Expect(len(superNodePayload2.UnclesRlp)).To(Equal(0))
 			Expect(len(superNodePayload2.TransactionsRlp)).To(Equal(0))
@@ -83,9 +91,11 @@ var _ = Describe("Filterer", func() {
 			Expect(len(superNodePayload2.ReceiptsRlp)).To(Equal(1))
 			Expect(superNodePayload2.ReceiptsRlp[0]).To(Equal(expectedRctForStorageRLP1))
 
-			superNodePayload3, err := filterer.FilterResponse(rctTopicsAndContractFilter, *mocks.MockIPLDPayload)
+			payload3, err := filterer.Filter(rctTopicsAndContractFilter, mocks.MockIPLDPayload)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(superNodePayload3.BlockNumber.Int64()).To(Equal(mocks.MockSeeNodePayload.BlockNumber.Int64()))
+			superNodePayload3, ok := payload3.(eth.StreamPayload)
+			Expect(ok).To(BeTrue())
+			Expect(superNodePayload3.BlockNumber.Int64()).To(Equal(mocks.MockSeedNodePayload.BlockNumber.Int64()))
 			Expect(len(superNodePayload3.HeadersRlp)).To(Equal(0))
 			Expect(len(superNodePayload3.UnclesRlp)).To(Equal(0))
 			Expect(len(superNodePayload3.TransactionsRlp)).To(Equal(0))
@@ -94,9 +104,11 @@ var _ = Describe("Filterer", func() {
 			Expect(len(superNodePayload3.ReceiptsRlp)).To(Equal(1))
 			Expect(superNodePayload3.ReceiptsRlp[0]).To(Equal(expectedRctForStorageRLP1))
 
-			superNodePayload4, err := filterer.FilterResponse(rctContractsAndTopicFilter, *mocks.MockIPLDPayload)
+			payload4, err := filterer.Filter(rctContractsAndTopicFilter, mocks.MockIPLDPayload)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(superNodePayload4.BlockNumber.Int64()).To(Equal(mocks.MockSeeNodePayload.BlockNumber.Int64()))
+			superNodePayload4, ok := payload4.(eth.StreamPayload)
+			Expect(ok).To(BeTrue())
+			Expect(superNodePayload4.BlockNumber.Int64()).To(Equal(mocks.MockSeedNodePayload.BlockNumber.Int64()))
 			Expect(len(superNodePayload4.HeadersRlp)).To(Equal(0))
 			Expect(len(superNodePayload4.UnclesRlp)).To(Equal(0))
 			Expect(len(superNodePayload4.TransactionsRlp)).To(Equal(0))
@@ -105,35 +117,41 @@ var _ = Describe("Filterer", func() {
 			Expect(len(superNodePayload4.ReceiptsRlp)).To(Equal(1))
 			Expect(superNodePayload4.ReceiptsRlp[0]).To(Equal(expectedRctForStorageRLP2))
 
-			superNodePayload5, err := filterer.FilterResponse(rctsForAllCollectedTrxs, *mocks.MockIPLDPayload)
+			payload5, err := filterer.Filter(rctsForAllCollectedTrxs, mocks.MockIPLDPayload)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(superNodePayload5.BlockNumber.Int64()).To(Equal(mocks.MockSeeNodePayload.BlockNumber.Int64()))
+			superNodePayload5, ok := payload5.(eth.StreamPayload)
+			Expect(ok).To(BeTrue())
+			Expect(superNodePayload5.BlockNumber.Int64()).To(Equal(mocks.MockSeedNodePayload.BlockNumber.Int64()))
 			Expect(len(superNodePayload5.HeadersRlp)).To(Equal(0))
 			Expect(len(superNodePayload5.UnclesRlp)).To(Equal(0))
 			Expect(len(superNodePayload5.TransactionsRlp)).To(Equal(2))
-			Expect(super_node.ListContainsBytes(superNodePayload5.TransactionsRlp, mocks.MockTransactions.GetRlp(0))).To(BeTrue())
+			Expect(shared.ListContainsBytes(superNodePayload5.TransactionsRlp, mocks.MockTransactions.GetRlp(0))).To(BeTrue())
-			Expect(super_node.ListContainsBytes(superNodePayload5.TransactionsRlp, mocks.MockTransactions.GetRlp(1))).To(BeTrue())
+			Expect(shared.ListContainsBytes(superNodePayload5.TransactionsRlp, mocks.MockTransactions.GetRlp(1))).To(BeTrue())
 			Expect(len(superNodePayload5.StorageNodesRlp)).To(Equal(0))
 			Expect(len(superNodePayload5.StateNodesRlp)).To(Equal(0))
 			Expect(len(superNodePayload5.ReceiptsRlp)).To(Equal(2))
-			Expect(super_node.ListContainsBytes(superNodePayload5.ReceiptsRlp, expectedRctForStorageRLP1)).To(BeTrue())
+			Expect(shared.ListContainsBytes(superNodePayload5.ReceiptsRlp, expectedRctForStorageRLP1)).To(BeTrue())
-			Expect(super_node.ListContainsBytes(superNodePayload5.ReceiptsRlp, expectedRctForStorageRLP2)).To(BeTrue())
+			Expect(shared.ListContainsBytes(superNodePayload5.ReceiptsRlp, expectedRctForStorageRLP2)).To(BeTrue())
 
-			superNodePayload6, err := filterer.FilterResponse(rctsForSelectCollectedTrxs, *mocks.MockIPLDPayload)
+			payload6, err := filterer.Filter(rctsForSelectCollectedTrxs, mocks.MockIPLDPayload)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(superNodePayload6.BlockNumber.Int64()).To(Equal(mocks.MockSeeNodePayload.BlockNumber.Int64()))
+			superNodePayload6, ok := payload6.(eth.StreamPayload)
+			Expect(ok).To(BeTrue())
+			Expect(superNodePayload6.BlockNumber.Int64()).To(Equal(mocks.MockSeedNodePayload.BlockNumber.Int64()))
 			Expect(len(superNodePayload6.HeadersRlp)).To(Equal(0))
 			Expect(len(superNodePayload6.UnclesRlp)).To(Equal(0))
 			Expect(len(superNodePayload6.TransactionsRlp)).To(Equal(1))
-			Expect(super_node.ListContainsBytes(superNodePayload5.TransactionsRlp, mocks.MockTransactions.GetRlp(1))).To(BeTrue())
+			Expect(shared.ListContainsBytes(superNodePayload5.TransactionsRlp, mocks.MockTransactions.GetRlp(1))).To(BeTrue())
 			Expect(len(superNodePayload6.StorageNodesRlp)).To(Equal(0))
 			Expect(len(superNodePayload6.StateNodesRlp)).To(Equal(0))
 			Expect(len(superNodePayload6.ReceiptsRlp)).To(Equal(1))
 			Expect(superNodePayload4.ReceiptsRlp[0]).To(Equal(expectedRctForStorageRLP2))
 
-			superNodePayload7, err := filterer.FilterResponse(stateFilter, *mocks.MockIPLDPayload)
+			payload7, err := filterer.Filter(stateFilter, mocks.MockIPLDPayload)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(superNodePayload7.BlockNumber.Int64()).To(Equal(mocks.MockSeeNodePayload.BlockNumber.Int64()))
+			superNodePayload7, ok := payload7.(eth.StreamPayload)
+			Expect(ok).To(BeTrue())
+			Expect(superNodePayload7.BlockNumber.Int64()).To(Equal(mocks.MockSeedNodePayload.BlockNumber.Int64()))
 			Expect(len(superNodePayload7.HeadersRlp)).To(Equal(0))
 			Expect(len(superNodePayload7.UnclesRlp)).To(Equal(0))
 			Expect(len(superNodePayload7.TransactionsRlp)).To(Equal(0))

pkg/super_node/eth/helpers.go

@@ -0,0 +1,33 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package eth
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/crypto"
+)
+
+// AddressToKey hashes an address
+func AddressToKey(address common.Address) common.Hash {
+	return crypto.Keccak256Hash(address[:])
+}
+
+// HexToKey hashes a hex (0x leading or not) string
+func HexToKey(hex string) common.Hash {
+	addr := common.FromHex(hex)
+	return crypto.Keccak256Hash(addr[:])
+}

pkg/super_node/eth/indexer.go

@@ -0,0 +1,150 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package eth
+
+import (
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/common"
+
+	"github.com/jmoiron/sqlx"
+	"github.com/lib/pq"
+	log "github.com/sirupsen/logrus"
+
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+)
+
+// Indexer satisfies the Indexer interface for ethereum
+type CIDIndexer struct {
+	db *postgres.DB
+}
+
+// NewCIDIndexer creates a new pointer to a Indexer which satisfies the CIDIndexer interface
+func NewCIDIndexer(db *postgres.DB) *CIDIndexer {
+	return &CIDIndexer{
+		db: db,
+	}
+}
+
+// Index indexes a cidPayload in Postgres
+func (in *CIDIndexer) Index(cids interface{}) error {
+	cidPayload, ok := cids.(*CIDPayload)
+	if !ok {
+		return fmt.Errorf("eth indexer expected cids type %T got %T", &CIDPayload{}, cids)
+	}
+	tx, err := in.db.Beginx()
+	if err != nil {
+		return err
+	}
+	headerID, err := in.indexHeaderCID(tx, cidPayload.HeaderCID)
+	if err != nil {
+		if err := tx.Rollback(); err != nil {
+			log.Error(err)
+		}
+		return err
+	}
+	for _, uncle := range cidPayload.UncleCIDs {
+		err := in.indexUncleCID(tx, uncle)
+		if err != nil {
+			if err := tx.Rollback(); err != nil {
+				log.Error(err)
+			}
+			return err
+		}
+	}
+	if err := in.indexTransactionAndReceiptCIDs(tx, cidPayload, headerID); err != nil {
+		if err := tx.Rollback(); err != nil {
+			log.Error(err)
+		}
+		return err
+	}
+	if err := in.indexStateAndStorageCIDs(tx, cidPayload, headerID); err != nil {
+		if err := tx.Rollback(); err != nil {
+			log.Error(err)
+		}
+		return err
+	}
+	return tx.Commit()
+}
+
+func (repo *CIDIndexer) indexHeaderCID(tx *sqlx.Tx, header HeaderModel) (int64, error) {
+	var headerID int64
+	err := tx.QueryRowx(`INSERT INTO public.header_cids (block_number, block_hash, parent_hash, cid, uncle, td) VALUES ($1, $2, $3, $4, $5, $6)
+								ON CONFLICT (block_number, block_hash) DO UPDATE SET (parent_hash, cid, uncle, td) = ($3, $4, $5, $6)
+								RETURNING id`,
+		header.BlockNumber, header.BlockHash, header.ParentHash, header.CID, false, header.TotalDifficulty).Scan(&headerID)
+	return headerID, err
+}
+
+func (in *CIDIndexer) indexUncleCID(tx *sqlx.Tx, uncle HeaderModel) error {
+	_, err := tx.Exec(`INSERT INTO public.header_cids (block_number, block_hash, parent_hash, cid, uncle) VALUES ($1, $2, $3, $4, $5)
+								ON CONFLICT (block_number, block_hash) DO UPDATE SET (parent_hash, cid, uncle) = ($3, $4, $5)`,
+		uncle.BlockNumber, uncle.BlockHash, uncle.ParentHash, uncle.CID, true)
+	return err
+}
+
+func (in *CIDIndexer) indexTransactionAndReceiptCIDs(tx *sqlx.Tx, payload *CIDPayload, headerID int64) error {
+	for _, trxCidMeta := range payload.TransactionCIDs {
+		var txID int64
+		err := tx.QueryRowx(`INSERT INTO public.transaction_cids (header_id, tx_hash, cid, dst, src) VALUES ($1, $2, $3, $4, $5)
+									ON CONFLICT (header_id, tx_hash) DO UPDATE SET (cid, dst, src) = ($3, $4, $5)
+									RETURNING id`,
+			headerID, trxCidMeta.TxHash, trxCidMeta.CID, trxCidMeta.Dst, trxCidMeta.Src).Scan(&txID)
+		if err != nil {
+			return err
+		}
+		receiptCidMeta, ok := payload.ReceiptCIDs[common.HexToHash(trxCidMeta.TxHash)]
+		if ok {
+			if err := in.indexReceiptCID(tx, receiptCidMeta, txID); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (in *CIDIndexer) indexReceiptCID(tx *sqlx.Tx, cidMeta ReceiptModel, txID int64) error {
+	_, err := tx.Exec(`INSERT INTO public.receipt_cids (tx_id, cid, contract, topic0s) VALUES ($1, $2, $3, $4)`,
+		txID, cidMeta.CID, cidMeta.Contract, pq.Array(cidMeta.Topic0s))
+	return err
+}
+
+func (in *CIDIndexer) indexStateAndStorageCIDs(tx *sqlx.Tx, payload *CIDPayload, headerID int64) error {
+	for _, stateCID := range payload.StateNodeCIDs {
+		var stateID int64
+		err := tx.QueryRowx(`INSERT INTO public.state_cids (header_id, state_key, cid, leaf) VALUES ($1, $2, $3, $4)
+									ON CONFLICT (header_id, state_key) DO UPDATE SET (cid, leaf) = ($3, $4)
+									RETURNING id`,
+			headerID, stateCID.StateKey, stateCID.CID, stateCID.Leaf).Scan(&stateID)
+		if err != nil {
+			return err
+		}
+		for _, storageCID := range payload.StorageNodeCIDs[common.HexToHash(stateCID.StateKey)] {
+			if err := in.indexStorageCID(tx, storageCID, stateID); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (in *CIDIndexer) indexStorageCID(tx *sqlx.Tx, storageCID StorageNodeModel, stateID int64) error {
+	_, err := tx.Exec(`INSERT INTO public.storage_cids (state_id, storage_key, cid, leaf) VALUES ($1, $2, $3, $4) 
+								   ON CONFLICT (state_id, storage_key) DO UPDATE SET (cid, leaf) = ($3, $4)`,
+		stateID, storageCID.StorageKey, storageCID.CID, storageCID.Leaf)
+	return err
+}

pkg/super_node/eth/indexer_test.go

@@ -14,30 +14,32 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package super_node_test
+package eth_test
 
 import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
+
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
-	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
-	"github.com/vulcanize/vulcanizedb/pkg/ipfs/mocks"
+	eth2 "github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
-	"github.com/vulcanize/vulcanizedb/pkg/super_node"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth/mocks"
 )
 
-var _ = Describe("Repository", func() {
+var _ = Describe("Indexer", func() {
 	var (
 		db   *postgres.DB
 		err  error
-		repo super_node.CIDRepository
+		repo *eth2.CIDIndexer
 	)
 	BeforeEach(func() {
-		db, err = super_node.SetupDB()
+		db, err = eth.SetupDB()
 		Expect(err).ToNot(HaveOccurred())
-		repo = super_node.NewCIDRepository(db)
+		repo = eth2.NewCIDIndexer(db)
 	})
 	AfterEach(func() {
-		super_node.TearDownDB(db)
+		eth.TearDownDB(db)
 	})
 
 	Describe("Index", func() {
@@ -63,8 +65,8 @@ var _ = Describe("Repository", func() {
 			err = db.Select(&trxs, pgStr, 1)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(trxs)).To(Equal(2))
-			Expect(super_node.ListContainsString(trxs, "mockTrxCID1")).To(BeTrue())
+			Expect(shared.ListContainsString(trxs, "mockTrxCID1")).To(BeTrue())
-			Expect(super_node.ListContainsString(trxs, "mockTrxCID2")).To(BeTrue())
+			Expect(shared.ListContainsString(trxs, "mockTrxCID2")).To(BeTrue())
 			// check receipts were properly indexed
 			rcts := make([]string, 0)
 			pgStr = `SELECT receipt_cids.cid FROM receipt_cids, transaction_cids, header_cids
@@ -74,10 +76,10 @@ var _ = Describe("Repository", func() {
 			err = db.Select(&rcts, pgStr, 1)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(rcts)).To(Equal(2))
-			Expect(super_node.ListContainsString(rcts, "mockRctCID1")).To(BeTrue())
+			Expect(shared.ListContainsString(rcts, "mockRctCID1")).To(BeTrue())
-			Expect(super_node.ListContainsString(rcts, "mockRctCID2")).To(BeTrue())
+			Expect(shared.ListContainsString(rcts, "mockRctCID2")).To(BeTrue())
 			// check that state nodes were properly indexed
-			stateNodes := make([]ipfs.StateNodeCID, 0)
+			stateNodes := make([]eth.StateNodeModel, 0)
 			pgStr = `SELECT state_cids.cid, state_cids.state_key, state_cids.leaf FROM state_cids INNER JOIN header_cids ON (state_cids.header_id = header_cids.id)
 				WHERE header_cids.block_number = $1`
 			err = db.Select(&stateNodes, pgStr, 1)
@@ -86,15 +88,15 @@ var _ = Describe("Repository", func() {
 			for _, stateNode := range stateNodes {
 				if stateNode.CID == "mockStateCID1" {
 					Expect(stateNode.Leaf).To(Equal(true))
-					Expect(stateNode.Key).To(Equal(mocks.ContractLeafKey.Hex()))
+					Expect(stateNode.StateKey).To(Equal(mocks.ContractLeafKey.Hex()))
 				}
 				if stateNode.CID == "mockStateCID2" {
 					Expect(stateNode.Leaf).To(Equal(true))
-					Expect(stateNode.Key).To(Equal(mocks.AnotherContractLeafKey.Hex()))
+					Expect(stateNode.StateKey).To(Equal(mocks.AnotherContractLeafKey.Hex()))
 				}
 			}
 			// check that storage nodes were properly indexed
-			storageNodes := make([]ipfs.StorageNodeCID, 0)
+			storageNodes := make([]eth.StorageNodeWithStateKeyModel, 0)
 			pgStr = `SELECT storage_cids.cid, state_cids.state_key, storage_cids.storage_key, storage_cids.leaf FROM storage_cids, state_cids, header_cids
 				WHERE storage_cids.state_id = state_cids.id 
 				AND state_cids.header_id = header_cids.id
@@ -102,11 +104,11 @@ var _ = Describe("Repository", func() {
 			err = db.Select(&storageNodes, pgStr, 1)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(storageNodes)).To(Equal(1))
-			Expect(storageNodes[0]).To(Equal(ipfs.StorageNodeCID{
+			Expect(storageNodes[0]).To(Equal(eth.StorageNodeWithStateKeyModel{
-				CID:      "mockStorageCID",
+				CID:        "mockStorageCID",
-				Leaf:     true,
+				Leaf:       true,
-				Key:      "0x0000000000000000000000000000000000000000000000000000000000000001",
+				StorageKey: "0x0000000000000000000000000000000000000000000000000000000000000001",
-				StateKey: mocks.ContractLeafKey.Hex(),
+				StateKey:   mocks.ContractLeafKey.Hex(),
 			}))
 		})
 	})

pkg/super_node/eth/ipld_fetcher.go

@@ -14,78 +14,73 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package ipfs
+package eth
 
 import (
 	"context"
 	"errors"
+	"fmt"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ipfs/go-block-format"
 	"github.com/ipfs/go-blockservice"
 	"github.com/ipfs/go-cid"
 	log "github.com/sirupsen/logrus"
+
+	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
 )
 
 var (
 	errUnexpectedNumberOfIPLDs = errors.New("ipfs batch fetch returned unexpected number of IPLDs")
 )
 
-// IPLDFetcher is an interface for fetching IPLDs
+// IPLDFetcher satisfies the IPLDFetcher interface for ethereum
-type IPLDFetcher interface {
+type IPLDFetcher struct {
-	FetchIPLDs(cids CIDWrapper) (*IPLDWrapper, error)
-	FetchHeaders(cids []string) ([]blocks.Block, error)
-	FetchUncles(cids []string) ([]blocks.Block, error)
-	FetchTrxs(cids []string) ([]blocks.Block, error)
-	FetchRcts(cids []string) ([]blocks.Block, error)
-	FetchState(cids []StateNodeCID) (map[common.Hash]blocks.Block, error)
-	FetchStorage(cids []StorageNodeCID) (map[common.Hash]map[common.Hash]blocks.Block, error)
-}
-
-// EthIPLDFetcher is used to fetch ETH IPLD objects from IPFS
-type EthIPLDFetcher struct {
 	BlockService blockservice.BlockService
 }
 
 // NewIPLDFetcher creates a pointer to a new IPLDFetcher
-func NewIPLDFetcher(ipfsPath string) (*EthIPLDFetcher, error) {
+func NewIPLDFetcher(ipfsPath string) (*IPLDFetcher, error) {
-	blockService, err := InitIPFSBlockService(ipfsPath)
+	blockService, err := ipfs.InitIPFSBlockService(ipfsPath)
 	if err != nil {
 		return nil, err
 	}
-	return &EthIPLDFetcher{
+	return &IPLDFetcher{
 		BlockService: blockService,
 	}, nil
 }
 
-// FetchIPLDs is the exported method for fetching and returning all the IPLDS specified in the CIDWrapper
+// Fetch is the exported method for fetching and returning all the IPLDS specified in the CIDWrapper
-func (f *EthIPLDFetcher) FetchIPLDs(cids CIDWrapper) (*IPLDWrapper, error) {
+func (f *IPLDFetcher) Fetch(cids interface{}) (interface{}, error) {
-
+	cidWrapper, ok := cids.(*CIDWrapper)
+	if !ok {
+		return nil, fmt.Errorf("eth fetcher: expected cids type %T got %T", &CIDWrapper{}, cids)
+	}
 	log.Debug("fetching iplds")
 	iplds := new(IPLDWrapper)
-	iplds.BlockNumber = cids.BlockNumber
+	iplds.BlockNumber = cidWrapper.BlockNumber
 	var err error
-	iplds.Headers, err = f.FetchHeaders(cids.Headers)
+	iplds.Headers, err = f.FetchHeaders(cidWrapper.Headers)
 	if err != nil {
 		return nil, err
 	}
-	iplds.Uncles, err = f.FetchUncles(cids.Uncles)
+	iplds.Uncles, err = f.FetchUncles(cidWrapper.Uncles)
 	if err != nil {
 		return nil, err
 	}
-	iplds.Transactions, err = f.FetchTrxs(cids.Transactions)
+	iplds.Transactions, err = f.FetchTrxs(cidWrapper.Transactions)
 	if err != nil {
 		return nil, err
 	}
-	iplds.Receipts, err = f.FetchRcts(cids.Receipts)
+	iplds.Receipts, err = f.FetchRcts(cidWrapper.Receipts)
 	if err != nil {
 		return nil, err
 	}
-	iplds.StateNodes, err = f.FetchState(cids.StateNodes)
+	iplds.StateNodes, err = f.FetchState(cidWrapper.StateNodes)
 	if err != nil {
 		return nil, err
 	}
-	iplds.StorageNodes, err = f.FetchStorage(cids.StorageNodes)
+	iplds.StorageNodes, err = f.FetchStorage(cidWrapper.StorageNodes)
 	if err != nil {
 		return nil, err
 	}
@@ -94,11 +89,11 @@ func (f *EthIPLDFetcher) FetchIPLDs(cids CIDWrapper) (*IPLDWrapper, error) {
 
 // FetchHeaders fetches headers
 // It uses the f.fetchBatch method
-func (f *EthIPLDFetcher) FetchHeaders(cids []string) ([]blocks.Block, error) {
+func (f *IPLDFetcher) FetchHeaders(cids []HeaderModel) ([]blocks.Block, error) {
 	log.Debug("fetching header iplds")
 	headerCids := make([]cid.Cid, 0, len(cids))
 	for _, c := range cids {
-		dc, err := cid.Decode(c)
+		dc, err := cid.Decode(c.CID)
 		if err != nil {
 			return nil, err
 		}
@@ -114,11 +109,11 @@ func (f *EthIPLDFetcher) FetchHeaders(cids []string) ([]blocks.Block, error) {
 
 // FetchUncles fetches uncles
 // It uses the f.fetchBatch method
-func (f *EthIPLDFetcher) FetchUncles(cids []string) ([]blocks.Block, error) {
+func (f *IPLDFetcher) FetchUncles(cids []HeaderModel) ([]blocks.Block, error) {
 	log.Debug("fetching uncle iplds")
 	uncleCids := make([]cid.Cid, 0, len(cids))
 	for _, c := range cids {
-		dc, err := cid.Decode(c)
+		dc, err := cid.Decode(c.CID)
 		if err != nil {
 			return nil, err
 		}
@@ -134,11 +129,11 @@ func (f *EthIPLDFetcher) FetchUncles(cids []string) ([]blocks.Block, error) {
 
 // FetchTrxs fetches transactions
 // It uses the f.fetchBatch method
-func (f *EthIPLDFetcher) FetchTrxs(cids []string) ([]blocks.Block, error) {
+func (f *IPLDFetcher) FetchTrxs(cids []TxModel) ([]blocks.Block, error) {
 	log.Debug("fetching transaction iplds")
 	trxCids := make([]cid.Cid, 0, len(cids))
 	for _, c := range cids {
-		dc, err := cid.Decode(c)
+		dc, err := cid.Decode(c.CID)
 		if err != nil {
 			return nil, err
 		}
@@ -154,11 +149,11 @@ func (f *EthIPLDFetcher) FetchTrxs(cids []string) ([]blocks.Block, error) {
 
 // FetchRcts fetches receipts
 // It uses the f.fetchBatch method
-func (f *EthIPLDFetcher) FetchRcts(cids []string) ([]blocks.Block, error) {
+func (f *IPLDFetcher) FetchRcts(cids []ReceiptModel) ([]blocks.Block, error) {
 	log.Debug("fetching receipt iplds")
 	rctCids := make([]cid.Cid, 0, len(cids))
 	for _, c := range cids {
-		dc, err := cid.Decode(c)
+		dc, err := cid.Decode(c.CID)
 		if err != nil {
 			return nil, err
 		}
@@ -175,11 +170,11 @@ func (f *EthIPLDFetcher) FetchRcts(cids []string) ([]blocks.Block, error) {
 // FetchState fetches state nodes
 // It uses the single f.fetch method instead of the batch fetch, because it
 // needs to maintain the data's relation to state keys
-func (f *EthIPLDFetcher) FetchState(cids []StateNodeCID) (map[common.Hash]blocks.Block, error) {
+func (f *IPLDFetcher) FetchState(cids []StateNodeModel) (map[common.Hash]blocks.Block, error) {
 	log.Debug("fetching state iplds")
 	stateNodes := make(map[common.Hash]blocks.Block)
 	for _, stateNode := range cids {
-		if stateNode.CID == "" || stateNode.Key == "" {
+		if stateNode.CID == "" || stateNode.StateKey == "" {
 			continue
 		}
 		dc, err := cid.Decode(stateNode.CID)
@@ -190,7 +185,7 @@ func (f *EthIPLDFetcher) FetchState(cids []StateNodeCID) (map[common.Hash]blocks
 		if err != nil {
 			return nil, err
 		}
-		stateNodes[common.HexToHash(stateNode.Key)] = state
+		stateNodes[common.HexToHash(stateNode.StateKey)] = state
 	}
 	return stateNodes, nil
 }
@@ -198,11 +193,11 @@ func (f *EthIPLDFetcher) FetchState(cids []StateNodeCID) (map[common.Hash]blocks
 // FetchStorage fetches storage nodes
 // It uses the single f.fetch method instead of the batch fetch, because it
 // needs to maintain the data's relation to state and storage keys
-func (f *EthIPLDFetcher) FetchStorage(cids []StorageNodeCID) (map[common.Hash]map[common.Hash]blocks.Block, error) {
+func (f *IPLDFetcher) FetchStorage(cids []StorageNodeWithStateKeyModel) (map[common.Hash]map[common.Hash]blocks.Block, error) {
 	log.Debug("fetching storage iplds")
 	storageNodes := make(map[common.Hash]map[common.Hash]blocks.Block)
 	for _, storageNode := range cids {
-		if storageNode.CID == "" || storageNode.Key == "" || storageNode.StateKey == "" {
+		if storageNode.CID == "" || storageNode.StorageKey == "" || storageNode.StateKey == "" {
 			continue
 		}
 		dc, err := cid.Decode(storageNode.CID)
@@ -216,20 +211,20 @@ func (f *EthIPLDFetcher) FetchStorage(cids []StorageNodeCID) (map[common.Hash]ma
 		if storageNodes[common.HexToHash(storageNode.StateKey)] == nil {
 			storageNodes[common.HexToHash(storageNode.StateKey)] = make(map[common.Hash]blocks.Block)
 		}
-		storageNodes[common.HexToHash(storageNode.StateKey)][common.HexToHash(storageNode.Key)] = storage
+		storageNodes[common.HexToHash(storageNode.StateKey)][common.HexToHash(storageNode.StorageKey)] = storage
 	}
 	return storageNodes, nil
 }
 
 // fetch is used to fetch a single cid
-func (f *EthIPLDFetcher) fetch(cid cid.Cid) (blocks.Block, error) {
+func (f *IPLDFetcher) fetch(cid cid.Cid) (blocks.Block, error) {
 	return f.BlockService.GetBlock(context.Background(), cid)
 }
 
 // fetchBatch is used to fetch a batch of IPFS data blocks by cid
 // There is no guarantee all are fetched, and no error in such a case, so
 // downstream we will need to confirm which CIDs were fetched in the result set
-func (f *EthIPLDFetcher) fetchBatch(cids []cid.Cid) []blocks.Block {
+func (f *IPLDFetcher) fetchBatch(cids []cid.Cid) []blocks.Block {
 	fetchedBlocks := make([]blocks.Block, 0, len(cids))
 	blockChan := f.BlockService.GetBlocks(context.Background(), cids)
 	for block := range blockChan {

pkg/super_node/eth/ipld_fetcher_test.go

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package ipfs_test
+package eth_test
 
 import (
 	"math/big"
@@ -24,8 +24,8 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
-	"github.com/vulcanize/vulcanizedb/pkg/ipfs/mocks"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth/mocks"
 )
 
 var (
@@ -45,28 +45,44 @@ var (
 	mockStorageBlock2 = blocks.NewBlock(mockStorageData2)
 	mockBlocks        = []blocks.Block{mockHeaderBlock, mockUncleBlock, mockTrxBlock, mockReceiptBlock, mockStateBlock, mockStorageBlock1, mockStorageBlock2}
 	mockBlockService  *mocks.MockIPFSBlockService
-	mockCIDWrapper    = ipfs.CIDWrapper{
+	mockCIDWrapper    = &eth.CIDWrapper{
-		BlockNumber:  big.NewInt(9000),
+		BlockNumber: big.NewInt(9000),
-		Headers:      []string{mockHeaderBlock.Cid().String()},
+		Headers: []eth.HeaderModel{
-		Uncles:       []string{mockUncleBlock.Cid().String()},
+			{
-		Transactions: []string{mockTrxBlock.Cid().String()},
+				CID: mockHeaderBlock.Cid().String(),
-		Receipts:     []string{mockReceiptBlock.Cid().String()},
+			},
-		StateNodes: []ipfs.StateNodeCID{{
+		},
-			CID:  mockStateBlock.Cid().String(),
+		Uncles: []eth.HeaderModel{
-			Leaf: true,
+			{
-			Key:  "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470",
+				CID: mockUncleBlock.Cid().String(),
-		}},
+			},
-		StorageNodes: []ipfs.StorageNodeCID{{
+		},
-			CID:      mockStorageBlock1.Cid().String(),
+		Transactions: []eth.TxModel{
+			{
+				CID: mockTrxBlock.Cid().String(),
+			},
+		},
+		Receipts: []eth.ReceiptModel{
+			{
+				CID: mockReceiptBlock.Cid().String(),
+			},
+		},
+		StateNodes: []eth.StateNodeModel{{
+			CID:      mockStateBlock.Cid().String(),
 			Leaf:     true,
 			StateKey: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470",
-			Key:      "0000000000000000000000000000000000000000000000000000000000000001",
+		}},
+		StorageNodes: []eth.StorageNodeWithStateKeyModel{{
+			CID:        mockStorageBlock1.Cid().String(),
+			Leaf:       true,
+			StateKey:   "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470",
+			StorageKey: "0000000000000000000000000000000000000000000000000000000000000001",
 		},
 			{
-				CID:      mockStorageBlock2.Cid().String(),
+				CID:        mockStorageBlock2.Cid().String(),
-				Leaf:     true,
+				Leaf:       true,
-				StateKey: "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470",
+				StateKey:   "0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470",
-				Key:      "0000000000000000000000000000000000000000000000000000000000000002",
+				StorageKey: "0000000000000000000000000000000000000000000000000000000000000002",
 			}},
 	}
 )
@@ -81,10 +97,12 @@ var _ = Describe("Fetcher", func() {
 		})
 
 		It("Fetches and returns IPLDs for the CIDs provided in the CIDWrapper", func() {
-			fetcher := new(ipfs.EthIPLDFetcher)
+			fetcher := new(eth.IPLDFetcher)
 			fetcher.BlockService = mockBlockService
-			iplds, err := fetcher.FetchIPLDs(mockCIDWrapper)
+			i, err := fetcher.Fetch(mockCIDWrapper)
 			Expect(err).ToNot(HaveOccurred())
+			iplds, ok := i.(*eth.IPLDWrapper)
+			Expect(ok).To(BeTrue())
 			Expect(iplds.BlockNumber).To(Equal(mockCIDWrapper.BlockNumber))
 			Expect(len(iplds.Headers)).To(Equal(1))
 			Expect(iplds.Headers[0]).To(Equal(mockHeaderBlock))

pkg/super_node/eth/mocks/converter.go

@@ -21,33 +21,41 @@ import (
 
 	"github.com/ethereum/go-ethereum/statediff"
 
-	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
 )
 
 // PayloadConverter is the underlying struct for the Converter interface
 type PayloadConverter struct {
 	PassedStatediffPayload statediff.Payload
-	ReturnIPLDPayload      *ipfs.IPLDPayload
+	ReturnIPLDPayload      *eth.IPLDPayload
 	ReturnErr              error
 }
 
 // Convert method is used to convert a geth statediff.Payload to a IPLDPayload
-func (pc *PayloadConverter) Convert(payload statediff.Payload) (*ipfs.IPLDPayload, error) {
+func (pc *PayloadConverter) Convert(payload interface{}) (interface{}, error) {
-	pc.PassedStatediffPayload = payload
+	stateDiffPayload, ok := payload.(statediff.Payload)
+	if !ok {
+		return nil, fmt.Errorf("convert expected payload type %T got %T", statediff.Payload{}, payload)
+	}
+	pc.PassedStatediffPayload = stateDiffPayload
 	return pc.ReturnIPLDPayload, pc.ReturnErr
 }
 
 // IterativePayloadConverter is the underlying struct for the Converter interface
 type IterativePayloadConverter struct {
 	PassedStatediffPayload []statediff.Payload
-	ReturnIPLDPayload      []*ipfs.IPLDPayload
+	ReturnIPLDPayload      []*eth.IPLDPayload
 	ReturnErr              error
 	iteration              int
 }
 
 // Convert method is used to convert a geth statediff.Payload to a IPLDPayload
-func (pc *IterativePayloadConverter) Convert(payload statediff.Payload) (*ipfs.IPLDPayload, error) {
+func (pc *IterativePayloadConverter) Convert(payload interface{}) (interface{}, error) {
-	pc.PassedStatediffPayload = append(pc.PassedStatediffPayload, payload)
+	stateDiffPayload, ok := payload.(statediff.Payload)
+	if !ok {
+		return nil, fmt.Errorf("convert expected payload type %T got %T", statediff.Payload{}, payload)
+	}
+	pc.PassedStatediffPayload = append(pc.PassedStatediffPayload, stateDiffPayload)
 	if len(pc.PassedStatediffPayload) < pc.iteration+1 {
 		return nil, fmt.Errorf("IterativePayloadConverter does not have a payload to return at iteration %d", pc.iteration)
 	}

pkg/super_node/eth/mocks/fetcher.go

@@ -0,0 +1,50 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package mocks
+
+import (
+	"errors"
+	"sync/atomic"
+
+	"github.com/ethereum/go-ethereum/statediff"
+)
+
+// StateDiffFetcher mock for tests
+type StateDiffFetcher struct {
+	PayloadsToReturn     map[uint64]statediff.Payload
+	FetchErrs            map[uint64]error
+	CalledAtBlockHeights [][]uint64
+	CalledTimes          int64
+}
+
+// FetchStateDiffsAt mock method
+func (fetcher *StateDiffFetcher) FetchAt(blockHeights []uint64) ([]interface{}, error) {
+	if fetcher.PayloadsToReturn == nil {
+		return nil, errors.New("mock StateDiffFetcher needs to be initialized with payloads to return")
+	}
+	atomic.AddInt64(&fetcher.CalledTimes, 1) // thread-safe increment
+	fetcher.CalledAtBlockHeights = append(fetcher.CalledAtBlockHeights, blockHeights)
+	results := make([]interface{}, 0, len(blockHeights))
+	for _, height := range blockHeights {
+		results = append(results, fetcher.PayloadsToReturn[height])
+		err, ok := fetcher.FetchErrs[height]
+		if ok && err != nil {
+			return nil, err
+		}
+	}
+	return results, nil
+}

pkg/super_node/eth/mocks/indexer.go

@@ -16,16 +16,24 @@
 
 package mocks
 
-import "github.com/vulcanize/vulcanizedb/pkg/ipfs"
+import (
+	"fmt"
 
-// CIDRepository is the underlying struct for the Repository interface
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
-type CIDRepository struct {
+)
-	PassedCIDPayload []*ipfs.CIDPayload
+
+// CIDIndexer is the underlying struct for the Indexer interface
+type CIDIndexer struct {
+	PassedCIDPayload []*eth.CIDPayload
 	ReturnErr        error
 }
 
 // Index indexes a cidPayload in Postgres
-func (repo *CIDRepository) Index(cidPayload *ipfs.CIDPayload) error {
+func (repo *CIDIndexer) Index(cids interface{}) error {
+	cidPayload, ok := cids.(*eth.CIDPayload)
+	if !ok {
+		return fmt.Errorf("index expected cids type %T got %T", &eth.CIDPayload{}, cids)
+	}
 	repo.PassedCIDPayload = append(repo.PassedCIDPayload, cidPayload)
 	return repo.ReturnErr
 }

pkg/super_node/eth/mocks/publisher.go

@@ -19,33 +19,41 @@ package mocks
 import (
 	"fmt"
 
-	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
 )
 
 // IPLDPublisher is the underlying struct for the Publisher interface
 type IPLDPublisher struct {
-	PassedIPLDPayload *ipfs.IPLDPayload
+	PassedIPLDPayload *eth.IPLDPayload
-	ReturnCIDPayload  *ipfs.CIDPayload
+	ReturnCIDPayload  *eth.CIDPayload
 	ReturnErr         error
 }
 
 // Publish publishes an IPLDPayload to IPFS and returns the corresponding CIDPayload
-func (pub *IPLDPublisher) Publish(payload *ipfs.IPLDPayload) (*ipfs.CIDPayload, error) {
+func (pub *IPLDPublisher) Publish(payload interface{}) (interface{}, error) {
-	pub.PassedIPLDPayload = payload
+	ipldPayload, ok := payload.(*eth.IPLDPayload)
+	if !ok {
+		return nil, fmt.Errorf("publish expected payload type %T got %T", &eth.IPLDPayload{}, payload)
+	}
+	pub.PassedIPLDPayload = ipldPayload
 	return pub.ReturnCIDPayload, pub.ReturnErr
 }
 
 // IterativeIPLDPublisher is the underlying struct for the Publisher interface; used in testing
 type IterativeIPLDPublisher struct {
-	PassedIPLDPayload []*ipfs.IPLDPayload
+	PassedIPLDPayload []*eth.IPLDPayload
-	ReturnCIDPayload  []*ipfs.CIDPayload
+	ReturnCIDPayload  []*eth.CIDPayload
 	ReturnErr         error
 	iteration         int
 }
 
 // Publish publishes an IPLDPayload to IPFS and returns the corresponding CIDPayload
-func (pub *IterativeIPLDPublisher) Publish(payload *ipfs.IPLDPayload) (*ipfs.CIDPayload, error) {
+func (pub *IterativeIPLDPublisher) Publish(payload interface{}) (interface{}, error) {
-	pub.PassedIPLDPayload = append(pub.PassedIPLDPayload, payload)
+	ipldPayload, ok := payload.(*eth.IPLDPayload)
+	if !ok {
+		return nil, fmt.Errorf("publish expected payload type %T got %T", &eth.IPLDPayload{}, payload)
+	}
+	pub.PassedIPLDPayload = append(pub.PassedIPLDPayload, ipldPayload)
 	if len(pub.ReturnCIDPayload) < pub.iteration+1 {
 		return nil, fmt.Errorf("IterativeIPLDPublisher does not have a payload to return at iteration %d", pub.iteration)
 	}

pkg/super_node/eth/mocks/retriever.go

@@ -0,0 +1,64 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package mocks
+
+import (
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
+)
+
+// MockCIDRetriever is a mock CID retriever for use in tests
+type MockCIDRetriever struct {
+	GapsToRetrieve              []shared.Gap
+	GapsToRetrieveErr           error
+	CalledTimes                 int
+	FirstBlockNumberToReturn    int64
+	RetrieveFirstBlockNumberErr error
+}
+
+// RetrieveCIDs mock method
+func (*MockCIDRetriever) Retrieve(filter interface{}, blockNumber int64) (interface{}, bool, error) {
+	panic("implement me")
+}
+
+// RetrieveLastBlockNumber mock method
+func (*MockCIDRetriever) RetrieveLastBlockNumber() (int64, error) {
+	panic("implement me")
+}
+
+// RetrieveFirstBlockNumber mock method
+func (mcr *MockCIDRetriever) RetrieveFirstBlockNumber() (int64, error) {
+	return mcr.FirstBlockNumberToReturn, mcr.RetrieveFirstBlockNumberErr
+}
+
+// RetrieveGapsInData mock method
+func (mcr *MockCIDRetriever) RetrieveGapsInData() ([]shared.Gap, error) {
+	mcr.CalledTimes++
+	return mcr.GapsToRetrieve, mcr.GapsToRetrieveErr
+}
+
+// SetGapsToRetrieve mock method
+func (mcr *MockCIDRetriever) SetGapsToRetrieve(gaps []shared.Gap) {
+	if mcr.GapsToRetrieve == nil {
+		mcr.GapsToRetrieve = make([]shared.Gap, 0)
+	}
+	mcr.GapsToRetrieve = append(mcr.GapsToRetrieve, gaps...)
+}
+
+func (mcr *MockCIDRetriever) Database() *postgres.DB {
+	panic("implement me")
+}

pkg/super_node/eth/mocks/streamer.go

@@ -0,0 +1,43 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package mocks
+
+import (
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/statediff"
+)
+
+// StateDiffStreamer is the underlying struct for the Streamer interface
+type StateDiffStreamer struct {
+	PassedPayloadChan chan interface{}
+	ReturnSub         *rpc.ClientSubscription
+	ReturnErr         error
+	StreamPayloads    []statediff.Payload
+}
+
+// Stream is the main loop for subscribing to data from the Geth state diff process
+func (sds *StateDiffStreamer) Stream(payloadChan chan interface{}) (*rpc.ClientSubscription, error) {
+	sds.PassedPayloadChan = payloadChan
+
+	go func() {
+		for _, payload := range sds.StreamPayloads {
+			sds.PassedPayloadChan <- payload
+		}
+	}()
+
+	return sds.ReturnSub, sds.ReturnErr
+}

pkg/super_node/eth/mocks/test_data.go

@@ -23,24 +23,23 @@ import (
 	"math/big"
 	rand2 "math/rand"
 
-	"github.com/ipfs/go-block-format"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/streamer"
-
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/statediff"
+	"github.com/ipfs/go-block-format"
 	log "github.com/sirupsen/logrus"
 
-	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
+	eth2 "github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
 )
 
 // Test variables
 var (
 	// block data
-	BlockNumber = big.NewInt(rand2.Int63())
+	BlockNumber = big.NewInt(1)
 	MockHeader  = types.Header{
 		Time:        0,
 		Number:      BlockNumber,
@@ -53,7 +52,7 @@ var (
 	MockBlock                                  = types.NewBlock(&MockHeader, MockTransactions, nil, MockReceipts)
 	MockBlockRlp, _                            = rlp.EncodeToBytes(MockBlock)
 	MockHeaderRlp, _                           = rlp.EncodeToBytes(MockBlock.Header())
-	MockTrxMeta                                = []*ipfs.TrxMetaData{
+	MockTrxMeta                                = []eth.TxModel{
 		{
 			CID: "", // This is empty until we go to publish to ipfs
 			Src: senderAddr.Hex(),
@@ -65,20 +64,20 @@ var (
 			Dst: "0x0000000000000000000000000000000000000001",
 		},
 	}
-	MockRctMeta = []*ipfs.ReceiptMetaData{
+	MockRctMeta = []eth.ReceiptModel{
 		{
 			CID: "",
 			Topic0s: []string{
 				"0x0000000000000000000000000000000000000000000000000000000000000004",
 			},
-			ContractAddress: "0x0000000000000000000000000000000000000000",
+			Contract: "0x0000000000000000000000000000000000000000",
 		},
 		{
 			CID: "",
 			Topic0s: []string{
 				"0x0000000000000000000000000000000000000000000000000000000000000005",
 			},
-			ContractAddress: "0x0000000000000000000000000000000000000001",
+			Contract: "0x0000000000000000000000000000000000000001",
 		},
 	}
 
@@ -101,9 +100,9 @@ var (
 	}}
 	emptyStorage           = make([]statediff.StorageDiff, 0)
 	Address                = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592")
-	ContractLeafKey        = ipfs.AddressToKey(Address)
+	ContractLeafKey        = eth.AddressToKey(Address)
 	AnotherAddress         = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476593")
-	AnotherContractLeafKey = ipfs.AddressToKey(AnotherAddress)
+	AnotherContractLeafKey = eth.AddressToKey(AnotherAddress)
 	testAccount            = state.Account{
 		Nonce:    NonceValue,
 		Balance:  big.NewInt(BalanceValue),
@@ -139,17 +138,19 @@ var (
 		CreatedAccounts: CreatedAccountDiffs,
 	}
 	MockStateDiffBytes, _ = rlp.EncodeToBytes(MockStateDiff)
-	MockStateNodes        = map[common.Hash]ipfs.StateNode{
+	MockStateNodes        = []eth.TrieNode{
-		ContractLeafKey: {
+		{
+			Key:   ContractLeafKey,
 			Value: ValueBytes,
 			Leaf:  true,
 		},
-		AnotherContractLeafKey: {
+		{
+			Key:   AnotherContractLeafKey,
 			Value: AnotherValueBytes,
 			Leaf:  true,
 		},
 	}
-	MockStorageNodes = map[common.Hash][]ipfs.StorageNode{
+	MockStorageNodes = map[common.Hash][]eth.TrieNode{
 		ContractLeafKey: {
 			{
 				Key:   common.BytesToHash(StorageKey),
@@ -167,14 +168,12 @@ var (
 		TotalDifficulty: big.NewInt(1337),
 	}
 
-	MockIPLDPayload = &ipfs.IPLDPayload{
+	MockIPLDPayload = &eth.IPLDPayload{
 		TotalDifficulty: big.NewInt(1337),
-		BlockNumber:     big.NewInt(1),
+		Block:           MockBlock,
-		BlockHash:       MockBlock.Hash(),
 		Receipts:        MockReceipts,
 		HeaderRLP:       MockHeaderRlp,
-		BlockBody:       MockBlock.Body(),
+		TrxMetaData: []eth.TxModel{
-		TrxMetaData: []*ipfs.TrxMetaData{
 			{
 				CID: "",
 				Src: senderAddr.Hex(),
@@ -186,109 +185,148 @@ var (
 				Dst: "0x0000000000000000000000000000000000000001",
 			},
 		},
-		ReceiptMetaData: []*ipfs.ReceiptMetaData{
+		ReceiptMetaData: []eth.ReceiptModel{
 			{
 				CID: "",
 				Topic0s: []string{
 					"0x0000000000000000000000000000000000000000000000000000000000000004",
 				},
-				ContractAddress: "0x0000000000000000000000000000000000000000",
+				Contract: "0x0000000000000000000000000000000000000000",
 			},
 			{
 				CID: "",
 				Topic0s: []string{
 					"0x0000000000000000000000000000000000000000000000000000000000000005",
 				},
-				ContractAddress: "0x0000000000000000000000000000000000000001",
+				Contract: "0x0000000000000000000000000000000000000001",
 			},
 		},
 		StorageNodes: MockStorageNodes,
 		StateNodes:   MockStateNodes,
 	}
 
-	MockCIDPayload = &ipfs.CIDPayload{
+	MockCIDPayload = &eth.CIDPayload{
-		TotalDifficulty: "1337",
+		HeaderCID: eth2.HeaderModel{
-		BlockNumber:     "1",
+			BlockHash:       MockBlock.Hash().String(),
-		BlockHash:       MockBlock.Hash(),
+			BlockNumber:     MockBlock.Number().String(),
-		HeaderCID:       "mockHeaderCID",
+			CID:             "mockHeaderCID",
-		UncleCIDs:       make(map[common.Hash]string),
+			Uncle:           false,
-		TransactionCIDs: map[common.Hash]*ipfs.TrxMetaData{
+			ParentHash:      MockBlock.ParentHash().String(),
+			TotalDifficulty: "1337",
+		},
+		UncleCIDs: []eth2.HeaderModel{},
+		TransactionCIDs: []eth.TxModel{
+			{
+				TxHash: MockTransactions[0].Hash().String(),
+				CID:    "mockTrxCID1",
+				Dst:    "0x0000000000000000000000000000000000000000",
+				Src:    senderAddr.Hex(),
+			},
+			{
+				TxHash: MockTransactions[1].Hash().String(),
+				CID:    "mockTrxCID2",
+				Dst:    "0x0000000000000000000000000000000000000001",
+				Src:    senderAddr.Hex(),
+			},
+		},
+		ReceiptCIDs: map[common.Hash]eth.ReceiptModel{
 			MockTransactions[0].Hash(): {
-				CID: "mockTrxCID1",
+				CID:      "mockRctCID1",
-				Dst: "0x0000000000000000000000000000000000000000",
+				Topic0s:  []string{"0x0000000000000000000000000000000000000000000000000000000000000004"},
-				Src: senderAddr.Hex(),
+				Contract: "0x0000000000000000000000000000000000000000",
 			},
 			MockTransactions[1].Hash(): {
-				CID: "mockTrxCID2",
+				CID:      "mockRctCID2",
-				Dst: "0x0000000000000000000000000000000000000001",
+				Topic0s:  []string{"0x0000000000000000000000000000000000000000000000000000000000000005"},
-				Src: senderAddr.Hex(),
+				Contract: "0x0000000000000000000000000000000000000001",
 			},
 		},
-		ReceiptCIDs: map[common.Hash]*ipfs.ReceiptMetaData{
+		StateNodeCIDs: []eth.StateNodeModel{
-			MockTransactions[0].Hash(): {
+			{
-				CID:             "mockRctCID1",
+				CID:      "mockStateCID1",
-				Topic0s:         []string{"0x0000000000000000000000000000000000000000000000000000000000000004"},
+				Leaf:     true,
-				ContractAddress: "0x0000000000000000000000000000000000000000",
+				StateKey: ContractLeafKey.String(),
 			},
-			MockTransactions[1].Hash(): {
+			{
-				CID:             "mockRctCID2",
+				CID:      "mockStateCID2",
-				Topic0s:         []string{"0x0000000000000000000000000000000000000000000000000000000000000005"},
+				Leaf:     true,
-				ContractAddress: "0x0000000000000000000000000000000000000001",
+				StateKey: AnotherContractLeafKey.String(),
 			},
 		},
-		StateNodeCIDs: map[common.Hash]ipfs.StateNodeCID{
+		StorageNodeCIDs: map[common.Hash][]eth.StorageNodeModel{
-			ContractLeafKey: {
-				CID:  "mockStateCID1",
-				Leaf: true,
-				Key:  "",
-			},
-			AnotherContractLeafKey: {
-				CID:  "mockStateCID2",
-				Leaf: true,
-				Key:  "",
-			},
-		},
-		StorageNodeCIDs: map[common.Hash][]ipfs.StorageNodeCID{
 			ContractLeafKey: {
 				{
-					CID:      "mockStorageCID",
+					CID:        "mockStorageCID",
-					Key:      "0x0000000000000000000000000000000000000000000000000000000000000001",
+					StorageKey: "0x0000000000000000000000000000000000000000000000000000000000000001",
-					Leaf:     true,
+					Leaf:       true,
-					StateKey: "",
 				},
 			},
 		},
 	}
 
-	MockCIDWrapper = &ipfs.CIDWrapper{
+	MockCIDWrapper = &eth.CIDWrapper{
-		BlockNumber:  big.NewInt(1),
+		BlockNumber: big.NewInt(1),
-		Headers:      []string{"mockHeaderCID"},
+		Headers: []eth2.HeaderModel{
-		Transactions: []string{"mockTrxCID1", "mockTrxCID2"},
-		Receipts:     []string{"mockRctCID1", "mockRctCID2"},
-		Uncles:       []string{},
-		StateNodes: []ipfs.StateNodeCID{
 			{
-				CID:  "mockStateCID1",
+				BlockNumber:     "1",
-				Leaf: true,
+				BlockHash:       MockBlock.Hash().String(),
-				Key:  ContractLeafKey.Hex(),
+				ParentHash:      "0x0000000000000000000000000000000000000000000000000000000000000000",
-			},
+				CID:             "mockHeaderCID",
-			{
+				Uncle:           false,
-				CID:  "mockStateCID2",
+				TotalDifficulty: "1337",
-				Leaf: true,
-				Key:  AnotherContractLeafKey.Hex(),
 			},
 		},
-		StorageNodes: []ipfs.StorageNodeCID{
+		Transactions: []eth2.TxModel{
 			{
-				CID:      "mockStorageCID",
+				CID: "mockTrxCID1",
+			},
+			{
+				TxHash: MockTransactions[1].Hash().String(),
+				CID:    "mockTrxCID2",
+				Dst:    "0x0000000000000000000000000000000000000001",
+				Src:    senderAddr.String(),
+			},
+		},
+		Receipts: []eth2.ReceiptModel{
+			{
+				CID:      "mockRctCID1",
+				Contract: "0x0000000000000000000000000000000000000000",
+				Topic0s: []string{
+					"0x0000000000000000000000000000000000000000000000000000000000000004",
+				},
+			},
+			{
+				CID:      "mockRctCID2",
+				Contract: "0x0000000000000000000000000000000000000001",
+				Topic0s: []string{
+					"0x0000000000000000000000000000000000000000000000000000000000000005",
+				},
+			},
+		},
+		Uncles: []eth2.HeaderModel{},
+		StateNodes: []eth.StateNodeModel{
+			{
+				CID:      "mockStateCID1",
 				Leaf:     true,
 				StateKey: ContractLeafKey.Hex(),
-				Key:      "0x0000000000000000000000000000000000000000000000000000000000000001",
+			},
+			{
+				CID:      "mockStateCID2",
+				Leaf:     true,
+				StateKey: AnotherContractLeafKey.Hex(),
+			},
+		},
+		StorageNodes: []eth.StorageNodeWithStateKeyModel{
+			{
+				CID:        "mockStorageCID",
+				Leaf:       true,
+				StateKey:   ContractLeafKey.Hex(),
+				StorageKey: "0x0000000000000000000000000000000000000000000000000000000000000001",
 			},
 		},
 	}
 
-	MockIPLDWrapper = ipfs.IPLDWrapper{
+	MockIPLDWrapper = &eth.IPLDWrapper{
 		BlockNumber: big.NewInt(1),
 		Headers: []blocks.Block{
 			blocks.NewBlock(MockHeaderRlp),
@@ -312,7 +350,7 @@ var (
 		},
 	}
 
-	MockSeeNodePayload = streamer.SuperNodePayload{
+	MockSeedNodePayload = eth2.StreamPayload{
 		BlockNumber:     big.NewInt(1),
 		HeadersRlp:      [][]byte{MockHeaderRlp},
 		UnclesRlp:       [][]byte{},

pkg/super_node/eth/models.go

@@ -0,0 +1,71 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package eth
+
+import "github.com/lib/pq"
+
+type HeaderModel struct {
+	ID              int64  `db:"id"`
+	BlockNumber     string `db:"block_number"`
+	BlockHash       string `db:"block_hash"`
+	ParentHash      string `db:"parent_hash"`
+	CID             string `db:"cid"`
+	Uncle           bool   `db:"uncle"`
+	TotalDifficulty string `db:"td"`
+}
+
+type TxModel struct {
+	ID       int64  `db:"id"`
+	HeaderID int64  `db:"header_id"`
+	TxHash   string `db:"tx_hash"`
+	CID      string `db:"cid"`
+	Dst      string `db:"dst"`
+	Src      string `db:"src"`
+}
+
+type ReceiptModel struct {
+	ID       int64          `db:"id"`
+	TxID     int64          `db:"tx_id"`
+	CID      string         `db:"cid"`
+	Contract string         `db:"contract"`
+	Topic0s  pq.StringArray `db:"topic0s"`
+}
+
+type StateNodeModel struct {
+	ID       int64  `db:"id"`
+	HeaderID int64  `db:"header_id"`
+	StateKey string `db:"state_key"`
+	Leaf     bool   `db:"leaf"`
+	CID      string `db:"cid"`
+}
+
+type StorageNodeModel struct {
+	ID         int64  `db:"id"`
+	StateID    int64  `db:"state_id"`
+	StorageKey string `db:"storage_key"`
+	Leaf       bool   `db:"leaf"`
+	CID        string `db:"cid"`
+}
+
+type StorageNodeWithStateKeyModel struct {
+	ID         int64  `db:"id"`
+	StateID    int64  `db:"state_id"`
+	StateKey   string `db:"state_key"`
+	StorageKey string `db:"storage_key"`
+	Leaf       bool   `db:"leaf"`
+	CID        string `db:"cid"`
+}

pkg/super_node/eth/payload_fetcher.go

@@ -0,0 +1,74 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package eth
+
+import (
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/statediff"
+
+	"github.com/vulcanize/vulcanizedb/pkg/eth/client"
+)
+
+// BatchClient is an interface to a batch-fetching geth rpc client; created to allow mock insertion
+type BatchClient interface {
+	BatchCall(batch []client.BatchElem) error
+}
+
+// PayloadFetcher satisfies the PayloadFetcher interface for ethereum
+type PayloadFetcher struct {
+	// PayloadFetcher is thread-safe as long as the underlying client is thread-safe, since it has/modifies no other state
+	// http.Client is thread-safe
+	client BatchClient
+}
+
+const method = "statediff_stateDiffAt"
+
+// NewStateDiffFetcher returns a PayloadFetcher
+func NewPayloadFetcher(bc BatchClient) *PayloadFetcher {
+	return &PayloadFetcher{
+		client: bc,
+	}
+}
+
+// FetchAt fetches the statediff payloads at the given block heights
+// Calls StateDiffAt(ctx context.Context, blockNumber uint64) (*Payload, error)
+func (fetcher *PayloadFetcher) FetchAt(blockHeights []uint64) ([]interface{}, error) {
+	batch := make([]client.BatchElem, 0)
+	for _, height := range blockHeights {
+		batch = append(batch, client.BatchElem{
+			Method: method,
+			Args:   []interface{}{height},
+			Result: new(statediff.Payload),
+		})
+	}
+	batchErr := fetcher.client.BatchCall(batch)
+	if batchErr != nil {
+		return nil, fmt.Errorf("PayloadFetcher err: %s", batchErr.Error())
+	}
+	results := make([]interface{}, 0, len(blockHeights))
+	for _, batchElem := range batch {
+		if batchElem.Error != nil {
+			return nil, fmt.Errorf("PayloadFetcher err: %s", batchElem.Error.Error())
+		}
+		payload, ok := batchElem.Result.(*statediff.Payload)
+		if ok {
+			results = append(results, *payload)
+		}
+	}
+	return results, nil
+}

pkg/super_node/eth/payload_fetcher_test.go

@@ -0,0 +1,59 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package eth_test
+
+import (
+	"github.com/ethereum/go-ethereum/statediff"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/libraries/shared/mocks"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/test_data"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
+)
+
+var _ = Describe("StateDiffFetcher", func() {
+	Describe("FetchStateDiffsAt", func() {
+		var (
+			mc               *mocks.BackFillerClient
+			stateDiffFetcher *eth.PayloadFetcher
+		)
+		BeforeEach(func() {
+			mc = new(mocks.BackFillerClient)
+			setDiffAtErr1 := mc.SetReturnDiffAt(test_data.BlockNumber.Uint64(), test_data.MockStatediffPayload)
+			Expect(setDiffAtErr1).ToNot(HaveOccurred())
+			setDiffAtErr2 := mc.SetReturnDiffAt(test_data.BlockNumber2.Uint64(), test_data.MockStatediffPayload2)
+			Expect(setDiffAtErr2).ToNot(HaveOccurred())
+			stateDiffFetcher = eth.NewPayloadFetcher(mc)
+		})
+		It("Batch calls statediff_stateDiffAt", func() {
+			blockHeights := []uint64{
+				test_data.BlockNumber.Uint64(),
+				test_data.BlockNumber2.Uint64(),
+			}
+			stateDiffPayloads, fetchErr := stateDiffFetcher.FetchAt(blockHeights)
+			Expect(fetchErr).ToNot(HaveOccurred())
+			Expect(len(stateDiffPayloads)).To(Equal(2))
+			payload1, ok := stateDiffPayloads[0].(statediff.Payload)
+			Expect(ok).To(BeTrue())
+			payload2, ok := stateDiffPayloads[1].(statediff.Payload)
+			Expect(ok).To(BeTrue())
+			Expect(payload1).To(Equal(test_data.MockStatediffPayload))
+			Expect(payload2).To(Equal(test_data.MockStatediffPayload2))
+		})
+	})
+})

pkg/super_node/eth/publisher.go

@@ -14,14 +14,16 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package ipfs
+package eth
 
 import (
 	"errors"
+	"fmt"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/rlp"
+
 	"github.com/vulcanize/eth-block-extractor/pkg/ipfs"
 	"github.com/vulcanize/eth-block-extractor/pkg/ipfs/eth_block_header"
 	"github.com/vulcanize/eth-block-extractor/pkg/ipfs/eth_block_receipts"
@@ -31,13 +33,8 @@ import (
 	rlp2 "github.com/vulcanize/eth-block-extractor/pkg/wrappers/rlp"
 )
 
-// IPLDPublisher is the interface for publishing an IPLD payload
+// IPLDPublisher satisfies the IPLDPublisher for ethereum
-type IPLDPublisher interface {
+type IPLDPublisher struct {
-	Publish(payload *IPLDPayload) (*CIDPayload, error)
-}
-
-// Publisher is the underlying struct for the IPLDPublisher interface
-type Publisher struct {
 	HeaderPutter      ipfs.DagPutter
 	TransactionPutter ipfs.DagPutter
 	ReceiptPutter     ipfs.DagPutter
@@ -46,12 +43,12 @@ type Publisher struct {
 }
 
 // NewIPLDPublisher creates a pointer to a new Publisher which satisfies the IPLDPublisher interface
-func NewIPLDPublisher(ipfsPath string) (*Publisher, error) {
+func NewIPLDPublisher(ipfsPath string) (*IPLDPublisher, error) {
 	node, err := ipfs.InitIPFSNode(ipfsPath)
 	if err != nil {
 		return nil, err
 	}
-	return &Publisher{
+	return &IPLDPublisher{
 		HeaderPutter:      eth_block_header.NewBlockHeaderDagPutter(node, rlp2.RlpDecoder{}),
 		TransactionPutter: eth_block_transactions.NewBlockTransactionsDagPutter(node),
 		ReceiptPutter:     eth_block_receipts.NewEthBlockReceiptDagPutter(node),
@@ -61,57 +58,72 @@ func NewIPLDPublisher(ipfsPath string) (*Publisher, error) {
 }
 
 // Publish publishes an IPLDPayload to IPFS and returns the corresponding CIDPayload
-func (pub *Publisher) Publish(payload *IPLDPayload) (*CIDPayload, error) {
+func (pub *IPLDPublisher) Publish(payload interface{}) (interface{}, error) {
+	ipldPayload, ok := payload.(*IPLDPayload)
+	if !ok {
+		return nil, fmt.Errorf("eth publisher expected payload type %T got %T", &IPLDPayload{}, payload)
+	}
 	// Process and publish headers
-	headerCid, err := pub.publishHeaders(payload.HeaderRLP)
+	headerCid, err := pub.publishHeader(ipldPayload.HeaderRLP)
 	if err != nil {
 		return nil, err
 	}
+	header := HeaderModel{
+		CID:             headerCid,
+		ParentHash:      ipldPayload.Block.ParentHash().String(),
+		BlockNumber:     ipldPayload.Block.Number().String(),
+		Uncle:           false,
+		BlockHash:       ipldPayload.Block.Hash().String(),
+		TotalDifficulty: ipldPayload.TotalDifficulty.String(),
+	}
 
 	// Process and publish uncles
-	uncleCids := make(map[common.Hash]string)
+	uncleCids := make([]HeaderModel, 0, len(ipldPayload.Block.Uncles()))
-	for _, uncle := range payload.BlockBody.Uncles {
+	for _, uncle := range ipldPayload.Block.Uncles() {
 		uncleRlp, err := rlp.EncodeToBytes(uncle)
 		if err != nil {
 			return nil, err
 		}
-		cid, err := pub.publishHeaders(uncleRlp)
+		uncleCid, err := pub.publishHeader(uncleRlp)
 		if err != nil {
 			return nil, err
 		}
-		uncleCids[uncle.Hash()] = cid
+		uncleCids = append(uncleCids, HeaderModel{
+			CID:         uncleCid,
+			ParentHash:  uncle.ParentHash.String(),
+			Uncle:       true,
+			BlockHash:   uncle.Hash().String(),
+			BlockNumber: uncle.Number.String(),
+		})
 	}
 
 	// Process and publish transactions
-	transactionCids, err := pub.publishTransactions(payload.BlockBody, payload.TrxMetaData)
+	transactionCids, err := pub.publishTransactions(ipldPayload.Block.Body(), ipldPayload.TrxMetaData)
 	if err != nil {
 		return nil, err
 	}
 
 	// Process and publish receipts
-	receiptsCids, err := pub.publishReceipts(payload.Receipts, payload.ReceiptMetaData)
+	receiptsCids, err := pub.publishReceipts(ipldPayload.Receipts, ipldPayload.ReceiptMetaData)
 	if err != nil {
 		return nil, err
 	}
 
 	// Process and publish state leafs
-	stateNodeCids, err := pub.publishStateNodes(payload.StateNodes)
+	stateNodeCids, err := pub.publishStateNodes(ipldPayload.StateNodes)
 	if err != nil {
 		return nil, err
 	}
 
 	// Process and publish storage leafs
-	storageNodeCids, err := pub.publishStorageNodes(payload.StorageNodes)
+	storageNodeCids, err := pub.publishStorageNodes(ipldPayload.StorageNodes)
 	if err != nil {
 		return nil, err
 	}
 
 	// Package CIDs and their metadata into a single struct
 	return &CIDPayload{
-		TotalDifficulty: payload.TotalDifficulty.String(),
+		HeaderCID:       header,
-		BlockHash:       payload.BlockHash,
-		BlockNumber:     payload.BlockNumber.String(),
-		HeaderCID:       headerCid,
 		UncleCIDs:       uncleCids,
 		TransactionCIDs: transactionCids,
 		ReceiptCIDs:     receiptsCids,
@@ -120,7 +132,7 @@ func (pub *Publisher) Publish(payload *IPLDPayload) (*CIDPayload, error) {
 	}, nil
 }
 
-func (pub *Publisher) publishHeaders(headerRLP []byte) (string, error) {
+func (pub *IPLDPublisher) publishHeader(headerRLP []byte) (string, error) {
 	headerCids, err := pub.HeaderPutter.DagPut(headerRLP)
 	if err != nil {
 		return "", err
@@ -131,7 +143,7 @@ func (pub *Publisher) publishHeaders(headerRLP []byte) (string, error) {
 	return headerCids[0], nil
 }
 
-func (pub *Publisher) publishTransactions(blockBody *types.Body, trxMeta []*TrxMetaData) (map[common.Hash]*TrxMetaData, error) {
+func (pub *IPLDPublisher) publishTransactions(blockBody *types.Body, trxMeta []TxModel) ([]TxModel, error) {
 	transactionCids, err := pub.TransactionPutter.DagPut(blockBody)
 	if err != nil {
 		return nil, err
@@ -139,15 +151,19 @@ func (pub *Publisher) publishTransactions(blockBody *types.Body, trxMeta []*TrxM
 	if len(transactionCids) != len(blockBody.Transactions) {
 		return nil, errors.New("expected one CID for each transaction")
 	}
-	mappedTrxCids := make(map[common.Hash]*TrxMetaData, len(transactionCids))
+	mappedTrxCids := make([]TxModel, len(transactionCids))
 	for i, trx := range blockBody.Transactions {
-		mappedTrxCids[trx.Hash()] = trxMeta[i]
+		mappedTrxCids[i] = TxModel{
-		mappedTrxCids[trx.Hash()].CID = transactionCids[i]
+			CID:    transactionCids[i],
+			TxHash: trx.Hash().Hex(),
+			Src:    trxMeta[i].Src,
+			Dst:    trxMeta[i].Dst,
+		}
 	}
 	return mappedTrxCids, nil
 }
 
-func (pub *Publisher) publishReceipts(receipts types.Receipts, receiptMeta []*ReceiptMetaData) (map[common.Hash]*ReceiptMetaData, error) {
+func (pub *IPLDPublisher) publishReceipts(receipts types.Receipts, receiptMeta []ReceiptModel) (map[common.Hash]ReceiptModel, error) {
 	receiptsCids, err := pub.ReceiptPutter.DagPut(receipts)
 	if err != nil {
 		return nil, err
@@ -155,18 +171,21 @@ func (pub *Publisher) publishReceipts(receipts types.Receipts, receiptMeta []*Re
 	if len(receiptsCids) != len(receipts) {
 		return nil, errors.New("expected one CID for each receipt")
 	}
-	// Keep receipts associated with their transaction
+	// Map receipt cids to their transaction hashes
-	mappedRctCids := make(map[common.Hash]*ReceiptMetaData, len(receiptsCids))
+	mappedRctCids := make(map[common.Hash]ReceiptModel, len(receiptsCids))
 	for i, rct := range receipts {
-		mappedRctCids[rct.TxHash] = receiptMeta[i]
+		mappedRctCids[rct.TxHash] = ReceiptModel{
-		mappedRctCids[rct.TxHash].CID = receiptsCids[i]
+			CID:      receiptsCids[i],
+			Contract: receiptMeta[i].Contract,
+			Topic0s:  receiptMeta[i].Topic0s,
+		}
 	}
 	return mappedRctCids, nil
 }
 
-func (pub *Publisher) publishStateNodes(stateNodes map[common.Hash]StateNode) (map[common.Hash]StateNodeCID, error) {
+func (pub *IPLDPublisher) publishStateNodes(stateNodes []TrieNode) ([]StateNodeModel, error) {
-	stateNodeCids := make(map[common.Hash]StateNodeCID)
+	stateNodeCids := make([]StateNodeModel, 0, len(stateNodes))
-	for addrKey, node := range stateNodes {
+	for _, node := range stateNodes {
 		stateNodeCid, err := pub.StatePutter.DagPut(node.Value)
 		if err != nil {
 			return nil, err
@@ -174,18 +193,19 @@ func (pub *Publisher) publishStateNodes(stateNodes map[common.Hash]StateNode) (m
 		if len(stateNodeCid) != 1 {
 			return nil, errors.New("single CID expected to be returned for state leaf")
 		}
-		stateNodeCids[addrKey] = StateNodeCID{
+		stateNodeCids = append(stateNodeCids, StateNodeModel{
-			CID:  stateNodeCid[0],
+			StateKey: node.Key.String(),
-			Leaf: node.Leaf,
+			CID:      stateNodeCid[0],
-		}
+			Leaf:     node.Leaf,
+		})
 	}
 	return stateNodeCids, nil
 }
 
-func (pub *Publisher) publishStorageNodes(storageNodes map[common.Hash][]StorageNode) (map[common.Hash][]StorageNodeCID, error) {
+func (pub *IPLDPublisher) publishStorageNodes(storageNodes map[common.Hash][]TrieNode) (map[common.Hash][]StorageNodeModel, error) {
-	storageLeafCids := make(map[common.Hash][]StorageNodeCID)
+	storageLeafCids := make(map[common.Hash][]StorageNodeModel)
 	for addrKey, storageTrie := range storageNodes {
-		storageLeafCids[addrKey] = make([]StorageNodeCID, 0, len(storageTrie))
+		storageLeafCids[addrKey] = make([]StorageNodeModel, 0, len(storageTrie))
 		for _, node := range storageTrie {
 			storageNodeCid, err := pub.StoragePutter.DagPut(node.Value)
 			if err != nil {
@@ -194,10 +214,11 @@ func (pub *Publisher) publishStorageNodes(storageNodes map[common.Hash][]Storage
 			if len(storageNodeCid) != 1 {
 				return nil, errors.New("single CID expected to be returned for storage leaf")
 			}
-			storageLeafCids[addrKey] = append(storageLeafCids[addrKey], StorageNodeCID{
+			// Map storage node cids to their state key hashes
-				Key:  node.Key.Hex(),
+			storageLeafCids[addrKey] = append(storageLeafCids[addrKey], StorageNodeModel{
-				CID:  storageNodeCid[0],
+				StorageKey: node.Key.Hex(),
-				Leaf: node.Leaf,
+				CID:        storageNodeCid[0],
+				Leaf:       node.Leaf,
 			})
 		}
 	}

pkg/super_node/eth/publisher_test.go

@@ -14,15 +14,15 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package ipfs_test
+package eth_test
 
 import (
 	"github.com/ethereum/go-ethereum/common"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
-	"github.com/vulcanize/vulcanizedb/pkg/ipfs/mocks"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth/mocks"
 )
 
 var (
@@ -47,36 +47,38 @@ var _ = Describe("Publisher", func() {
 			mockHeaderDagPutter.CIDsToReturn = []string{"mockHeaderCID"}
 			mockTrxDagPutter.CIDsToReturn = []string{"mockTrxCID1", "mockTrxCID2"}
 			mockRctDagPutter.CIDsToReturn = []string{"mockRctCID1", "mockRctCID2"}
-			val1 := common.BytesToHash(mocks.MockIPLDPayload.StateNodes[mocks.ContractLeafKey].Value)
+			val1 := common.BytesToHash(mocks.MockIPLDPayload.StateNodes[0].Value)
-			val2 := common.BytesToHash(mocks.MockIPLDPayload.StateNodes[mocks.AnotherContractLeafKey].Value)
+			val2 := common.BytesToHash(mocks.MockIPLDPayload.StateNodes[1].Value)
 			mockStateDagPutter.CIDsToReturn = map[common.Hash][]string{
 				val1: {"mockStateCID1"},
 				val2: {"mockStateCID2"},
 			}
 			mockStorageDagPutter.CIDsToReturn = []string{"mockStorageCID"}
-			publisher := ipfs.Publisher{
+			publisher := eth.IPLDPublisher{
 				HeaderPutter:      mockHeaderDagPutter,
 				TransactionPutter: mockTrxDagPutter,
 				ReceiptPutter:     mockRctDagPutter,
 				StatePutter:       mockStateDagPutter,
 				StoragePutter:     mockStorageDagPutter,
 			}
-			cidPayload, err := publisher.Publish(mocks.MockIPLDPayload)
+			payload, err := publisher.Publish(mocks.MockIPLDPayload)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(cidPayload.TotalDifficulty).To(Equal(mocks.MockIPLDPayload.TotalDifficulty.String()))
+			cidPayload, ok := payload.(*eth.CIDPayload)
-			Expect(cidPayload.BlockNumber).To(Equal(mocks.MockCIDPayload.BlockNumber))
+			Expect(ok).To(BeTrue())
-			Expect(cidPayload.BlockHash).To(Equal(mocks.MockCIDPayload.BlockHash))
+			Expect(cidPayload.HeaderCID.TotalDifficulty).To(Equal(mocks.MockIPLDPayload.TotalDifficulty.String()))
+			Expect(cidPayload.HeaderCID.BlockNumber).To(Equal(mocks.MockCIDPayload.HeaderCID.BlockNumber))
+			Expect(cidPayload.HeaderCID.BlockHash).To(Equal(mocks.MockCIDPayload.HeaderCID.BlockHash))
 			Expect(cidPayload.UncleCIDs).To(Equal(mocks.MockCIDPayload.UncleCIDs))
 			Expect(cidPayload.HeaderCID).To(Equal(mocks.MockCIDPayload.HeaderCID))
 			Expect(len(cidPayload.TransactionCIDs)).To(Equal(2))
-			Expect(cidPayload.TransactionCIDs[mocks.MockTransactions[0].Hash()]).To(Equal(mocks.MockCIDPayload.TransactionCIDs[mocks.MockTransactions[0].Hash()]))
+			Expect(cidPayload.TransactionCIDs[0]).To(Equal(mocks.MockCIDPayload.TransactionCIDs[0]))
-			Expect(cidPayload.TransactionCIDs[mocks.MockTransactions[1].Hash()]).To(Equal(mocks.MockCIDPayload.TransactionCIDs[mocks.MockTransactions[1].Hash()]))
+			Expect(cidPayload.TransactionCIDs[1]).To(Equal(mocks.MockCIDPayload.TransactionCIDs[1]))
 			Expect(len(cidPayload.ReceiptCIDs)).To(Equal(2))
 			Expect(cidPayload.ReceiptCIDs[mocks.MockTransactions[0].Hash()]).To(Equal(mocks.MockCIDPayload.ReceiptCIDs[mocks.MockTransactions[0].Hash()]))
 			Expect(cidPayload.ReceiptCIDs[mocks.MockTransactions[1].Hash()]).To(Equal(mocks.MockCIDPayload.ReceiptCIDs[mocks.MockTransactions[1].Hash()]))
 			Expect(len(cidPayload.StateNodeCIDs)).To(Equal(2))
-			Expect(cidPayload.StateNodeCIDs[mocks.ContractLeafKey]).To(Equal(mocks.MockCIDPayload.StateNodeCIDs[mocks.ContractLeafKey]))
+			Expect(cidPayload.StateNodeCIDs[0]).To(Equal(mocks.MockCIDPayload.StateNodeCIDs[0]))
-			Expect(cidPayload.StateNodeCIDs[mocks.AnotherContractLeafKey]).To(Equal(mocks.MockCIDPayload.StateNodeCIDs[mocks.AnotherContractLeafKey]))
+			Expect(cidPayload.StateNodeCIDs[1]).To(Equal(mocks.MockCIDPayload.StateNodeCIDs[1]))
 			Expect(cidPayload.StorageNodeCIDs).To(Equal(mocks.MockCIDPayload.StorageNodeCIDs))
 		})
 	})

pkg/super_node/eth/resolver.go

@@ -14,36 +14,30 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package ipfs
+package eth
 
 import (
+	"fmt"
+
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ipfs/go-block-format"
-	"github.com/vulcanize/vulcanizedb/libraries/shared/streamer"
 )
 
-// IPLDResolver is the interface to resolving IPLDs
+// IPLDResolver satisfies the IPLDResolver interface for ethereum
-type IPLDResolver interface {
+type IPLDResolver struct{}
-	ResolveIPLDs(ipfsBlocks IPLDWrapper) streamer.SuperNodePayload
+
-	ResolveHeaders(iplds []blocks.Block) [][]byte
+// NewIPLDResolver returns a pointer to an IPLDResolver which satisfies the IPLDResolver interface
-	ResolveUncles(iplds []blocks.Block) [][]byte
+func NewIPLDResolver() *IPLDResolver {
-	ResolveTransactions(iplds []blocks.Block) [][]byte
+	return &IPLDResolver{}
-	ResolveReceipts(blocks []blocks.Block) [][]byte
-	ResolveState(iplds map[common.Hash]blocks.Block) map[common.Hash][]byte
-	ResolveStorage(iplds map[common.Hash]map[common.Hash]blocks.Block) map[common.Hash]map[common.Hash][]byte
 }
 
-// EthIPLDResolver is the underlying struct to support the IPLDResolver interface
+// Resolve is the exported method for resolving all of the ETH IPLDs packaged in an IpfsBlockWrapper
-type EthIPLDResolver struct{}
+func (eir *IPLDResolver) Resolve(iplds interface{}) (interface{}, error) {
-
+	ipfsBlocks, ok := iplds.(*IPLDWrapper)
-// NewIPLDResolver returns a pointer to an EthIPLDResolver which satisfies the IPLDResolver interface
+	if !ok {
-func NewIPLDResolver() *EthIPLDResolver {
+		return StreamPayload{}, fmt.Errorf("eth resolver expected iplds type %T got %T", &IPLDWrapper{}, iplds)
-	return &EthIPLDResolver{}
+	}
-}
+	return StreamPayload{
-
-// ResolveIPLDs is the exported method for resolving all of the ETH IPLDs packaged in an IpfsBlockWrapper
-func (eir *EthIPLDResolver) ResolveIPLDs(ipfsBlocks IPLDWrapper) streamer.SuperNodePayload {
-	return streamer.SuperNodePayload{
 		BlockNumber:     ipfsBlocks.BlockNumber,
 		HeadersRlp:      eir.ResolveHeaders(ipfsBlocks.Headers),
 		UnclesRlp:       eir.ResolveUncles(ipfsBlocks.Uncles),
@@ -51,10 +45,10 @@ func (eir *EthIPLDResolver) ResolveIPLDs(ipfsBlocks IPLDWrapper) streamer.SuperN
 		ReceiptsRlp:     eir.ResolveReceipts(ipfsBlocks.Receipts),
 		StateNodesRlp:   eir.ResolveState(ipfsBlocks.StateNodes),
 		StorageNodesRlp: eir.ResolveStorage(ipfsBlocks.StorageNodes),
-	}
+	}, nil
 }
 
-func (eir *EthIPLDResolver) ResolveHeaders(iplds []blocks.Block) [][]byte {
+func (eir *IPLDResolver) ResolveHeaders(iplds []blocks.Block) [][]byte {
 	headerRlps := make([][]byte, 0, len(iplds))
 	for _, ipld := range iplds {
 		headerRlps = append(headerRlps, ipld.RawData())
@@ -62,7 +56,7 @@ func (eir *EthIPLDResolver) ResolveHeaders(iplds []blocks.Block) [][]byte {
 	return headerRlps
 }
 
-func (eir *EthIPLDResolver) ResolveUncles(iplds []blocks.Block) [][]byte {
+func (eir *IPLDResolver) ResolveUncles(iplds []blocks.Block) [][]byte {
 	uncleRlps := make([][]byte, 0, len(iplds))
 	for _, ipld := range iplds {
 		uncleRlps = append(uncleRlps, ipld.RawData())
@@ -70,7 +64,7 @@ func (eir *EthIPLDResolver) ResolveUncles(iplds []blocks.Block) [][]byte {
 	return uncleRlps
 }
 
-func (eir *EthIPLDResolver) ResolveTransactions(iplds []blocks.Block) [][]byte {
+func (eir *IPLDResolver) ResolveTransactions(iplds []blocks.Block) [][]byte {
 	trxs := make([][]byte, 0, len(iplds))
 	for _, ipld := range iplds {
 		trxs = append(trxs, ipld.RawData())
@@ -78,7 +72,7 @@ func (eir *EthIPLDResolver) ResolveTransactions(iplds []blocks.Block) [][]byte {
 	return trxs
 }
 
-func (eir *EthIPLDResolver) ResolveReceipts(iplds []blocks.Block) [][]byte {
+func (eir *IPLDResolver) ResolveReceipts(iplds []blocks.Block) [][]byte {
 	rcts := make([][]byte, 0, len(iplds))
 	for _, ipld := range iplds {
 		rcts = append(rcts, ipld.RawData())
@@ -86,7 +80,7 @@ func (eir *EthIPLDResolver) ResolveReceipts(iplds []blocks.Block) [][]byte {
 	return rcts
 }
 
-func (eir *EthIPLDResolver) ResolveState(iplds map[common.Hash]blocks.Block) map[common.Hash][]byte {
+func (eir *IPLDResolver) ResolveState(iplds map[common.Hash]blocks.Block) map[common.Hash][]byte {
 	stateNodes := make(map[common.Hash][]byte, len(iplds))
 	for key, ipld := range iplds {
 		stateNodes[key] = ipld.RawData()
@@ -94,7 +88,7 @@ func (eir *EthIPLDResolver) ResolveState(iplds map[common.Hash]blocks.Block) map
 	return stateNodes
 }
 
-func (eir *EthIPLDResolver) ResolveStorage(iplds map[common.Hash]map[common.Hash]blocks.Block) map[common.Hash]map[common.Hash][]byte {
+func (eir *IPLDResolver) ResolveStorage(iplds map[common.Hash]map[common.Hash]blocks.Block) map[common.Hash]map[common.Hash][]byte {
 	storageNodes := make(map[common.Hash]map[common.Hash][]byte)
 	for stateKey, storageIPLDs := range iplds {
 		storageNodes[stateKey] = make(map[common.Hash][]byte)

pkg/super_node/eth/resolver_test.go

@@ -14,39 +14,42 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package ipfs_test
+package eth_test
 
 import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
 
-	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
-	"github.com/vulcanize/vulcanizedb/pkg/ipfs/mocks"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth/mocks"
-	"github.com/vulcanize/vulcanizedb/pkg/super_node"
 )
 
 var (
-	resolver ipfs.IPLDResolver
+	resolver *eth.IPLDResolver
 )
 
 var _ = Describe("Resolver", func() {
 	Describe("ResolveIPLDs", func() {
 		BeforeEach(func() {
-			resolver = ipfs.NewIPLDResolver()
+			resolver = eth.NewIPLDResolver()
 		})
 		It("Resolves IPLD data to their correct geth data types and packages them to send to requesting transformers", func() {
-			superNodePayload := resolver.ResolveIPLDs(mocks.MockIPLDWrapper)
+			payload, err := resolver.Resolve(mocks.MockIPLDWrapper)
-			Expect(superNodePayload.BlockNumber.Int64()).To(Equal(mocks.MockSeeNodePayload.BlockNumber.Int64()))
+			Expect(err).ToNot(HaveOccurred())
-			Expect(superNodePayload.HeadersRlp).To(Equal(mocks.MockSeeNodePayload.HeadersRlp))
+			superNodePayload, ok := payload.(eth.StreamPayload)
-			Expect(superNodePayload.UnclesRlp).To(Equal(mocks.MockSeeNodePayload.UnclesRlp))
+			Expect(ok).To(BeTrue())
+			Expect(superNodePayload.BlockNumber.Int64()).To(Equal(mocks.MockSeedNodePayload.BlockNumber.Int64()))
+			Expect(superNodePayload.HeadersRlp).To(Equal(mocks.MockSeedNodePayload.HeadersRlp))
+			Expect(superNodePayload.UnclesRlp).To(Equal(mocks.MockSeedNodePayload.UnclesRlp))
 			Expect(len(superNodePayload.TransactionsRlp)).To(Equal(2))
-			Expect(super_node.ListContainsBytes(superNodePayload.TransactionsRlp, mocks.MockTransactions.GetRlp(0))).To(BeTrue())
+			Expect(shared.ListContainsBytes(superNodePayload.TransactionsRlp, mocks.MockTransactions.GetRlp(0))).To(BeTrue())
-			Expect(super_node.ListContainsBytes(superNodePayload.TransactionsRlp, mocks.MockTransactions.GetRlp(1))).To(BeTrue())
+			Expect(shared.ListContainsBytes(superNodePayload.TransactionsRlp, mocks.MockTransactions.GetRlp(1))).To(BeTrue())
 			Expect(len(superNodePayload.ReceiptsRlp)).To(Equal(2))
-			Expect(super_node.ListContainsBytes(superNodePayload.ReceiptsRlp, mocks.MockReceipts.GetRlp(0))).To(BeTrue())
+			Expect(shared.ListContainsBytes(superNodePayload.ReceiptsRlp, mocks.MockReceipts.GetRlp(0))).To(BeTrue())
-			Expect(super_node.ListContainsBytes(superNodePayload.ReceiptsRlp, mocks.MockReceipts.GetRlp(1))).To(BeTrue())
+			Expect(shared.ListContainsBytes(superNodePayload.ReceiptsRlp, mocks.MockReceipts.GetRlp(1))).To(BeTrue())
 			Expect(len(superNodePayload.StateNodesRlp)).To(Equal(2))
-			Expect(superNodePayload.StorageNodesRlp).To(Equal(mocks.MockSeeNodePayload.StorageNodesRlp))
+			Expect(superNodePayload.StorageNodesRlp).To(Equal(mocks.MockSeedNodePayload.StorageNodesRlp))
 		})
 	})
 })

pkg/super_node/eth/retriever.go

@@ -14,161 +14,156 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package super_node
+package eth
 
 import (
+	"fmt"
 	"math/big"
 
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
+
 	"github.com/jmoiron/sqlx"
 	"github.com/lib/pq"
 	log "github.com/sirupsen/logrus"
 
-	"github.com/vulcanize/vulcanizedb/pkg/config"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
-	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/config"
 )
 
-// CIDRetriever is the interface for retrieving CIDs from the Postgres cache
+// CIDRetriever satisfies the CIDRetriever interface for ethereum
-type CIDRetriever interface {
+type CIDRetriever struct {
-	RetrieveCIDs(streamFilters config.Subscription, blockNumber int64) (*ipfs.CIDWrapper, error)
-	RetrieveLastBlockNumber() (int64, error)
-	RetrieveFirstBlockNumber() (int64, error)
-	RetrieveGapsInData() ([][2]uint64, error)
-	RetrieveHeaderCIDs(tx *sqlx.Tx, blockNumber int64) ([]string, error)
-	RetrieveUncleCIDs(tx *sqlx.Tx, blockNumber int64) ([]string, error)
-	RetrieveTrxCIDs(tx *sqlx.Tx, txFilter config.TrxFilter, blockNumber int64) ([]string, []int64, error)
-	RetrieveRctCIDs(tx *sqlx.Tx, rctFilter config.ReceiptFilter, blockNumber int64, trxIds []int64) ([]string, error)
-	RetrieveStateCIDs(tx *sqlx.Tx, stateFilter config.StateFilter, blockNumber int64) ([]ipfs.StateNodeCID, error)
-	Database() *postgres.DB
-}
-
-// EthCIDRetriever is the underlying struct supporting the CIDRetriever interface
-type EthCIDRetriever struct {
 	db *postgres.DB
 }
 
-// NewCIDRetriever returns a pointer to a new EthCIDRetriever which supports the CIDRetriever interface
+// NewCIDRetriever returns a pointer to a new CIDRetriever which supports the CIDRetriever interface
-func NewCIDRetriever(db *postgres.DB) *EthCIDRetriever {
+func NewCIDRetriever(db *postgres.DB) *CIDRetriever {
-	return &EthCIDRetriever{
+	return &CIDRetriever{
 		db: db,
 	}
 }
 
 // RetrieveFirstBlockNumber is used to retrieve the first block number in the db
-func (ecr *EthCIDRetriever) RetrieveFirstBlockNumber() (int64, error) {
+func (ecr *CIDRetriever) RetrieveFirstBlockNumber() (int64, error) {
 	var blockNumber int64
 	err := ecr.db.Get(&blockNumber, "SELECT block_number FROM header_cids ORDER BY block_number ASC LIMIT 1")
 	return blockNumber, err
 }
 
 // RetrieveLastBlockNumber is used to retrieve the latest block number in the db
-func (ecr *EthCIDRetriever) RetrieveLastBlockNumber() (int64, error) {
+func (ecr *CIDRetriever) RetrieveLastBlockNumber() (int64, error) {
 	var blockNumber int64
 	err := ecr.db.Get(&blockNumber, "SELECT block_number FROM header_cids ORDER BY block_number DESC LIMIT 1 ")
 	return blockNumber, err
 }
 
-// RetrieveCIDs is used to retrieve all of the CIDs which conform to the passed StreamFilters
+// Retrieve is used to retrieve all of the CIDs which conform to the passed StreamFilters
-func (ecr *EthCIDRetriever) RetrieveCIDs(streamFilters config.Subscription, blockNumber int64) (*ipfs.CIDWrapper, error) {
+func (ecr *CIDRetriever) Retrieve(filter interface{}, blockNumber int64) (interface{}, bool, error) {
+	streamFilter, ok := filter.(*config.EthSubscription)
+	if !ok {
+		return nil, true, fmt.Errorf("eth retriever expected filter type %T got %T", &config.EthSubscription{}, filter)
+	}
 	log.Debug("retrieving cids")
 	tx, err := ecr.db.Beginx()
 	if err != nil {
-		return nil, err
+		return nil, true, err
 	}
 
-	cw := new(ipfs.CIDWrapper)
+	cw := new(CIDWrapper)
 	cw.BlockNumber = big.NewInt(blockNumber)
-
 	// Retrieve cached header CIDs
-	if !streamFilters.HeaderFilter.Off {
+	if !streamFilter.HeaderFilter.Off {
 		cw.Headers, err = ecr.RetrieveHeaderCIDs(tx, blockNumber)
 		if err != nil {
 			if err := tx.Rollback(); err != nil {
 				log.Error(err)
 			}
 			log.Error("header cid retrieval error")
-			return nil, err
+			return nil, true, err
 		}
-		if streamFilters.HeaderFilter.Uncles {
+		if streamFilter.HeaderFilter.Uncles {
 			cw.Uncles, err = ecr.RetrieveUncleCIDs(tx, blockNumber)
 			if err != nil {
 				if err := tx.Rollback(); err != nil {
 					log.Error(err)
 				}
 				log.Error("uncle cid retrieval error")
-				return nil, err
+				return nil, true, err
 			}
 		}
 	}
-
 	// Retrieve cached trx CIDs
-	var trxIds []int64
+	if !streamFilter.TxFilter.Off {
-	if !streamFilters.TrxFilter.Off {
+		cw.Transactions, err = ecr.RetrieveTrxCIDs(tx, streamFilter.TxFilter, blockNumber)
-		cw.Transactions, trxIds, err = ecr.RetrieveTrxCIDs(tx, streamFilters.TrxFilter, blockNumber)
 		if err != nil {
-			err := tx.Rollback()
+			if err := tx.Rollback(); err != nil {
-			if err != nil {
 				log.Error(err)
 			}
 			log.Error("transaction cid retrieval error")
-			return nil, err
+			return nil, true, err
 		}
 	}
-
+	trxIds := make([]int64, 0, len(cw.Transactions))
+	for _, tx := range cw.Transactions {
+		trxIds = append(trxIds, tx.ID)
+	}
 	// Retrieve cached receipt CIDs
-	if !streamFilters.ReceiptFilter.Off {
+	if !streamFilter.ReceiptFilter.Off {
-		cw.Receipts, err = ecr.RetrieveRctCIDs(tx, streamFilters.ReceiptFilter, blockNumber, trxIds)
+		cw.Receipts, err = ecr.RetrieveRctCIDs(tx, streamFilter.ReceiptFilter, blockNumber, trxIds)
 		if err != nil {
 			if err := tx.Rollback(); err != nil {
 				log.Error(err)
 			}
 			log.Error("receipt cid retrieval error")
-			return nil, err
+			return nil, true, err
 		}
 	}
-
 	// Retrieve cached state CIDs
-	if !streamFilters.StateFilter.Off {
+	if !streamFilter.StateFilter.Off {
-		cw.StateNodes, err = ecr.RetrieveStateCIDs(tx, streamFilters.StateFilter, blockNumber)
+		cw.StateNodes, err = ecr.RetrieveStateCIDs(tx, streamFilter.StateFilter, blockNumber)
 		if err != nil {
 			if err := tx.Rollback(); err != nil {
 				log.Error(err)
 			}
 			log.Error("state cid retrieval error")
-			return nil, err
+			return nil, true, err
 		}
 	}
-
 	// Retrieve cached storage CIDs
-	if !streamFilters.StorageFilter.Off {
+	if !streamFilter.StorageFilter.Off {
-		cw.StorageNodes, err = ecr.RetrieveStorageCIDs(tx, streamFilters.StorageFilter, blockNumber)
+		cw.StorageNodes, err = ecr.RetrieveStorageCIDs(tx, streamFilter.StorageFilter, blockNumber)
 		if err != nil {
 			if err := tx.Rollback(); err != nil {
 				log.Error(err)
 			}
 			log.Error("storage cid retrieval error")
-			return nil, err
+			return nil, true, err
 		}
 	}
+	return cw, empty(cw), tx.Commit()
+}
 
-	return cw, tx.Commit()
+func empty(cidWrapper *CIDWrapper) bool {
+	if len(cidWrapper.Transactions) > 0 || len(cidWrapper.Headers) > 0 || len(cidWrapper.Uncles) > 0 || len(cidWrapper.Receipts) > 0 || len(cidWrapper.StateNodes) > 0 || len(cidWrapper.StorageNodes) > 0 {
+		return false
+	}
+	return true
 }
 
 // RetrieveHeaderCIDs retrieves and returns all of the header cids at the provided blockheight
-func (ecr *EthCIDRetriever) RetrieveHeaderCIDs(tx *sqlx.Tx, blockNumber int64) ([]string, error) {
+func (ecr *CIDRetriever) RetrieveHeaderCIDs(tx *sqlx.Tx, blockNumber int64) ([]HeaderModel, error) {
 	log.Debug("retrieving header cids for block ", blockNumber)
-	headers := make([]string, 0)
+	headers := make([]HeaderModel, 0)
-	pgStr := `SELECT cid FROM header_cids
+	pgStr := `SELECT * FROM header_cids
 				WHERE block_number = $1 AND uncle IS FALSE`
 	err := tx.Select(&headers, pgStr, blockNumber)
 	return headers, err
 }
 
 // RetrieveUncleCIDs retrieves and returns all of the uncle cids at the provided blockheight
-func (ecr *EthCIDRetriever) RetrieveUncleCIDs(tx *sqlx.Tx, blockNumber int64) ([]string, error) {
+func (ecr *CIDRetriever) RetrieveUncleCIDs(tx *sqlx.Tx, blockNumber int64) ([]HeaderModel, error) {
 	log.Debug("retrieving header cids for block ", blockNumber)
-	headers := make([]string, 0)
+	headers := make([]HeaderModel, 0)
-	pgStr := `SELECT cid FROM header_cids
+	pgStr := `SELECT * FROM header_cids
 				WHERE block_number = $1 AND uncle IS TRUE`
 	err := tx.Select(&headers, pgStr, blockNumber)
 	return headers, err
@@ -176,15 +171,14 @@ func (ecr *EthCIDRetriever) RetrieveUncleCIDs(tx *sqlx.Tx, blockNumber int64) ([
 
 // RetrieveTrxCIDs retrieves and returns all of the trx cids at the provided blockheight that conform to the provided filter parameters
 // also returns the ids for the returned transaction cids
-func (ecr *EthCIDRetriever) RetrieveTrxCIDs(tx *sqlx.Tx, txFilter config.TrxFilter, blockNumber int64) ([]string, []int64, error) {
+func (ecr *CIDRetriever) RetrieveTrxCIDs(tx *sqlx.Tx, txFilter config.TxFilter, blockNumber int64) ([]TxModel, error) {
 	log.Debug("retrieving transaction cids for block ", blockNumber)
 	args := make([]interface{}, 0, 3)
-	type result struct {
+	results := make([]TxModel, 0)
-		ID  int64  `db:"id"`
+	pgStr := `SELECT transaction_cids.id, transaction_cids.header_id,
-		Cid string `db:"cid"`
+ 			transaction_cids.tx_hash, transaction_cids.cid,
-	}
+ 			transaction_cids.dst, transaction_cids.src
-	results := make([]result, 0)
+ 			FROM transaction_cids INNER JOIN header_cids ON (transaction_cids.header_id = header_cids.id)
-	pgStr := `SELECT transaction_cids.id, transaction_cids.cid FROM transaction_cids INNER JOIN header_cids ON (transaction_cids.header_id = header_cids.id)
 			WHERE header_cids.block_number = $1`
 	args = append(args, blockNumber)
 	if len(txFilter.Dst) > 0 {
@@ -197,23 +191,19 @@ func (ecr *EthCIDRetriever) RetrieveTrxCIDs(tx *sqlx.Tx, txFilter config.TrxFilt
 	}
 	err := tx.Select(&results, pgStr, args...)
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
-	ids := make([]int64, 0, len(results))
+	return results, nil
-	cids := make([]string, 0, len(results))
-	for _, res := range results {
-		cids = append(cids, res.Cid)
-		ids = append(ids, res.ID)
-	}
-	return cids, ids, nil
 }
 
 // RetrieveRctCIDs retrieves and returns all of the rct cids at the provided blockheight that conform to the provided
 // filter parameters and correspond to the provided tx ids
-func (ecr *EthCIDRetriever) RetrieveRctCIDs(tx *sqlx.Tx, rctFilter config.ReceiptFilter, blockNumber int64, trxIds []int64) ([]string, error) {
+func (ecr *CIDRetriever) RetrieveRctCIDs(tx *sqlx.Tx, rctFilter config.ReceiptFilter, blockNumber int64, trxIds []int64) ([]ReceiptModel, error) {
 	log.Debug("retrieving receipt cids for block ", blockNumber)
 	args := make([]interface{}, 0, 4)
-	pgStr := `SELECT receipt_cids.cid FROM receipt_cids, transaction_cids, header_cids
+	pgStr := `SELECT receipt_cids.id, receipt_cids.tx_id, receipt_cids.cid,
+ 			receipt_cids.contract, receipt_cids.topic0s
+ 			FROM receipt_cids, transaction_cids, header_cids
 			WHERE receipt_cids.tx_id = transaction_cids.id 
 			AND transaction_cids.header_id = header_cids.id
 			AND header_cids.block_number = $1`
@@ -254,23 +244,29 @@ func (ecr *EthCIDRetriever) RetrieveRctCIDs(tx *sqlx.Tx, rctFilter config.Receip
 			args = append(args, pq.Array(trxIds))
 		}
 	}
-	receiptCids := make([]string, 0)
+	receiptCids := make([]ReceiptModel, 0)
 	err := tx.Select(&receiptCids, pgStr, args...)
+	if err != nil {
+		println(pgStr)
+		println("FUCK YOU\r\n\r\n\r\n")
+	}
 	return receiptCids, err
 }
 
 // RetrieveStateCIDs retrieves and returns all of the state node cids at the provided blockheight that conform to the provided filter parameters
-func (ecr *EthCIDRetriever) RetrieveStateCIDs(tx *sqlx.Tx, stateFilter config.StateFilter, blockNumber int64) ([]ipfs.StateNodeCID, error) {
+func (ecr *CIDRetriever) RetrieveStateCIDs(tx *sqlx.Tx, stateFilter config.StateFilter, blockNumber int64) ([]StateNodeModel, error) {
 	log.Debug("retrieving state cids for block ", blockNumber)
 	args := make([]interface{}, 0, 2)
-	pgStr := `SELECT state_cids.cid, state_cids.state_key, state_cids.leaf FROM state_cids INNER JOIN header_cids ON (state_cids.header_id = header_cids.id)
+	pgStr := `SELECT state_cids.id, state_cids.header_id,
+			state_cids.state_key, state_cids.leaf, state_cids.cid
+			FROM state_cids INNER JOIN header_cids ON (state_cids.header_id = header_cids.id)
 			WHERE header_cids.block_number = $1`
 	args = append(args, blockNumber)
 	addrLen := len(stateFilter.Addresses)
 	if addrLen > 0 {
 		keys := make([]string, 0, addrLen)
 		for _, addr := range stateFilter.Addresses {
-			keys = append(keys, ipfs.HexToKey(addr).Hex())
+			keys = append(keys, HexToKey(addr).Hex())
 		}
 		pgStr += ` AND state_cids.state_key = ANY($2::VARCHAR(66)[])`
 		args = append(args, pq.Array(keys))
@@ -278,16 +274,17 @@ func (ecr *EthCIDRetriever) RetrieveStateCIDs(tx *sqlx.Tx, stateFilter config.St
 	if !stateFilter.IntermediateNodes {
 		pgStr += ` AND state_cids.leaf = TRUE`
 	}
-	stateNodeCIDs := make([]ipfs.StateNodeCID, 0)
+	stateNodeCIDs := make([]StateNodeModel, 0)
 	err := tx.Select(&stateNodeCIDs, pgStr, args...)
 	return stateNodeCIDs, err
 }
 
 // RetrieveStorageCIDs retrieves and returns all of the storage node cids at the provided blockheight that conform to the provided filter parameters
-func (ecr *EthCIDRetriever) RetrieveStorageCIDs(tx *sqlx.Tx, storageFilter config.StorageFilter, blockNumber int64) ([]ipfs.StorageNodeCID, error) {
+func (ecr *CIDRetriever) RetrieveStorageCIDs(tx *sqlx.Tx, storageFilter config.StorageFilter, blockNumber int64) ([]StorageNodeWithStateKeyModel, error) {
 	log.Debug("retrieving storage cids for block ", blockNumber)
 	args := make([]interface{}, 0, 3)
-	pgStr := `SELECT storage_cids.cid, state_cids.state_key, storage_cids.storage_key, storage_cids.leaf FROM storage_cids, state_cids, header_cids
+	pgStr := `SELECT storage_cids.id, storage_cids.state_id, storage_cids.storage_key,
+ 			storage_cids.leaf, storage_cids.cid, state_cids.state_key FROM storage_cids, state_cids, header_cids
 			WHERE storage_cids.state_id = state_cids.id 
 			AND state_cids.header_id = header_cids.id
 			AND header_cids.block_number = $1`
@@ -296,7 +293,7 @@ func (ecr *EthCIDRetriever) RetrieveStorageCIDs(tx *sqlx.Tx, storageFilter confi
 	if addrLen > 0 {
 		keys := make([]string, 0, addrLen)
 		for _, addr := range storageFilter.Addresses {
-			keys = append(keys, ipfs.HexToKey(addr).Hex())
+			keys = append(keys, HexToKey(addr).Hex())
 		}
 		pgStr += ` AND state_cids.state_key = ANY($2::VARCHAR(66)[])`
 		args = append(args, pq.Array(keys))
@@ -311,35 +308,36 @@ func (ecr *EthCIDRetriever) RetrieveStorageCIDs(tx *sqlx.Tx, storageFilter confi
 	if !storageFilter.IntermediateNodes {
 		pgStr += ` AND storage_cids.leaf = TRUE`
 	}
-	storageNodeCIDs := make([]ipfs.StorageNodeCID, 0)
+	storageNodeCIDs := make([]StorageNodeWithStateKeyModel, 0)
 	err := tx.Select(&storageNodeCIDs, pgStr, args...)
 	return storageNodeCIDs, err
 }
 
-type gap struct {
-	Start uint64 `db:"start"`
-	Stop  uint64 `db:"stop"`
-}
-
 // RetrieveGapsInData is used to find the the block numbers at which we are missing data in the db
-func (ecr *EthCIDRetriever) RetrieveGapsInData() ([][2]uint64, error) {
+func (ecr *CIDRetriever) RetrieveGapsInData() ([]shared.Gap, error) {
 	pgStr := `SELECT header_cids.block_number + 1 AS start, min(fr.block_number) - 1 AS stop FROM header_cids
 				LEFT JOIN header_cids r on header_cids.block_number = r.block_number - 1
 				LEFT JOIN header_cids fr on header_cids.block_number < fr.block_number
 				WHERE r.block_number is NULL and fr.block_number IS NOT NULL
 				GROUP BY header_cids.block_number, r.block_number`
-	gaps := make([]gap, 0)
+	results := make([]struct {
-	err := ecr.db.Select(&gaps, pgStr)
+		Start uint64 `db:"start"`
+		Stop  uint64 `db:"stop"`
+	}, 0)
+	err := ecr.db.Select(&results, pgStr)
 	if err != nil {
 		return nil, err
 	}
-	gapRanges := make([][2]uint64, 0)
+	gaps := make([]shared.Gap, len(results))
-	for _, gap := range gaps {
+	for i, res := range results {
-		gapRanges = append(gapRanges, [2]uint64{gap.Start, gap.Stop})
+		gaps[i] = shared.Gap{
+			Start: res.Start,
+			Stop:  res.Stop,
+		}
 	}
-	return gapRanges, nil
+	return gaps, nil
 }
 
-func (ecr *EthCIDRetriever) Database() *postgres.DB {
+func (ecr *CIDRetriever) Database() *postgres.DB {
 	return ecr.db
 }

pkg/super_node/eth/retriever_test.go

@@ -14,39 +14,40 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package super_node_test
+package eth_test
 
 import (
 	"math/big"
 
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
+
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	"github.com/vulcanize/vulcanizedb/pkg/config"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
-	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/config"
-	"github.com/vulcanize/vulcanizedb/pkg/ipfs/mocks"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
-	"github.com/vulcanize/vulcanizedb/pkg/super_node"
+	eth2 "github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth/mocks"
 )
 
 var (
-	retriever  super_node.CIDRetriever
+	openFilter = &config.EthSubscription{
-	openFilter = config.Subscription{
+		Start:         big.NewInt(0),
-		StartingBlock: big.NewInt(0),
+		End:           big.NewInt(1),
-		EndingBlock:   big.NewInt(1),
 		HeaderFilter:  config.HeaderFilter{},
-		TrxFilter:     config.TrxFilter{},
+		TxFilter:      config.TxFilter{},
 		ReceiptFilter: config.ReceiptFilter{},
 		StateFilter:   config.StateFilter{},
 		StorageFilter: config.StorageFilter{},
 	}
-	rctContractFilter = config.Subscription{
+	rctContractFilter = &config.EthSubscription{
-		StartingBlock: big.NewInt(0),
+		Start: big.NewInt(0),
-		EndingBlock:   big.NewInt(1),
+		End:   big.NewInt(1),
 		HeaderFilter: config.HeaderFilter{
 			Off: true,
 		},
-		TrxFilter: config.TrxFilter{
+		TxFilter: config.TxFilter{
 			Off: true,
 		},
 		ReceiptFilter: config.ReceiptFilter{
@@ -59,13 +60,13 @@ var (
 			Off: true,
 		},
 	}
-	rctTopicsFilter = config.Subscription{
+	rctTopicsFilter = &config.EthSubscription{
-		StartingBlock: big.NewInt(0),
+		Start: big.NewInt(0),
-		EndingBlock:   big.NewInt(1),
+		End:   big.NewInt(1),
 		HeaderFilter: config.HeaderFilter{
 			Off: true,
 		},
-		TrxFilter: config.TrxFilter{
+		TxFilter: config.TxFilter{
 			Off: true,
 		},
 		ReceiptFilter: config.ReceiptFilter{
@@ -78,13 +79,13 @@ var (
 			Off: true,
 		},
 	}
-	rctTopicsAndContractFilter = config.Subscription{
+	rctTopicsAndContractFilter = &config.EthSubscription{
-		StartingBlock: big.NewInt(0),
+		Start: big.NewInt(0),
-		EndingBlock:   big.NewInt(1),
+		End:   big.NewInt(1),
 		HeaderFilter: config.HeaderFilter{
 			Off: true,
 		},
-		TrxFilter: config.TrxFilter{
+		TxFilter: config.TxFilter{
 			Off: true,
 		},
 		ReceiptFilter: config.ReceiptFilter{
@@ -98,13 +99,13 @@ var (
 			Off: true,
 		},
 	}
-	rctContractsAndTopicFilter = config.Subscription{
+	rctContractsAndTopicFilter = &config.EthSubscription{
-		StartingBlock: big.NewInt(0),
+		Start: big.NewInt(0),
-		EndingBlock:   big.NewInt(1),
+		End:   big.NewInt(1),
 		HeaderFilter: config.HeaderFilter{
 			Off: true,
 		},
-		TrxFilter: config.TrxFilter{
+		TxFilter: config.TxFilter{
 			Off: true,
 		},
 		ReceiptFilter: config.ReceiptFilter{
@@ -118,13 +119,13 @@ var (
 			Off: true,
 		},
 	}
-	rctsForAllCollectedTrxs = config.Subscription{
+	rctsForAllCollectedTrxs = &config.EthSubscription{
-		StartingBlock: big.NewInt(0),
+		Start: big.NewInt(0),
-		EndingBlock:   big.NewInt(1),
+		End:   big.NewInt(1),
 		HeaderFilter: config.HeaderFilter{
 			Off: true,
 		},
-		TrxFilter: config.TrxFilter{}, // Trx filter open so we will collect all trxs, therefore we will also collect all corresponding rcts despite rct filter
+		TxFilter: config.TxFilter{}, // Trx filter open so we will collect all trxs, therefore we will also collect all corresponding rcts despite rct filter
 		ReceiptFilter: config.ReceiptFilter{
 			MatchTxs:  true,
 			Topic0s:   []string{"0x0000000000000000000000000000000000000000000000000000000000000006"}, // Topic isn't one of the topics we have
@@ -137,13 +138,13 @@ var (
 			Off: true,
 		},
 	}
-	rctsForSelectCollectedTrxs = config.Subscription{
+	rctsForSelectCollectedTrxs = &config.EthSubscription{
-		StartingBlock: big.NewInt(0),
+		Start: big.NewInt(0),
-		EndingBlock:   big.NewInt(1),
+		End:   big.NewInt(1),
 		HeaderFilter: config.HeaderFilter{
 			Off: true,
 		},
-		TrxFilter: config.TrxFilter{
+		TxFilter: config.TxFilter{
 			Dst: []string{"0x0000000000000000000000000000000000000001"}, // We only filter for one of the trxs so we will only get the one corresponding receipt
 		},
 		ReceiptFilter: config.ReceiptFilter{
@@ -158,13 +159,13 @@ var (
 			Off: true,
 		},
 	}
-	stateFilter = config.Subscription{
+	stateFilter = &config.EthSubscription{
-		StartingBlock: big.NewInt(0),
+		Start: big.NewInt(0),
-		EndingBlock:   big.NewInt(1),
+		End:   big.NewInt(1),
 		HeaderFilter: config.HeaderFilter{
 			Off: true,
 		},
-		TrxFilter: config.TrxFilter{
+		TxFilter: config.TxFilter{
 			Off: true,
 		},
 		ReceiptFilter: config.ReceiptFilter{
@@ -181,129 +182,179 @@ var (
 
 var _ = Describe("Retriever", func() {
 	var (
-		db   *postgres.DB
+		db        *postgres.DB
-		repo super_node.CIDRepository
+		repo      *eth2.CIDIndexer
+		retriever *eth2.CIDRetriever
 	)
 	BeforeEach(func() {
 		var err error
-		db, err = super_node.SetupDB()
+		db, err = eth.SetupDB()
 		Expect(err).ToNot(HaveOccurred())
-		repo = super_node.NewCIDRepository(db)
+		repo = eth2.NewCIDIndexer(db)
-		retriever = super_node.NewCIDRetriever(db)
+		retriever = eth2.NewCIDRetriever(db)
 	})
 	AfterEach(func() {
-		super_node.TearDownDB(db)
+		eth.TearDownDB(db)
 	})
 
-	Describe("RetrieveCIDs", func() {
+	Describe("Retrieve", func() {
 		BeforeEach(func() {
 			err := repo.Index(mocks.MockCIDPayload)
 			Expect(err).ToNot(HaveOccurred())
 		})
 		It("Retrieves all CIDs for the given blocknumber when provided an open filter", func() {
-			cidWrapper, err := retriever.RetrieveCIDs(openFilter, 1)
+			cids, empty, err := retriever.Retrieve(openFilter, 1)
 			Expect(err).ToNot(HaveOccurred())
+			Expect(empty).ToNot(BeTrue())
+			cidWrapper, ok := cids.(*eth.CIDWrapper)
+			Expect(ok).To(BeTrue())
 			Expect(cidWrapper.BlockNumber).To(Equal(mocks.MockCIDWrapper.BlockNumber))
 			Expect(len(cidWrapper.Headers)).To(Equal(1))
-			Expect(cidWrapper.Headers).To(Equal(mocks.MockCIDWrapper.Headers))
+			expectedHeaderCIDs := mocks.MockCIDWrapper.Headers
+			expectedHeaderCIDs[0].ID = cidWrapper.Headers[0].ID
+			Expect(cidWrapper.Headers).To(Equal(expectedHeaderCIDs))
 			Expect(len(cidWrapper.Transactions)).To(Equal(2))
-			Expect(super_node.ListContainsString(cidWrapper.Transactions, mocks.MockCIDWrapper.Transactions[0])).To(BeTrue())
+			Expect(eth.TxModelsContainsCID(cidWrapper.Transactions, mocks.MockCIDWrapper.Transactions[0].CID)).To(BeTrue())
-			Expect(super_node.ListContainsString(cidWrapper.Transactions, mocks.MockCIDWrapper.Transactions[1])).To(BeTrue())
+			Expect(eth.TxModelsContainsCID(cidWrapper.Transactions, mocks.MockCIDWrapper.Transactions[1].CID)).To(BeTrue())
 			Expect(len(cidWrapper.Receipts)).To(Equal(2))
-			Expect(super_node.ListContainsString(cidWrapper.Receipts, mocks.MockCIDWrapper.Receipts[0])).To(BeTrue())
+			Expect(eth.ReceiptModelsContainsCID(cidWrapper.Receipts, mocks.MockCIDWrapper.Receipts[0].CID)).To(BeTrue())
-			Expect(super_node.ListContainsString(cidWrapper.Receipts, mocks.MockCIDWrapper.Receipts[1])).To(BeTrue())
+			Expect(eth.ReceiptModelsContainsCID(cidWrapper.Receipts, mocks.MockCIDWrapper.Receipts[1].CID)).To(BeTrue())
 			Expect(len(cidWrapper.StateNodes)).To(Equal(2))
 			for _, stateNode := range cidWrapper.StateNodes {
 				if stateNode.CID == "mockStateCID1" {
-					Expect(stateNode.Key).To(Equal(mocks.ContractLeafKey.Hex()))
+					Expect(stateNode.StateKey).To(Equal(mocks.ContractLeafKey.Hex()))
 					Expect(stateNode.Leaf).To(Equal(true))
 				}
 				if stateNode.CID == "mockStateCID2" {
-					Expect(stateNode.Key).To(Equal(mocks.AnotherContractLeafKey.Hex()))
+					Expect(stateNode.StateKey).To(Equal(mocks.AnotherContractLeafKey.Hex()))
 					Expect(stateNode.Leaf).To(Equal(true))
 				}
 			}
 			Expect(len(cidWrapper.StorageNodes)).To(Equal(1))
-			Expect(cidWrapper.StorageNodes).To(Equal(mocks.MockCIDWrapper.StorageNodes))
+			expectedStorageNodeCIDs := mocks.MockCIDWrapper.StorageNodes
+			expectedStorageNodeCIDs[0].ID = cidWrapper.StorageNodes[0].ID
+			expectedStorageNodeCIDs[0].StateID = cidWrapper.StorageNodes[0].StateID
+			Expect(cidWrapper.StorageNodes).To(Equal(expectedStorageNodeCIDs))
 		})
 
 		It("Applies filters from the provided config.Subscription", func() {
-			cidWrapper1, err1 := retriever.RetrieveCIDs(rctContractFilter, 1)
+			cids1, empty, err := retriever.Retrieve(rctContractFilter, 1)
-			Expect(err1).ToNot(HaveOccurred())
+			Expect(err).ToNot(HaveOccurred())
+			Expect(empty).ToNot(BeTrue())
+			cidWrapper1, ok := cids1.(*eth.CIDWrapper)
+			Expect(ok).To(BeTrue())
 			Expect(cidWrapper1.BlockNumber).To(Equal(mocks.MockCIDWrapper.BlockNumber))
 			Expect(len(cidWrapper1.Headers)).To(Equal(0))
 			Expect(len(cidWrapper1.Transactions)).To(Equal(0))
 			Expect(len(cidWrapper1.StateNodes)).To(Equal(0))
 			Expect(len(cidWrapper1.StorageNodes)).To(Equal(0))
 			Expect(len(cidWrapper1.Receipts)).To(Equal(1))
-			Expect(cidWrapper1.Receipts[0]).To(Equal("mockRctCID2"))
+			expectedReceiptCID := mocks.MockCIDWrapper.Receipts[1]
+			expectedReceiptCID.ID = cidWrapper1.Receipts[0].ID
+			expectedReceiptCID.TxID = cidWrapper1.Receipts[0].TxID
+			Expect(cidWrapper1.Receipts[0]).To(Equal(expectedReceiptCID))
 
-			cidWrapper2, err2 := retriever.RetrieveCIDs(rctTopicsFilter, 1)
+			cids2, empty, err := retriever.Retrieve(rctTopicsFilter, 1)
-			Expect(err2).ToNot(HaveOccurred())
+			Expect(err).ToNot(HaveOccurred())
+			Expect(empty).ToNot(BeTrue())
+			cidWrapper2, ok := cids2.(*eth.CIDWrapper)
+			Expect(ok).To(BeTrue())
 			Expect(cidWrapper2.BlockNumber).To(Equal(mocks.MockCIDWrapper.BlockNumber))
 			Expect(len(cidWrapper2.Headers)).To(Equal(0))
 			Expect(len(cidWrapper2.Transactions)).To(Equal(0))
 			Expect(len(cidWrapper2.StateNodes)).To(Equal(0))
 			Expect(len(cidWrapper2.StorageNodes)).To(Equal(0))
 			Expect(len(cidWrapper2.Receipts)).To(Equal(1))
-			Expect(cidWrapper2.Receipts[0]).To(Equal("mockRctCID1"))
+			expectedReceiptCID = mocks.MockCIDWrapper.Receipts[0]
+			expectedReceiptCID.ID = cidWrapper2.Receipts[0].ID
+			expectedReceiptCID.TxID = cidWrapper2.Receipts[0].TxID
+			Expect(cidWrapper2.Receipts[0]).To(Equal(expectedReceiptCID))
 
-			cidWrapper3, err3 := retriever.RetrieveCIDs(rctTopicsAndContractFilter, 1)
+			cids3, empty, err := retriever.Retrieve(rctTopicsAndContractFilter, 1)
-			Expect(err3).ToNot(HaveOccurred())
+			Expect(err).ToNot(HaveOccurred())
+			Expect(empty).ToNot(BeTrue())
+			cidWrapper3, ok := cids3.(*eth.CIDWrapper)
+			Expect(ok).To(BeTrue())
 			Expect(cidWrapper3.BlockNumber).To(Equal(mocks.MockCIDWrapper.BlockNumber))
 			Expect(len(cidWrapper3.Headers)).To(Equal(0))
 			Expect(len(cidWrapper3.Transactions)).To(Equal(0))
 			Expect(len(cidWrapper3.StateNodes)).To(Equal(0))
 			Expect(len(cidWrapper3.StorageNodes)).To(Equal(0))
 			Expect(len(cidWrapper3.Receipts)).To(Equal(1))
-			Expect(cidWrapper3.Receipts[0]).To(Equal("mockRctCID1"))
+			expectedReceiptCID = mocks.MockCIDWrapper.Receipts[0]
+			expectedReceiptCID.ID = cidWrapper3.Receipts[0].ID
+			expectedReceiptCID.TxID = cidWrapper3.Receipts[0].TxID
+			Expect(cidWrapper3.Receipts[0]).To(Equal(expectedReceiptCID))
 
-			cidWrapper4, err4 := retriever.RetrieveCIDs(rctContractsAndTopicFilter, 1)
+			cids4, empty, err := retriever.Retrieve(rctContractsAndTopicFilter, 1)
-			Expect(err4).ToNot(HaveOccurred())
+			Expect(err).ToNot(HaveOccurred())
+			Expect(empty).ToNot(BeTrue())
+			cidWrapper4, ok := cids4.(*eth.CIDWrapper)
+			Expect(ok).To(BeTrue())
 			Expect(cidWrapper4.BlockNumber).To(Equal(mocks.MockCIDWrapper.BlockNumber))
 			Expect(len(cidWrapper4.Headers)).To(Equal(0))
 			Expect(len(cidWrapper4.Transactions)).To(Equal(0))
 			Expect(len(cidWrapper4.StateNodes)).To(Equal(0))
 			Expect(len(cidWrapper4.StorageNodes)).To(Equal(0))
 			Expect(len(cidWrapper4.Receipts)).To(Equal(1))
-			Expect(cidWrapper4.Receipts[0]).To(Equal("mockRctCID2"))
+			expectedReceiptCID = mocks.MockCIDWrapper.Receipts[1]
+			expectedReceiptCID.ID = cidWrapper4.Receipts[0].ID
+			expectedReceiptCID.TxID = cidWrapper4.Receipts[0].TxID
+			Expect(cidWrapper4.Receipts[0]).To(Equal(expectedReceiptCID))
 
-			cidWrapper5, err5 := retriever.RetrieveCIDs(rctsForAllCollectedTrxs, 1)
+			cids5, empty, err := retriever.Retrieve(rctsForAllCollectedTrxs, 1)
-			Expect(err5).ToNot(HaveOccurred())
+			Expect(err).ToNot(HaveOccurred())
+			Expect(empty).ToNot(BeTrue())
+			cidWrapper5, ok := cids5.(*eth.CIDWrapper)
+			Expect(ok).To(BeTrue())
 			Expect(cidWrapper5.BlockNumber).To(Equal(mocks.MockCIDWrapper.BlockNumber))
 			Expect(len(cidWrapper5.Headers)).To(Equal(0))
 			Expect(len(cidWrapper5.Transactions)).To(Equal(2))
-			Expect(super_node.ListContainsString(cidWrapper5.Transactions, "mockTrxCID1")).To(BeTrue())
+			Expect(eth.TxModelsContainsCID(cidWrapper5.Transactions, "mockTrxCID1")).To(BeTrue())
-			Expect(super_node.ListContainsString(cidWrapper5.Transactions, "mockTrxCID2")).To(BeTrue())
+			Expect(eth.TxModelsContainsCID(cidWrapper5.Transactions, "mockTrxCID2")).To(BeTrue())
 			Expect(len(cidWrapper5.StateNodes)).To(Equal(0))
 			Expect(len(cidWrapper5.StorageNodes)).To(Equal(0))
 			Expect(len(cidWrapper5.Receipts)).To(Equal(2))
-			Expect(super_node.ListContainsString(cidWrapper5.Receipts, "mockRctCID1")).To(BeTrue())
+			Expect(eth.ReceiptModelsContainsCID(cidWrapper5.Receipts, "mockRctCID1")).To(BeTrue())
-			Expect(super_node.ListContainsString(cidWrapper5.Receipts, "mockRctCID2")).To(BeTrue())
+			Expect(eth.ReceiptModelsContainsCID(cidWrapper5.Receipts, "mockRctCID2")).To(BeTrue())
 
-			cidWrapper6, err6 := retriever.RetrieveCIDs(rctsForSelectCollectedTrxs, 1)
+			cids6, empty, err := retriever.Retrieve(rctsForSelectCollectedTrxs, 1)
-			Expect(err6).ToNot(HaveOccurred())
+			Expect(err).ToNot(HaveOccurred())
+			Expect(empty).ToNot(BeTrue())
+			cidWrapper6, ok := cids6.(*eth.CIDWrapper)
+			Expect(ok).To(BeTrue())
 			Expect(cidWrapper6.BlockNumber).To(Equal(mocks.MockCIDWrapper.BlockNumber))
 			Expect(len(cidWrapper6.Headers)).To(Equal(0))
 			Expect(len(cidWrapper6.Transactions)).To(Equal(1))
-			Expect(cidWrapper6.Transactions[0]).To(Equal("mockTrxCID2"))
+			expectedTxCID := mocks.MockCIDWrapper.Transactions[1]
+			expectedTxCID.ID = cidWrapper6.Transactions[0].ID
+			expectedTxCID.HeaderID = cidWrapper6.Transactions[0].HeaderID
+			Expect(cidWrapper6.Transactions[0]).To(Equal(expectedTxCID))
 			Expect(len(cidWrapper6.StateNodes)).To(Equal(0))
 			Expect(len(cidWrapper6.StorageNodes)).To(Equal(0))
 			Expect(len(cidWrapper6.Receipts)).To(Equal(1))
-			Expect(cidWrapper6.Receipts[0]).To(Equal("mockRctCID2"))
+			expectedReceiptCID = mocks.MockCIDWrapper.Receipts[1]
+			expectedReceiptCID.ID = cidWrapper6.Receipts[0].ID
+			expectedReceiptCID.TxID = cidWrapper6.Receipts[0].TxID
+			Expect(cidWrapper6.Receipts[0]).To(Equal(expectedReceiptCID))
 
-			cidWrapper7, err7 := retriever.RetrieveCIDs(stateFilter, 1)
+			cids7, empty, err := retriever.Retrieve(stateFilter, 1)
-			Expect(err7).ToNot(HaveOccurred())
+			Expect(err).ToNot(HaveOccurred())
+			Expect(empty).ToNot(BeTrue())
+			cidWrapper7, ok := cids7.(*eth.CIDWrapper)
+			Expect(ok).To(BeTrue())
 			Expect(cidWrapper7.BlockNumber).To(Equal(mocks.MockCIDWrapper.BlockNumber))
 			Expect(len(cidWrapper7.Headers)).To(Equal(0))
 			Expect(len(cidWrapper7.Transactions)).To(Equal(0))
 			Expect(len(cidWrapper7.Receipts)).To(Equal(0))
 			Expect(len(cidWrapper7.StorageNodes)).To(Equal(0))
 			Expect(len(cidWrapper7.StateNodes)).To(Equal(1))
-			Expect(cidWrapper7.StateNodes[0]).To(Equal(ipfs.StateNodeCID{
+			Expect(cidWrapper7.StateNodes[0]).To(Equal(eth.StateNodeModel{
-				Leaf: true,
+				ID:       cidWrapper7.StateNodes[0].ID,
-				Key:  mocks.ContractLeafKey.Hex(),
+				HeaderID: cidWrapper7.StateNodes[0].HeaderID,
-				CID:  "mockStateCID1",
+				Leaf:     true,
+				StateKey: mocks.ContractLeafKey.Hex(),
+				CID:      "mockStateCID1",
 			}))
 		})
 	})
@@ -319,7 +370,7 @@ var _ = Describe("Retriever", func() {
 
 		It("Gets the number of the first block that has data in the database", func() {
 			payload := *mocks.MockCIDPayload
-			payload.BlockNumber = "1010101"
+			payload.HeaderCID.BlockNumber = "1010101"
 			err := repo.Index(&payload)
 			Expect(err).ToNot(HaveOccurred())
 			num, err := retriever.RetrieveFirstBlockNumber()
@@ -329,9 +380,9 @@ var _ = Describe("Retriever", func() {
 
 		It("Gets the number of the first block that has data in the database", func() {
 			payload1 := *mocks.MockCIDPayload
-			payload1.BlockNumber = "1010101"
+			payload1.HeaderCID.BlockNumber = "1010101"
 			payload2 := payload1
-			payload2.BlockNumber = "5"
+			payload2.HeaderCID.BlockNumber = "5"
 			err := repo.Index(&payload1)
 			Expect(err).ToNot(HaveOccurred())
 			err = repo.Index(&payload2)
@@ -353,7 +404,7 @@ var _ = Describe("Retriever", func() {
 
 		It("Gets the number of the latest block that has data in the database", func() {
 			payload := *mocks.MockCIDPayload
-			payload.BlockNumber = "1010101"
+			payload.HeaderCID.BlockNumber = "1010101"
 			err := repo.Index(&payload)
 			Expect(err).ToNot(HaveOccurred())
 			num, err := retriever.RetrieveLastBlockNumber()
@@ -363,9 +414,9 @@ var _ = Describe("Retriever", func() {
 
 		It("Gets the number of the latest block that has data in the database", func() {
 			payload1 := *mocks.MockCIDPayload
-			payload1.BlockNumber = "1010101"
+			payload1.HeaderCID.BlockNumber = "1010101"
 			payload2 := payload1
-			payload2.BlockNumber = "5"
+			payload2.HeaderCID.BlockNumber = "5"
 			err := repo.Index(&payload1)
 			Expect(err).ToNot(HaveOccurred())
 			err = repo.Index(&payload2)
@@ -379,9 +430,9 @@ var _ = Describe("Retriever", func() {
 	Describe("RetrieveGapsInData", func() {
 		It("Doesn't return gaps if there are none", func() {
 			payload1 := *mocks.MockCIDPayload
-			payload1.BlockNumber = "2"
+			payload1.HeaderCID.BlockNumber = "2"
 			payload2 := payload1
-			payload2.BlockNumber = "3"
+			payload2.HeaderCID.BlockNumber = "3"
 			err := repo.Index(mocks.MockCIDPayload)
 			Expect(err).ToNot(HaveOccurred())
 			err = repo.Index(&payload1)
@@ -395,7 +446,7 @@ var _ = Describe("Retriever", func() {
 
 		It("Doesn't return the gap from 0 to the earliest block", func() {
 			payload := *mocks.MockCIDPayload
-			payload.BlockNumber = "5"
+			payload.HeaderCID.BlockNumber = "5"
 			err := repo.Index(&payload)
 			Expect(err).ToNot(HaveOccurred())
 			gaps, err := retriever.RetrieveGapsInData()
@@ -405,9 +456,9 @@ var _ = Describe("Retriever", func() {
 
 		It("Finds gap between two entries", func() {
 			payload1 := *mocks.MockCIDPayload
-			payload1.BlockNumber = "1010101"
+			payload1.HeaderCID.BlockNumber = "1010101"
 			payload2 := payload1
-			payload2.BlockNumber = "5"
+			payload2.HeaderCID.BlockNumber = "5"
 			err := repo.Index(&payload1)
 			Expect(err).ToNot(HaveOccurred())
 			err = repo.Index(&payload2)
@@ -415,23 +466,23 @@ var _ = Describe("Retriever", func() {
 			gaps, err := retriever.RetrieveGapsInData()
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(gaps)).To(Equal(1))
-			Expect(gaps[0][0]).To(Equal(uint64(6)))
+			Expect(gaps[0].Start).To(Equal(uint64(6)))
-			Expect(gaps[0][1]).To(Equal(uint64(1010100)))
+			Expect(gaps[0].Stop).To(Equal(uint64(1010100)))
 		})
 
 		It("Finds gaps between multiple entries", func() {
 			payload1 := *mocks.MockCIDPayload
-			payload1.BlockNumber = "1010101"
+			payload1.HeaderCID.BlockNumber = "1010101"
 			payload2 := payload1
-			payload2.BlockNumber = "5"
+			payload2.HeaderCID.BlockNumber = "5"
 			payload3 := payload2
-			payload3.BlockNumber = "100"
+			payload3.HeaderCID.BlockNumber = "100"
 			payload4 := payload3
-			payload4.BlockNumber = "101"
+			payload4.HeaderCID.BlockNumber = "101"
 			payload5 := payload4
-			payload5.BlockNumber = "102"
+			payload5.HeaderCID.BlockNumber = "102"
 			payload6 := payload5
-			payload6.BlockNumber = "1000"
+			payload6.HeaderCID.BlockNumber = "1000"
 			err := repo.Index(&payload1)
 			Expect(err).ToNot(HaveOccurred())
 			err = repo.Index(&payload2)
@@ -447,9 +498,9 @@ var _ = Describe("Retriever", func() {
 			gaps, err := retriever.RetrieveGapsInData()
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(gaps)).To(Equal(3))
-			Expect(super_node.ListContainsRange(gaps, [2]uint64{6, 99})).To(BeTrue())
+			Expect(shared.ListContainsGap(gaps, shared.Gap{Start: 6, Stop: 99})).To(BeTrue())
-			Expect(super_node.ListContainsRange(gaps, [2]uint64{103, 999})).To(BeTrue())
+			Expect(shared.ListContainsGap(gaps, shared.Gap{Start: 103, Stop: 999})).To(BeTrue())
-			Expect(super_node.ListContainsRange(gaps, [2]uint64{1001, 1010100})).To(BeTrue())
+			Expect(shared.ListContainsGap(gaps, shared.Gap{Start: 1001, Stop: 1010100})).To(BeTrue())
 		})
 	})
 })

pkg/super_node/eth/streamer.go

@@ -0,0 +1,46 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package eth
+
+import (
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/sirupsen/logrus"
+
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+)
+
+const (
+	PayloadChanBufferSize = 20000 // the max eth sub buffer size
+)
+
+// PayloadStreamer satisfies the PayloadStreamer interface for ethereum
+type PayloadStreamer struct {
+	Client core.RPCClient
+}
+
+// NewPayloadStreamer creates a pointer to a new StateDiffStreamer which satisfies the PayloadStreamer interface
+func NewPayloadStreamer(client core.RPCClient) *PayloadStreamer {
+	return &PayloadStreamer{
+		Client: client,
+	}
+}
+
+// Stream is the main loop for subscribing to data from the Geth state diff process
+func (sds *PayloadStreamer) Stream(payloadChan chan interface{}) (*rpc.ClientSubscription, error) {
+	logrus.Info("streaming diffs from geth")
+	return sds.Client.Subscribe("statediff", payloadChan, "stream")
+}

pkg/super_node/eth/streamer_test.go

@@ -0,0 +1,34 @@
+// Copyright 2019 Vulcanize
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//      http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package eth_test
+
+import (
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/pkg/fakes"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/eth"
+)
+
+var _ = Describe("StateDiff Streamer", func() {
+	It("subscribes to the geth statediff service", func() {
+		client := &fakes.MockRPCClient{}
+		streamer := eth.NewPayloadStreamer(client)
+		payloadChan := make(chan interface{})
+		_, err := streamer.Stream(payloadChan)
+		Expect(err).NotTo(HaveOccurred())
+		client.AssertSubscribeCalledWith("statediff", payloadChan, []interface{}{"stream"})
+	})
+})

pkg/super_node/eth/test_helpers.go

@@ -14,11 +14,9 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package super_node
+package eth
 
 import (
-	"bytes"
-
 	. "github.com/onsi/gomega"
 
 	"github.com/vulcanize/vulcanizedb/pkg/config"
@@ -57,10 +55,10 @@ func TearDownDB(db *postgres.DB) {
 	Expect(err).NotTo(HaveOccurred())
 }
 
-// ListContainsString used to check if a list of strings contains a particular string
+// TxModelsContainsCID used to check if a list of TxModels contains a specific cid string
-func ListContainsString(sss []string, s string) bool {
+func TxModelsContainsCID(txs []TxModel, cid string) bool {
-	for _, str := range sss {
+	for _, tx := range txs {
-		if s == str {
+		if tx.CID == cid {
 			return true
 		}
 	}
@@ -68,9 +66,9 @@ func ListContainsString(sss []string, s string) bool {
 }
 
 // ListContainsBytes used to check if a list of byte arrays contains a particular byte array
-func ListContainsBytes(bbb [][]byte, b []byte) bool {
+func ReceiptModelsContainsCID(rcts []ReceiptModel, cid string) bool {
-	for _, by := range bbb {
+	for _, rct := range rcts {
-		if bytes.Equal(by, b) {
+		if rct.CID == cid {
 			return true
 		}
 	}

pkg/super_node/eth/types.go

@@ -0,0 +1,119 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package eth
+
+import (
+	"encoding/json"
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ipfs/go-block-format"
+)
+
+// IPLDPayload is a custom type which packages raw ETH data for publishing to IPFS and filtering to subscribers
+// Returned by PayloadConverter
+// Passed to IPLDPublisher and ResponseFilterer
+type IPLDPayload struct {
+	TotalDifficulty *big.Int
+	Block           *types.Block
+	HeaderRLP       []byte
+	TrxMetaData     []TxModel
+	Receipts        types.Receipts
+	ReceiptMetaData []ReceiptModel
+	StateNodes      []TrieNode
+	StorageNodes    map[common.Hash][]TrieNode
+}
+
+// Trie struct used to flag node as leaf or not
+type TrieNode struct {
+	Key   common.Hash
+	Value []byte
+	Leaf  bool
+}
+
+// CIDPayload is a struct to hold all the CIDs and their associated meta data for indexing in Postgres
+// Returned by IPLDPublisher
+// Passed to CIDIndexer
+type CIDPayload struct {
+	HeaderCID       HeaderModel
+	UncleCIDs       []HeaderModel
+	TransactionCIDs []TxModel
+	ReceiptCIDs     map[common.Hash]ReceiptModel
+	StateNodeCIDs   []StateNodeModel
+	StorageNodeCIDs map[common.Hash][]StorageNodeModel
+}
+
+// CIDWrapper is used to direct fetching of IPLDs from IPFS
+// Returned by CIDRetriever
+// Passed to IPLDFetcher
+type CIDWrapper struct {
+	BlockNumber  *big.Int
+	Headers      []HeaderModel
+	Uncles       []HeaderModel
+	Transactions []TxModel
+	Receipts     []ReceiptModel
+	StateNodes   []StateNodeModel
+	StorageNodes []StorageNodeWithStateKeyModel
+}
+
+// IPLDWrapper is used to package raw IPLD block data fetched from IPFS
+// Returned by IPLDFetcher
+// Passed to IPLDResolver
+type IPLDWrapper struct {
+	BlockNumber  *big.Int
+	Headers      []blocks.Block
+	Uncles       []blocks.Block
+	Transactions []blocks.Block
+	Receipts     []blocks.Block
+	StateNodes   map[common.Hash]blocks.Block
+	StorageNodes map[common.Hash]map[common.Hash]blocks.Block
+}
+
+// StreamPayload holds the data streamed from the super node eth service to the requesting clients
+// Returned by IPLDResolver and ResponseFilterer
+// Passed to client subscriptions
+type StreamPayload struct {
+	BlockNumber     *big.Int                               `json:"blockNumber"`
+	HeadersRlp      [][]byte                               `json:"headersRlp"`
+	UnclesRlp       [][]byte                               `json:"unclesRlp"`
+	TransactionsRlp [][]byte                               `json:"transactionsRlp"`
+	ReceiptsRlp     [][]byte                               `json:"receiptsRlp"`
+	StateNodesRlp   map[common.Hash][]byte                 `json:"stateNodesRlp"`
+	StorageNodesRlp map[common.Hash]map[common.Hash][]byte `json:"storageNodesRlp"`
+
+	encoded []byte
+	err     error
+}
+
+func (sd *StreamPayload) ensureEncoded() {
+	if sd.encoded == nil && sd.err == nil {
+		sd.encoded, sd.err = json.Marshal(sd)
+	}
+}
+
+// Length to implement Encoder interface for StateDiff
+func (sd *StreamPayload) Length() int {
+	sd.ensureEncoded()
+	return len(sd.encoded)
+}
+
+// Encode to implement Encoder interface for StateDiff
+func (sd *StreamPayload) Encode() ([]byte, error) {
+	sd.ensureEncoded()
+	return sd.encoded, sd.err
+}

pkg/super_node/helpers.go

@@ -0,0 +1,37 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package super_node
+
+import log "github.com/sirupsen/logrus"
+
+func sendNonBlockingErr(sub Subscription, err error) {
+	log.Error(err)
+	select {
+	case sub.PayloadChan <- Payload{nil, err.Error()}:
+	default:
+		log.Infof("unable to send error to subscription %s", sub.ID)
+	}
+}
+
+func sendNonBlockingQuit(sub Subscription) {
+	select {
+	case sub.QuitChan <- true:
+		log.Infof("closing subscription %s", sub.ID)
+	default:
+		log.Infof("unable to close subscription %s; channel has no receiver", sub.ID)
+	}
+}

pkg/super_node/mocks/retriever.go

@@ -1,80 +0,0 @@
-package mocks
-
-import (
-	"github.com/jmoiron/sqlx"
-	"github.com/vulcanize/vulcanizedb/pkg/config"
-	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
-	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
-)
-
-// MockCIDRetriever is a mock CID retriever for use in tests
-type MockCIDRetriever struct {
-	GapsToRetrieve              [][2]uint64
-	GapsToRetrieveErr           error
-	CalledTimes                 int
-	FirstBlockNumberToReturn    int64
-	RetrieveFirstBlockNumberErr error
-}
-
-// RetrieveCIDs mock method
-func (*MockCIDRetriever) RetrieveCIDs(streamFilters config.Subscription, blockNumber int64) (*ipfs.CIDWrapper, error) {
-	panic("implement me")
-}
-
-// RetrieveHeaderCIDs mock method
-func (*MockCIDRetriever) RetrieveHeaderCIDs(tx *sqlx.Tx, blockNumber int64) ([]string, error) {
-	panic("implement me")
-
-}
-
-// RetrieveUncleCIDs mock method
-func (*MockCIDRetriever) RetrieveUncleCIDs(tx *sqlx.Tx, blockNumber int64) ([]string, error) {
-	panic("implement me")
-
-}
-
-// RetrieveTrxCIDs mock method
-func (*MockCIDRetriever) RetrieveTrxCIDs(tx *sqlx.Tx, txFilter config.TrxFilter, blockNumber int64) ([]string, []int64, error) {
-	panic("implement me")
-
-}
-
-// RetrieveRctCIDs mock method
-func (*MockCIDRetriever) RetrieveRctCIDs(tx *sqlx.Tx, rctFilter config.ReceiptFilter, blockNumber int64, trxIds []int64) ([]string, error) {
-	panic("implement me")
-
-}
-
-// RetrieveStateCIDs mock method
-func (*MockCIDRetriever) RetrieveStateCIDs(tx *sqlx.Tx, stateFilter config.StateFilter, blockNumber int64) ([]ipfs.StateNodeCID, error) {
-	panic("implement me")
-
-}
-
-// RetrieveLastBlockNumber mock method
-func (*MockCIDRetriever) RetrieveLastBlockNumber() (int64, error) {
-	panic("implement me")
-}
-
-// RetrieveFirstBlockNumber mock method
-func (mcr *MockCIDRetriever) RetrieveFirstBlockNumber() (int64, error) {
-	return mcr.FirstBlockNumberToReturn, mcr.RetrieveFirstBlockNumberErr
-}
-
-// RetrieveGapsInData mock method
-func (mcr *MockCIDRetriever) RetrieveGapsInData() ([][2]uint64, error) {
-	mcr.CalledTimes++
-	return mcr.GapsToRetrieve, mcr.GapsToRetrieveErr
-}
-
-// SetGapsToRetrieve mock method
-func (mcr *MockCIDRetriever) SetGapsToRetrieve(gaps [][2]uint64) {
-	if mcr.GapsToRetrieve == nil {
-		mcr.GapsToRetrieve = make([][2]uint64, 0)
-	}
-	mcr.GapsToRetrieve = append(mcr.GapsToRetrieve, gaps...)
-}
-
-func (mcr *MockCIDRetriever) Database() *postgres.DB {
-	panic("implement me")
-}

pkg/super_node/repository.go

@@ -1,148 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package super_node
-
-import (
-	"github.com/jmoiron/sqlx"
-	"github.com/lib/pq"
-	log "github.com/sirupsen/logrus"
-
-	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
-	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
-)
-
-// CIDRepository is an interface for indexing ipfs.CIDPayloads
-type CIDRepository interface {
-	Index(cidPayload *ipfs.CIDPayload) error
-}
-
-// Repository is the underlying struct for the CIDRepository interface
-type Repository struct {
-	db *postgres.DB
-}
-
-// NewCIDRepository creates a new pointer to a Repository which satisfies the CIDRepository interface
-func NewCIDRepository(db *postgres.DB) *Repository {
-	return &Repository{
-		db: db,
-	}
-}
-
-// Index indexes a cidPayload in Postgres
-func (repo *Repository) Index(cidPayload *ipfs.CIDPayload) error {
-	tx, err := repo.db.Beginx()
-	if err != nil {
-		return err
-	}
-	headerID, err := repo.indexHeaderCID(tx, cidPayload.HeaderCID, cidPayload.BlockNumber, cidPayload.BlockHash.Hex(), cidPayload.TotalDifficulty)
-	if err != nil {
-		if err := tx.Rollback(); err != nil {
-			log.Error(err)
-		}
-		return err
-	}
-	for uncleHash, cid := range cidPayload.UncleCIDs {
-		err := repo.indexUncleCID(tx, cid, cidPayload.BlockNumber, uncleHash.Hex(), cidPayload.TotalDifficulty)
-		if err != nil {
-			if err := tx.Rollback(); err != nil {
-				log.Error(err)
-			}
-			return err
-		}
-	}
-	if err := repo.indexTransactionAndReceiptCIDs(tx, cidPayload, headerID); err != nil {
-		if err := tx.Rollback(); err != nil {
-			log.Error(err)
-		}
-		return err
-	}
-	if err := repo.indexStateAndStorageCIDs(tx, cidPayload, headerID); err != nil {
-		if err := tx.Rollback(); err != nil {
-			log.Error(err)
-		}
-		return err
-	}
-	return tx.Commit()
-}
-
-func (repo *Repository) indexHeaderCID(tx *sqlx.Tx, cid, blockNumber, hash, td string) (int64, error) {
-	var headerID int64
-	err := tx.QueryRowx(`INSERT INTO public.header_cids (block_number, block_hash, cid, uncle, td) VALUES ($1, $2, $3, $4, $5)
-								ON CONFLICT (block_number, block_hash) DO UPDATE SET (cid, uncle, td) = ($3, $4, $5)
-								RETURNING id`,
-		blockNumber, hash, cid, false, td).Scan(&headerID)
-	return headerID, err
-}
-
-func (repo *Repository) indexUncleCID(tx *sqlx.Tx, cid, blockNumber, hash, td string) error {
-	_, err := tx.Exec(`INSERT INTO public.header_cids (block_number, block_hash, cid, uncle, td) VALUES ($1, $2, $3, $4, $5)
-								ON CONFLICT (block_number, block_hash) DO UPDATE SET (cid, uncle, td) = ($3, $4, $5)`,
-		blockNumber, hash, cid, true, td)
-	return err
-}
-
-func (repo *Repository) indexTransactionAndReceiptCIDs(tx *sqlx.Tx, payload *ipfs.CIDPayload, headerID int64) error {
-	for hash, trxCidMeta := range payload.TransactionCIDs {
-		var txID int64
-		err := tx.QueryRowx(`INSERT INTO public.transaction_cids (header_id, tx_hash, cid, dst, src) VALUES ($1, $2, $3, $4, $5)
-									ON CONFLICT (header_id, tx_hash) DO UPDATE SET (cid, dst, src) = ($3, $4, $5)
-									RETURNING id`,
-			headerID, hash.Hex(), trxCidMeta.CID, trxCidMeta.Dst, trxCidMeta.Src).Scan(&txID)
-		if err != nil {
-			return err
-		}
-		receiptCidMeta, ok := payload.ReceiptCIDs[hash]
-		if ok {
-			if err := repo.indexReceiptCID(tx, receiptCidMeta, txID); err != nil {
-				return err
-			}
-		}
-	}
-	return nil
-}
-
-func (repo *Repository) indexReceiptCID(tx *sqlx.Tx, cidMeta *ipfs.ReceiptMetaData, txID int64) error {
-	_, err := tx.Exec(`INSERT INTO public.receipt_cids (tx_id, cid, contract, topic0s) VALUES ($1, $2, $3, $4)`,
-		txID, cidMeta.CID, cidMeta.ContractAddress, pq.Array(cidMeta.Topic0s))
-	return err
-}
-
-func (repo *Repository) indexStateAndStorageCIDs(tx *sqlx.Tx, payload *ipfs.CIDPayload, headerID int64) error {
-	for accountKey, stateCID := range payload.StateNodeCIDs {
-		var stateID int64
-		err := tx.QueryRowx(`INSERT INTO public.state_cids (header_id, state_key, cid, leaf) VALUES ($1, $2, $3, $4)
-									ON CONFLICT (header_id, state_key) DO UPDATE SET (cid, leaf) = ($3, $4)
-									RETURNING id`,
-			headerID, accountKey.Hex(), stateCID.CID, stateCID.Leaf).Scan(&stateID)
-		if err != nil {
-			return err
-		}
-		for _, storageCID := range payload.StorageNodeCIDs[accountKey] {
-			if err := repo.indexStorageCID(tx, storageCID, stateID); err != nil {
-				return err
-			}
-		}
-	}
-	return nil
-}
-
-func (repo *Repository) indexStorageCID(tx *sqlx.Tx, storageCID ipfs.StorageNodeCID, stateID int64) error {
-	_, err := tx.Exec(`INSERT INTO public.storage_cids (state_id, storage_key, cid, leaf) VALUES ($1, $2, $3, $4) 
-								   ON CONFLICT (state_id, storage_key) DO UPDATE SET (cid, leaf) = ($3, $4)`,
-		stateID, storageCID.Key, storageCID.CID, storageCID.Leaf)
-	return err
-}

pkg/super_node/service.go

@@ -17,6 +17,7 @@
 package super_node
 
 import (
+	"fmt"
 	"sync"
 
 	"github.com/ethereum/go-ethereum/common"
@@ -26,35 +27,34 @@ import (
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/rpc"
-	"github.com/ethereum/go-ethereum/statediff"
 	log "github.com/sirupsen/logrus"
 
-	"github.com/vulcanize/vulcanizedb/libraries/shared/streamer"
-	"github.com/vulcanize/vulcanizedb/pkg/config"
 	"github.com/vulcanize/vulcanizedb/pkg/core"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/config"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/shared"
 )
 
 const (
-	payloadChanBufferSize = 20000 // the max eth sub buffer size
+	PayloadChanBufferSize = 20000
 )
 
-// NodeInterface is the top level interface for streaming, converting to IPLDs, publishing,
+// SuperNode is the top level interface for streaming, converting to IPLDs, publishing,
 // and indexing all Ethereum data; screening this data; and serving it up to subscribed clients
 // This service is compatible with the Ethereum service interface (node.Service)
-type NodeInterface interface {
+type SuperNode interface {
 	// APIs(), Protocols(), Start() and Stop()
 	node.Service
 	// Main event loop for syncAndPublish processes
-	SyncAndPublish(wg *sync.WaitGroup, forwardPayloadChan chan<- ipfs.IPLDPayload, forwardQuitchan chan<- bool) error
+	SyncAndPublish(wg *sync.WaitGroup, forwardPayloadChan chan<- interface{}, forwardQuitchan chan<- bool) error
 	// Main event loop for handling client pub-sub
-	ScreenAndServe(wg *sync.WaitGroup, screenAndServePayload <-chan ipfs.IPLDPayload, screenAndServeQuit <-chan bool)
+	ScreenAndServe(wg *sync.WaitGroup, screenAndServePayload <-chan interface{}, screenAndServeQuit <-chan bool)
 	// Method to subscribe to receive state diff processing output
-	Subscribe(id rpc.ID, sub chan<- streamer.SuperNodePayload, quitChan chan<- bool, streamFilters config.Subscription)
+	Subscribe(id rpc.ID, sub chan<- Payload, quitChan chan<- bool, params SubscriptionSettings)
 	// Method to unsubscribe from state diff processing
 	Unsubscribe(id rpc.ID)
-	// Method to access the Geth node info for this service
+	// Method to access the node info for this service
 	Node() core.Node
 }
 
@@ -62,66 +62,96 @@ type NodeInterface interface {
 type Service struct {
 	// Used to sync access to the Subscriptions
 	sync.Mutex
-	// Interface for streaming statediff payloads over a geth rpc subscription
+	// Interface for streaming payloads over an rpc subscription
-	Streamer streamer.Streamer
+	Streamer shared.PayloadStreamer
-	// Interface for converting statediff payloads into ETH-IPLD object payloads
+	// Interface for converting raw payloads into IPLD object payloads
-	Converter ipfs.PayloadConverter
+	Converter shared.PayloadConverter
-	// Interface for publishing the ETH-IPLD payloads to IPFS
+	// Interface for publishing the IPLD payloads to IPFS
-	Publisher ipfs.IPLDPublisher
+	Publisher shared.IPLDPublisher
-	// Interface for indexing the CIDs of the published ETH-IPLDs in Postgres
+	// Interface for indexing the CIDs of the published IPLDs in Postgres
-	Repository CIDRepository
+	Indexer shared.CIDIndexer
 	// Interface for filtering and serving data according to subscribed clients according to their specification
-	Filterer ResponseFilterer
+	Filterer shared.ResponseFilterer
-	// Interface for fetching ETH-IPLD objects from IPFS
+	// Interface for fetching IPLD objects from IPFS
-	IPLDFetcher ipfs.IPLDFetcher
+	IPLDFetcher shared.IPLDFetcher
 	// Interface for searching and retrieving CIDs from Postgres index
-	Retriever CIDRetriever
+	Retriever shared.CIDRetriever
-	// Interface for resolving ipfs blocks to their data types
+	// Interface for resolving IPLDs to their data types
-	Resolver ipfs.IPLDResolver
+	Resolver shared.IPLDResolver
-	// Chan the processor uses to subscribe to state diff payloads from the Streamer
+	// Chan the processor uses to subscribe to payloads from the Streamer
-	PayloadChan chan statediff.Payload
+	PayloadChan chan interface{}
 	// Used to signal shutdown of the service
 	QuitChan chan bool
 	// A mapping of rpc.IDs to their subscription channels, mapped to their subscription type (hash of the StreamFilters)
 	Subscriptions map[common.Hash]map[rpc.ID]Subscription
-	// A mapping of subscription hash type to the corresponding StreamFilters
+	// A mapping of subscription params hash to the corresponding subscription params
-	SubscriptionTypes map[common.Hash]config.Subscription
+	SubscriptionTypes map[common.Hash]SubscriptionSettings
-	// Number of workers
-	WorkerPoolSize int
 	// Info for the Geth node that this super node is working with
-	GethNode core.Node
+	NodeInfo core.Node
+	// Number of publishAndIndex workers
+	WorkerPoolSize int
+	// chain type for this service
+	chain config.ChainType
+	// Path to ipfs data dir
+	ipfsPath string
+	// Underlying db
+	db *postgres.DB
 }
 
 // NewSuperNode creates a new super_node.Interface using an underlying super_node.Service struct
-func NewSuperNode(ipfsPath string, db *postgres.DB, rpcClient core.RPCClient, qc chan bool, workers int, node core.Node) (NodeInterface, error) {
+func NewSuperNode(settings *config.SuperNode) (SuperNode, error) {
-	ipfsInitErr := ipfs.InitIPFSPlugins()
+	if err := ipfs.InitIPFSPlugins(); err != nil {
-	if ipfsInitErr != nil {
+		return nil, err
-		return nil, ipfsInitErr
 	}
-	publisher, newPublisherErr := ipfs.NewIPLDPublisher(ipfsPath)
+	sn := new(Service)
-	if newPublisherErr != nil {
+	var err error
-		return nil, newPublisherErr
+	// If we are syncing, initialize the needed interfaces
+	if settings.Sync {
+		sn.Streamer, sn.PayloadChan, err = NewPayloadStreamer(settings.Chain, settings.WSClient)
+		if err != nil {
+			return nil, err
+		}
+		sn.Converter, err = NewPayloadConverter(settings.Chain, params.MainnetChainConfig)
+		if err != nil {
+			return nil, err
+		}
+		sn.Publisher, err = NewIPLDPublisher(settings.Chain, settings.IPFSPath)
+		if err != nil {
+			return nil, err
+		}
+		sn.Indexer, err = NewCIDIndexer(settings.Chain, settings.DB)
+		if err != nil {
+			return nil, err
+		}
+		sn.Filterer, err = NewResponseFilterer(settings.Chain)
+		if err != nil {
+			return nil, err
+		}
 	}
-	ipldFetcher, newFetcherErr := ipfs.NewIPLDFetcher(ipfsPath)
+	// If we are serving, initialize the needed interfaces
-	if newFetcherErr != nil {
+	if settings.Serve {
-		return nil, newFetcherErr
+		sn.Retriever, err = NewCIDRetriever(settings.Chain, settings.DB)
+		if err != nil {
+			return nil, err
+		}
+		sn.IPLDFetcher, err = NewIPLDFetcher(settings.Chain, settings.IPFSPath)
+		if err != nil {
+			return nil, err
+		}
+		sn.Resolver, err = NewIPLDResolver(settings.Chain)
+		if err != nil {
+			return nil, err
+		}
 	}
-	return &Service{
+	sn.QuitChan = settings.Quit
-		Streamer:          streamer.NewStateDiffStreamer(rpcClient),
+	sn.Subscriptions = make(map[common.Hash]map[rpc.ID]Subscription)
-		Repository:        NewCIDRepository(db),
+	sn.SubscriptionTypes = make(map[common.Hash]SubscriptionSettings)
-		Converter:         ipfs.NewPayloadConverter(params.MainnetChainConfig),
+	sn.WorkerPoolSize = settings.Workers
-		Publisher:         publisher,
+	sn.NodeInfo = settings.NodeInfo
-		Filterer:          NewResponseFilterer(),
+	sn.ipfsPath = settings.IPFSPath
-		IPLDFetcher:       ipldFetcher,
+	sn.chain = settings.Chain
-		Retriever:         NewCIDRetriever(db),
+	sn.db = settings.DB
-		Resolver:          ipfs.NewIPLDResolver(),
+	return sn, nil
-		PayloadChan:       make(chan statediff.Payload, payloadChanBufferSize),
-		QuitChan:          qc,
-		Subscriptions:     make(map[common.Hash]map[rpc.ID]Subscription),
-		SubscriptionTypes: make(map[common.Hash]config.Subscription),
-		WorkerPoolSize:    workers,
-		GethNode:          node,
-	}, nil
 }
 
 // Protocols exports the services p2p protocols, this service has none
@@ -131,7 +161,7 @@ func (sap *Service) Protocols() []p2p.Protocol {
 
 // APIs returns the RPC descriptors the super node service offers
 func (sap *Service) APIs() []rpc.API {
-	return []rpc.API{
+	apis := []rpc.API{
 		{
 			Namespace: APIName,
 			Version:   APIVersion,
@@ -139,20 +169,26 @@ func (sap *Service) APIs() []rpc.API {
 			Public:    true,
 		},
 	}
+	chainAPI, err := NewPublicAPI(sap.chain, sap.db, sap.ipfsPath)
+	if err != nil {
+		log.Error(err)
+		return apis
+	}
+	return append(apis, chainAPI)
 }
 
 // SyncAndPublish is the backend processing loop which streams data from geth, converts it to iplds, publishes them to ipfs, and indexes their cids
 // This continues on no matter if or how many subscribers there are, it then forwards the data to the ScreenAndServe() loop
 // which filters and sends relevant data to client subscriptions, if there are any
-func (sap *Service) SyncAndPublish(wg *sync.WaitGroup, screenAndServePayload chan<- ipfs.IPLDPayload, screenAndServeQuit chan<- bool) error {
+func (sap *Service) SyncAndPublish(wg *sync.WaitGroup, screenAndServePayload chan<- interface{}, screenAndServeQuit chan<- bool) error {
-	sub, streamErr := sap.Streamer.Stream(sap.PayloadChan)
+	sub, err := sap.Streamer.Stream(sap.PayloadChan)
-	if streamErr != nil {
+	if err != nil {
-		return streamErr
+		return err
 	}
 	wg.Add(1)
 
 	// Channels for forwarding data to the publishAndIndex workers
-	publishAndIndexPayload := make(chan ipfs.IPLDPayload, payloadChanBufferSize)
+	publishAndIndexPayload := make(chan interface{}, PayloadChanBufferSize)
 	publishAndIndexQuit := make(chan bool, sap.WorkerPoolSize)
 	// publishAndIndex worker pool to handle publishing and indexing concurrently, while
 	// limiting the number of Postgres connections we can possibly open so as to prevent error
@@ -163,30 +199,30 @@ func (sap *Service) SyncAndPublish(wg *sync.WaitGroup, screenAndServePayload cha
 		for {
 			select {
 			case payload := <-sap.PayloadChan:
-				ipldPayload, convertErr := sap.Converter.Convert(payload)
+				ipldPayload, err := sap.Converter.Convert(payload)
-				if convertErr != nil {
+				if err != nil {
-					log.Error(convertErr)
+					log.Error(err)
 					continue
 				}
 				// If we have a ScreenAndServe process running, forward the payload to it
 				select {
-				case screenAndServePayload <- *ipldPayload:
+				case screenAndServePayload <- ipldPayload:
 				default:
 				}
 				// Forward the payload to the publishAndIndex workers
 				select {
-				case publishAndIndexPayload <- *ipldPayload:
+				case publishAndIndexPayload <- ipldPayload:
 				default:
 				}
-			case subErr := <-sub.Err():
+			case err := <-sub.Err():
-				log.Error(subErr)
+				log.Error(err)
 			case <-sap.QuitChan:
 				// If we have a ScreenAndServe process running, forward the quit signal to it
 				select {
 				case screenAndServeQuit <- true:
 				default:
 				}
-				// Also forward a quit signal for each of the workers
+				// Also forward a quit signal for each of the publishAndIndex workers
 				for i := 0; i < sap.WorkerPoolSize; i++ {
 					select {
 					case publishAndIndexQuit <- true:
@@ -203,19 +239,18 @@ func (sap *Service) SyncAndPublish(wg *sync.WaitGroup, screenAndServePayload cha
 	return nil
 }
 
-func (sap *Service) publishAndIndex(id int, publishAndIndexPayload <-chan ipfs.IPLDPayload, publishAndIndexQuit <-chan bool) {
+func (sap *Service) publishAndIndex(id int, publishAndIndexPayload <-chan interface{}, publishAndIndexQuit <-chan bool) {
 	go func() {
 		for {
 			select {
 			case payload := <-publishAndIndexPayload:
-				cidPayload, publishErr := sap.Publisher.Publish(&payload)
+				cidPayload, err := sap.Publisher.Publish(payload)
-				if publishErr != nil {
+				if err != nil {
-					log.Errorf("worker %d error: %v", id, publishErr)
+					log.Errorf("worker %d error: %v", id, err)
 					continue
 				}
-				indexErr := sap.Repository.Index(cidPayload)
+				if err := sap.Indexer.Index(cidPayload); err != nil {
-				if indexErr != nil {
+					log.Errorf("worker %d error: %v", id, err)
-					log.Errorf("worker %d error: %v", id, indexErr)
 				}
 			case <-publishAndIndexQuit:
 				log.Infof("quiting publishAndIndex worker %d", id)
@@ -228,16 +263,13 @@ func (sap *Service) publishAndIndex(id int, publishAndIndexPayload <-chan ipfs.I
 
 // ScreenAndServe is the loop used to screen data streamed from the state diffing eth node
 // and send the appropriate portions of it to a requesting client subscription, according to their subscription configuration
-func (sap *Service) ScreenAndServe(wg *sync.WaitGroup, screenAndServePayload <-chan ipfs.IPLDPayload, screenAndServeQuit <-chan bool) {
+func (sap *Service) ScreenAndServe(wg *sync.WaitGroup, screenAndServePayload <-chan interface{}, screenAndServeQuit <-chan bool) {
 	wg.Add(1)
 	go func() {
 		for {
 			select {
 			case payload := <-screenAndServePayload:
-				sendErr := sap.sendResponse(payload)
+				sap.sendResponse(payload)
-				if sendErr != nil {
-					log.Error(sendErr)
-				}
 			case <-screenAndServeQuit:
 				log.Info("quiting ScreenAndServe process")
 				wg.Done()
@@ -248,23 +280,25 @@ func (sap *Service) ScreenAndServe(wg *sync.WaitGroup, screenAndServePayload <-c
 	log.Info("screenAndServe goroutine successfully spun up")
 }
 
-func (sap *Service) sendResponse(payload ipfs.IPLDPayload) error {
+func (sap *Service) sendResponse(payload interface{}) {
 	sap.Lock()
 	for ty, subs := range sap.Subscriptions {
 		// Retrieve the subscription parameters for this subscription type
 		subConfig, ok := sap.SubscriptionTypes[ty]
 		if !ok {
 			log.Errorf("subscription configuration for subscription type %s not available", ty.Hex())
+			sap.closeType(ty)
 			continue
 		}
-		response, filterErr := sap.Filterer.FilterResponse(subConfig, payload)
+		response, err := sap.Filterer.Filter(subConfig, payload)
-		if filterErr != nil {
+		if err != nil {
-			log.Error(filterErr)
+			log.Error(err)
+			sap.closeType(ty)
 			continue
 		}
 		for id, sub := range subs {
 			select {
-			case sub.PayloadChan <- response:
+			case sub.PayloadChan <- Payload{response, ""}:
 				log.Infof("sending super node payload to subscription %s", id)
 			default:
 				log.Infof("unable to send payload to subscription %s; channel has no receiver", id)
@@ -272,99 +306,102 @@ func (sap *Service) sendResponse(payload ipfs.IPLDPayload) error {
 		}
 	}
 	sap.Unlock()
-	return nil
 }
 
 // Subscribe is used by the API to subscribe to the service loop
-func (sap *Service) Subscribe(id rpc.ID, sub chan<- streamer.SuperNodePayload, quitChan chan<- bool, streamFilters config.Subscription) {
+// The params must be rlp serializable and satisfy the Params() interface
+func (sap *Service) Subscribe(id rpc.ID, sub chan<- Payload, quitChan chan<- bool, params SubscriptionSettings) {
 	log.Info("Subscribing to the super node service")
-	// Subscription type is defined as the hash of its content
-	// Group subscriptions by type and screen payloads once for subs of the same type
-	by, encodeErr := rlp.EncodeToBytes(streamFilters)
-	if encodeErr != nil {
-		log.Error(encodeErr)
-	}
-	subscriptionHash := crypto.Keccak256(by)
-	subscriptionType := common.BytesToHash(subscriptionHash)
 	subscription := Subscription{
+		ID:          id,
 		PayloadChan: sub,
 		QuitChan:    quitChan,
 	}
+	if params.ChainType() != sap.chain {
+		sendNonBlockingErr(subscription, fmt.Errorf("subscription %s is for chain %s, service supports chain %s", id, params.ChainType().String(), sap.chain.String()))
+		sendNonBlockingQuit(subscription)
+		return
+	}
+	// Subscription type is defined as the hash of the subscription settings
+	by, err := rlp.EncodeToBytes(params)
+	if err != nil {
+		sendNonBlockingErr(subscription, err)
+		sendNonBlockingQuit(subscription)
+		return
+	}
+	subscriptionType := crypto.Keccak256Hash(by)
 	// If the subscription requests a backfill, use the Postgres index to lookup and retrieve historical data
 	// Otherwise we only filter new data as it is streamed in from the state diffing geth node
-	if streamFilters.BackFill || streamFilters.BackFillOnly {
+	if params.HistoricalData() || params.HistoricalDataOnly() {
-		sap.backFill(subscription, id, streamFilters)
+		if err := sap.backFill(subscription, id, params); err != nil {
+			sendNonBlockingErr(subscription, err)
+			sendNonBlockingQuit(subscription)
+			return
+		}
 	}
-	if !streamFilters.BackFillOnly {
+	if !params.HistoricalDataOnly() {
+		// Add subscriber
 		sap.Lock()
 		if sap.Subscriptions[subscriptionType] == nil {
 			sap.Subscriptions[subscriptionType] = make(map[rpc.ID]Subscription)
 		}
 		sap.Subscriptions[subscriptionType][id] = subscription
-		sap.SubscriptionTypes[subscriptionType] = streamFilters
+		sap.SubscriptionTypes[subscriptionType] = params
 		sap.Unlock()
 	}
 }
 
-func (sap *Service) backFill(sub Subscription, id rpc.ID, con config.Subscription) {
+func (sap *Service) backFill(sub Subscription, id rpc.ID, params SubscriptionSettings) error {
-	log.Debug("back-filling data for id", id)
+	log.Debug("sending historical data for subscriber", id)
 	// Retrieve cached CIDs relevant to this subscriber
 	var endingBlock int64
 	var startingBlock int64
-	var retrieveFirstBlockErr error
+	var err error
-	var retrieveLastBlockErr error
+	startingBlock, err = sap.Retriever.RetrieveFirstBlockNumber()
-	startingBlock, retrieveFirstBlockErr = sap.Retriever.RetrieveFirstBlockNumber()
+	if err != nil {
-	if retrieveFirstBlockErr != nil {
+		return err
-		sub.PayloadChan <- streamer.SuperNodePayload{
-			ErrMsg: "unable to set block range start; error: " + retrieveFirstBlockErr.Error(),
-		}
 	}
-	if startingBlock < con.StartingBlock.Int64() {
+	if startingBlock < params.StartingBlock().Int64() {
-		startingBlock = con.StartingBlock.Int64()
+		startingBlock = params.StartingBlock().Int64()
 	}
-	endingBlock, retrieveLastBlockErr = sap.Retriever.RetrieveLastBlockNumber()
+	endingBlock, err = sap.Retriever.RetrieveLastBlockNumber()
-	if retrieveLastBlockErr != nil {
+	if err != nil {
-		sub.PayloadChan <- streamer.SuperNodePayload{
+		return err
-			ErrMsg: "unable to set block range end; error: " + retrieveLastBlockErr.Error(),
-		}
 	}
-	if endingBlock > con.EndingBlock.Int64() && con.EndingBlock.Int64() > 0 && con.EndingBlock.Int64() > startingBlock {
+	if endingBlock > params.EndingBlock().Int64() && params.EndingBlock().Int64() > 0 && params.EndingBlock().Int64() > startingBlock {
-		endingBlock = con.EndingBlock.Int64()
+		endingBlock = params.EndingBlock().Int64()
 	}
-	log.Debug("backfill starting block:", con.StartingBlock)
+	log.Debug("historical data starting block:", params.StartingBlock())
-	log.Debug("backfill ending block:", endingBlock)
+	log.Debug("histocial data ending block:", endingBlock)
-	// Backfilled payloads are sent concurrently to the streamed payloads, so the receiver needs to pay attention to
-	// the blocknumbers in the payloads they receive to keep things in order
-	// TODO: separate backfill into a different rpc subscription method altogether?
 	go func() {
 		for i := startingBlock; i <= endingBlock; i++ {
-			cidWrapper, retrieveCIDsErr := sap.Retriever.RetrieveCIDs(con, i)
+			cidWrapper, empty, err := sap.Retriever.Retrieve(params, i)
-			if retrieveCIDsErr != nil {
+			if err != nil {
-				sub.PayloadChan <- streamer.SuperNodePayload{
+				sendNonBlockingErr(sub, fmt.Errorf("CID Retrieval error at block %d\r%s", i, err.Error()))
-					ErrMsg: "CID retrieval error: " + retrieveCIDsErr.Error(),
-				}
 				continue
 			}
-			if ipfs.EmptyCIDWrapper(*cidWrapper) {
+			if empty {
 				continue
 			}
-			blocksWrapper, fetchIPLDsErr := sap.IPLDFetcher.FetchIPLDs(*cidWrapper)
+			blocksWrapper, err := sap.IPLDFetcher.Fetch(cidWrapper)
-			if fetchIPLDsErr != nil {
+			if err != nil {
-				log.Error(fetchIPLDsErr)
+				sendNonBlockingErr(sub, fmt.Errorf("IPLD Fetching error at block %d\r%s", i, err.Error()))
-				sub.PayloadChan <- streamer.SuperNodePayload{
+				continue
-					ErrMsg: "IPLD fetching error: " + fetchIPLDsErr.Error(),
+			}
-				}
+			backFillIplds, err := sap.Resolver.Resolve(blocksWrapper)
+			if err != nil {
+				sendNonBlockingErr(sub, fmt.Errorf("IPLD Resolving error at block %d\r%s", i, err.Error()))
 				continue
 			}
-			backFillIplds := sap.Resolver.ResolveIPLDs(*blocksWrapper)
 			select {
-			case sub.PayloadChan <- backFillIplds:
+			case sub.PayloadChan <- Payload{backFillIplds, ""}:
-				log.Infof("sending super node back-fill payload to subscription %s", id)
+				log.Infof("sending super node historical data payload to subscription %s", id)
 			default:
 				log.Infof("unable to send back-fill payload to subscription %s; channel has no receiver", id)
 			}
 		}
 	}()
+	return nil
 }
 
 // Unsubscribe is used to unsubscribe to the StateDiffingService loop
@@ -386,7 +423,7 @@ func (sap *Service) Unsubscribe(id rpc.ID) {
 func (sap *Service) Start(*p2p.Server) error {
 	log.Info("Starting super node service")
 	wg := new(sync.WaitGroup)
-	payloadChan := make(chan ipfs.IPLDPayload, payloadChanBufferSize)
+	payloadChan := make(chan interface{}, PayloadChanBufferSize)
 	quitChan := make(chan bool, 1)
 	if err := sap.SyncAndPublish(wg, payloadChan, quitChan); err != nil {
 		return err
@@ -398,29 +435,37 @@ func (sap *Service) Start(*p2p.Server) error {
 // Stop is used to close down the service
 func (sap *Service) Stop() error {
 	log.Info("Stopping super node service")
+	sap.Lock()
 	close(sap.QuitChan)
+	sap.close()
+	sap.Unlock()
 	return nil
 }
 
-// Node returns the Geth node info for this service
+// Node returns the node info for this service
 func (sap *Service) Node() core.Node {
-	return sap.GethNode
+	return sap.NodeInfo
 }
 
 // close is used to close all listening subscriptions
+// close needs to be called with subscription access locked
 func (sap *Service) close() {
-	sap.Lock()
+	for subType, subs := range sap.Subscriptions {
-	for ty, subs := range sap.Subscriptions {
+		for _, sub := range subs {
-		for id, sub := range subs {
+			sendNonBlockingQuit(sub)
-			select {
-			case sub.QuitChan <- true:
-				log.Infof("closing subscription %s", id)
-			default:
-				log.Infof("unable to close subscription %s; channel has no receiver", id)
-			}
 		}
-		delete(sap.Subscriptions, ty)
+		delete(sap.Subscriptions, subType)
-		delete(sap.SubscriptionTypes, ty)
+		delete(sap.SubscriptionTypes, subType)
 	}
-	sap.Unlock()
+}
+
+// closeType is used to close all subscriptions of given type
+// closeType needs to be called with subscription access locked
+func (sap *Service) closeType(subType common.Hash) {
+	subs := sap.Subscriptions[subType]
+	for _, sub := range subs {
+		sendNonBlockingQuit(sub)
+	}
+	delete(sap.Subscriptions, subType)
+	delete(sap.SubscriptionTypes, subType)
 }

pkg/super_node/service_test.go

@@ -25,38 +25,36 @@ import (
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	mocks2 "github.com/vulcanize/vulcanizedb/libraries/shared/mocks"
-	"github.com/vulcanize/vulcanizedb/pkg/ipfs/mocks"
 	"github.com/vulcanize/vulcanizedb/pkg/super_node"
-	mocks3 "github.com/vulcanize/vulcanizedb/pkg/super_node/mocks"
+	mocks2 "github.com/vulcanize/vulcanizedb/pkg/super_node/eth/mocks"
 )
 
 var _ = Describe("Service", func() {
 	Describe("SyncAndPublish", func() {
 		It("Streams statediff.Payloads, converts them to IPLDPayloads, publishes IPLDPayloads, and indexes CIDPayloads", func() {
 			wg := new(sync.WaitGroup)
-			payloadChan := make(chan statediff.Payload, 1)
+			payloadChan := make(chan interface{}, 1)
 			quitChan := make(chan bool, 1)
-			mockCidRepo := &mocks3.CIDRepository{
+			mockCidIndexer := &mocks2.CIDIndexer{
 				ReturnErr: nil,
 			}
-			mockPublisher := &mocks.IPLDPublisher{
+			mockPublisher := &mocks2.IPLDPublisher{
-				ReturnCIDPayload: mocks.MockCIDPayload,
+				ReturnCIDPayload: mocks2.MockCIDPayload,
 				ReturnErr:        nil,
 			}
 			mockStreamer := &mocks2.StateDiffStreamer{
 				ReturnSub: &rpc.ClientSubscription{},
 				StreamPayloads: []statediff.Payload{
-					mocks.MockStateDiffPayload,
+					mocks2.MockStateDiffPayload,
 				},
 				ReturnErr: nil,
 			}
-			mockConverter := &mocks.PayloadConverter{
+			mockConverter := &mocks2.PayloadConverter{
-				ReturnIPLDPayload: mocks.MockIPLDPayload,
+				ReturnIPLDPayload: mocks2.MockIPLDPayload,
 				ReturnErr:         nil,
 			}
 			processor := &super_node.Service{
-				Repository:     mockCidRepo,
+				Indexer:        mockCidIndexer,
 				Publisher:      mockPublisher,
 				Streamer:       mockStreamer,
 				Converter:      mockConverter,
@@ -69,10 +67,10 @@ var _ = Describe("Service", func() {
 			time.Sleep(2 * time.Second)
 			quitChan <- true
 			wg.Wait()
-			Expect(mockConverter.PassedStatediffPayload).To(Equal(mocks.MockStateDiffPayload))
+			Expect(mockConverter.PassedStatediffPayload).To(Equal(mocks2.MockStateDiffPayload))
-			Expect(len(mockCidRepo.PassedCIDPayload)).To(Equal(1))
+			Expect(len(mockCidIndexer.PassedCIDPayload)).To(Equal(1))
-			Expect(mockCidRepo.PassedCIDPayload[0]).To(Equal(mocks.MockCIDPayload))
+			Expect(mockCidIndexer.PassedCIDPayload[0]).To(Equal(mocks2.MockCIDPayload))
-			Expect(mockPublisher.PassedIPLDPayload).To(Equal(mocks.MockIPLDPayload))
+			Expect(mockPublisher.PassedIPLDPayload).To(Equal(mocks2.MockIPLDPayload))
 			Expect(mockStreamer.PassedPayloadChan).To(Equal(payloadChan))
 		})
 	})

pkg/super_node/shared/functions.go

@@ -0,0 +1,49 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package shared
+
+import "bytes"
+
+// ListContainsString used to check if a list of strings contains a particular string
+func ListContainsString(sss []string, s string) bool {
+	for _, str := range sss {
+		if s == str {
+			return true
+		}
+	}
+	return false
+}
+
+// ListContainsBytes used to check if a list of byte arrays contains a particular byte array
+func ListContainsBytes(bbb [][]byte, b []byte) bool {
+	for _, by := range bbb {
+		if bytes.Equal(by, b) {
+			return true
+		}
+	}
+	return false
+}
+
+// ListContainsGap used to check if a list of Gaps contains a particular Gap
+func ListContainsGap(gapList []Gap, gap Gap) bool {
+	for _, listGap := range gapList {
+		if listGap == gap {
+			return true
+		}
+	}
+	return false
+}

pkg/super_node/shared/intefaces.go

@@ -0,0 +1,63 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package shared
+
+import (
+	"github.com/ethereum/go-ethereum/rpc"
+)
+
+// ResponseFilterer applies a filter to the streamed payload and returns a subscription response packet
+type ResponseFilterer interface {
+	Filter(filter, payload interface{}) (response interface{}, err error)
+}
+
+// CIDIndexer indexes a set of cids with their associated meta data in Postgres
+type CIDIndexer interface {
+	Index(cids interface{}) error
+}
+
+// CIDRetriever retrieves cids according to a provided filter and returns a cid
+type CIDRetriever interface {
+	Retrieve(filter interface{}, blockNumber int64) (interface{}, bool, error)
+	RetrieveFirstBlockNumber() (int64, error)
+	RetrieveLastBlockNumber() (int64, error)
+	RetrieveGapsInData() ([]Gap, error)
+}
+
+type PayloadStreamer interface {
+	Stream(payloadChan chan interface{}) (*rpc.ClientSubscription, error)
+}
+
+type PayloadFetcher interface {
+	FetchAt(blockHeights []uint64) ([]interface{}, error)
+}
+
+type IPLDFetcher interface {
+	Fetch(cids interface{}) (interface{}, error)
+}
+
+type PayloadConverter interface {
+	Convert(payload interface{}) (interface{}, error)
+}
+
+type IPLDPublisher interface {
+	Publish(payload interface{}) (interface{}, error)
+}
+
+type IPLDResolver interface {
+	Resolve(iplds interface{}) (interface{}, error)
+}

pkg/super_node/shared/types.go

@@ -0,0 +1,22 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package shared
+
+type Gap struct {
+	Start uint64
+	Stop  uint64
+}

pkg/super_node/subscription.go

@@ -17,11 +17,35 @@
 package super_node
 
 import (
-	"github.com/vulcanize/vulcanizedb/libraries/shared/streamer"
+	"math/big"
+
+	"github.com/ethereum/go-ethereum/rpc"
+
+	"github.com/vulcanize/vulcanizedb/pkg/super_node/config"
 )
 
 // Subscription holds the information for an individual client subscription to the super node
 type Subscription struct {
-	PayloadChan chan<- streamer.SuperNodePayload
+	ID          rpc.ID
+	PayloadChan chan<- Payload
 	QuitChan    chan<- bool
 }
+
+// Payload is the struct for a super node stream payload
+// It carries data of a type specific to the chain being supported/queried and an error message
+type Payload struct {
+	Data interface{} `json:"data"` // e.g. for Ethereum eth.StreamPayload
+	Err  string      `json:"err"`
+}
+
+// SubscriptionSettings is the interface every subscription filter type needs to satisfy, no matter the chain
+// Further specifics of the underlying filter type depend on the internal needs of the types
+// which satisfy the ResponseFilterer and CIDRetriever interfaces for a specific chain
+// The underlying type needs to be rlp serializable
+type SubscriptionSettings interface {
+	StartingBlock() *big.Int
+	EndingBlock() *big.Int
+	ChainType() config.ChainType
+	HistoricalData() bool
+	HistoricalDataOnly() bool
+}