seed => super; port 80 => port 8080; backfill process for the super_node

pkg/config/subscription.go

@@ -18,7 +18,7 @@ package config
 
 import "math/big"
 
-// Subscription config is used by a subscribing transformer to specifiy which data to receive from the seed node
+// Subscription config is used by a subscribing transformer to specifiy which data to receive from the super node
 type Subscription struct {
 	BackFill      bool
 	BackFillOnly  bool

pkg/ipfs/mocks/converter.go

@@ -17,6 +17,8 @@
 package mocks
 
 import (
+	"fmt"
+
 	"github.com/ethereum/go-ethereum/statediff"
 
 	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
@@ -34,3 +36,22 @@ func (pc *PayloadConverter) Convert(payload statediff.Payload) (*ipfs.IPLDPayloa
 	pc.PassedStatediffPayload = payload
 	return pc.ReturnIPLDPayload, pc.ReturnErr
 }
+
+// IterativePayloadConverter is the underlying struct for the Converter interface
+type IterativePayloadConverter struct {
+	PassedStatediffPayload []statediff.Payload
+	ReturnIPLDPayload      []*ipfs.IPLDPayload
+	ReturnErr              error
+	iteration              int
+}
+
+// Convert method is used to convert a geth statediff.Payload to a IPLDPayload
+func (pc *IterativePayloadConverter) Convert(payload statediff.Payload) (*ipfs.IPLDPayload, error) {
+	pc.PassedStatediffPayload = append(pc.PassedStatediffPayload, payload)
+	if len(pc.PassedStatediffPayload) < pc.iteration+1 {
+		return nil, fmt.Errorf("IterativePayloadConverter does not have a payload to return at iteration %d", pc.iteration)
+	}
+	returnPayload := pc.ReturnIPLDPayload[pc.iteration]
+	pc.iteration++
+	return returnPayload, pc.ReturnErr
+}

pkg/ipfs/mocks/publisher.go

@@ -17,6 +17,7 @@
 package mocks
 
 import (
+	"fmt"
 	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
 )
 
@@ -32,3 +33,22 @@ func (pub *IPLDPublisher) Publish(payload *ipfs.IPLDPayload) (*ipfs.CIDPayload,
 	pub.PassedIPLDPayload = payload
 	return pub.ReturnCIDPayload, pub.ReturnErr
 }
+
+// IterativeIPLDPublisher is the underlying struct for the Publisher interface; used in testing
+type IterativeIPLDPublisher struct {
+	PassedIPLDPayload []*ipfs.IPLDPayload
+	ReturnCIDPayload  []*ipfs.CIDPayload
+	ReturnErr         error
+	iteration         int
+}
+
+// Publish publishes an IPLDPayload to IPFS and returns the corresponding CIDPayload
+func (pub *IterativeIPLDPublisher) Publish(payload *ipfs.IPLDPayload) (*ipfs.CIDPayload, error) {
+	pub.PassedIPLDPayload = append(pub.PassedIPLDPayload, payload)
+	if len(pub.ReturnCIDPayload) < pub.iteration+1 {
+		return nil, fmt.Errorf("IterativeIPLDPublisher does not have a payload to return at iteration %d", pub.iteration)
+	}
+	returnPayload := pub.ReturnCIDPayload[pub.iteration]
+	pub.iteration++
+	return returnPayload, pub.ReturnErr
+}

pkg/ipfs/mocks/test_data.go

@@ -309,7 +309,7 @@ var (
 		},
 	}
 
-	MockSeeNodePayload = streamer.SeedNodePayload{
+	MockSeeNodePayload = streamer.SuperNodePayload{
 		BlockNumber:     big.NewInt(1),
 		HeadersRlp:      [][]byte{MockHeaderRlp},
 		TransactionsRlp: [][]byte{MockTransactions.GetRlp(0), MockTransactions.GetRlp(1)},

pkg/ipfs/resolver.go

@@ -24,7 +24,7 @@ import (
 
 // IPLDResolver is the interface to resolving IPLDs
 type IPLDResolver interface {
-	ResolveIPLDs(ipfsBlocks IPLDWrapper) (streamer.SeedNodePayload, error)
+	ResolveIPLDs(ipfsBlocks IPLDWrapper) (streamer.SuperNodePayload, error)
 }
 
 // EthIPLDResolver is the underlying struct to support the IPLDResolver interface
@@ -36,8 +36,8 @@ func NewIPLDResolver() *EthIPLDResolver {
 }
 
 // ResolveIPLDs is the exported method for resolving all of the ETH IPLDs packaged in an IpfsBlockWrapper
-func (eir *EthIPLDResolver) ResolveIPLDs(ipfsBlocks IPLDWrapper) (streamer.SeedNodePayload, error) {
-	response := &streamer.SeedNodePayload{
+func (eir *EthIPLDResolver) ResolveIPLDs(ipfsBlocks IPLDWrapper) (streamer.SuperNodePayload, error) {
+	response := &streamer.SuperNodePayload{
 		BlockNumber:     ipfsBlocks.BlockNumber,
 		StateNodesRlp:   make(map[common.Hash][]byte),
 		StorageNodesRlp: make(map[common.Hash]map[common.Hash][]byte),
@@ -51,42 +51,42 @@ func (eir *EthIPLDResolver) ResolveIPLDs(ipfsBlocks IPLDWrapper) (streamer.SeedN
 	return *response, nil
 }
 
-func (eir *EthIPLDResolver) resolveHeaders(blocks []blocks.Block, response *streamer.SeedNodePayload) {
+func (eir *EthIPLDResolver) resolveHeaders(blocks []blocks.Block, response *streamer.SuperNodePayload) {
 	for _, block := range blocks {
 		raw := block.RawData()
 		response.HeadersRlp = append(response.HeadersRlp, raw)
 	}
 }
 
-func (eir *EthIPLDResolver) resolveUncles(blocks []blocks.Block, response *streamer.SeedNodePayload) {
+func (eir *EthIPLDResolver) resolveUncles(blocks []blocks.Block, response *streamer.SuperNodePayload) {
 	for _, block := range blocks {
 		raw := block.RawData()
 		response.UnclesRlp = append(response.UnclesRlp, raw)
 	}
 }
 
-func (eir *EthIPLDResolver) resolveTransactions(blocks []blocks.Block, response *streamer.SeedNodePayload) {
+func (eir *EthIPLDResolver) resolveTransactions(blocks []blocks.Block, response *streamer.SuperNodePayload) {
 	for _, block := range blocks {
 		raw := block.RawData()
 		response.TransactionsRlp = append(response.TransactionsRlp, raw)
 	}
 }
 
-func (eir *EthIPLDResolver) resolveReceipts(blocks []blocks.Block, response *streamer.SeedNodePayload) {
+func (eir *EthIPLDResolver) resolveReceipts(blocks []blocks.Block, response *streamer.SuperNodePayload) {
 	for _, block := range blocks {
 		raw := block.RawData()
 		response.ReceiptsRlp = append(response.ReceiptsRlp, raw)
 	}
 }
 
-func (eir *EthIPLDResolver) resolveState(blocks map[common.Hash]blocks.Block, response *streamer.SeedNodePayload) {
+func (eir *EthIPLDResolver) resolveState(blocks map[common.Hash]blocks.Block, response *streamer.SuperNodePayload) {
 	for key, block := range blocks {
 		raw := block.RawData()
 		response.StateNodesRlp[key] = raw
 	}
 }
 
-func (eir *EthIPLDResolver) resolveStorage(blocks map[common.Hash]map[common.Hash]blocks.Block, response *streamer.SeedNodePayload) {
+func (eir *EthIPLDResolver) resolveStorage(blocks map[common.Hash]map[common.Hash]blocks.Block, response *streamer.SuperNodePayload) {
 	for stateKey, storageBlocks := range blocks {
 		response.StorageNodesRlp[stateKey] = make(map[common.Hash][]byte)
 		for storageKey, storageVal := range storageBlocks {

pkg/ipfs/resolver_test.go

@@ -22,7 +22,7 @@ import (
 
 	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
 	"github.com/vulcanize/vulcanizedb/pkg/ipfs/mocks"
-	"github.com/vulcanize/vulcanizedb/pkg/seed_node"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node"
 )
 
 var (
@@ -35,19 +35,19 @@ var _ = Describe("Resolver", func() {
 			resolver = ipfs.NewIPLDResolver()
 		})
 		It("Resolves IPLD data to their correct geth data types and packages them to send to requesting transformers", func() {
-			seedNodePayload, err := resolver.ResolveIPLDs(mocks.MockIPLDWrapper)
+			superNodePayload, err := resolver.ResolveIPLDs(mocks.MockIPLDWrapper)
 			Expect(err).ToNot(HaveOccurred())
-			Expect(seedNodePayload.BlockNumber.Int64()).To(Equal(mocks.MockSeeNodePayload.BlockNumber.Int64()))
-			Expect(seedNodePayload.HeadersRlp).To(Equal(mocks.MockSeeNodePayload.HeadersRlp))
-			Expect(seedNodePayload.UnclesRlp).To(Equal(mocks.MockSeeNodePayload.UnclesRlp))
-			Expect(len(seedNodePayload.TransactionsRlp)).To(Equal(2))
-			Expect(seed_node.ListContainsBytes(seedNodePayload.TransactionsRlp, mocks.MockTransactions.GetRlp(0))).To(BeTrue())
-			Expect(seed_node.ListContainsBytes(seedNodePayload.TransactionsRlp, mocks.MockTransactions.GetRlp(1))).To(BeTrue())
-			Expect(len(seedNodePayload.ReceiptsRlp)).To(Equal(2))
-			Expect(seed_node.ListContainsBytes(seedNodePayload.ReceiptsRlp, mocks.MockReceipts.GetRlp(0))).To(BeTrue())
-			Expect(seed_node.ListContainsBytes(seedNodePayload.ReceiptsRlp, mocks.MockReceipts.GetRlp(1))).To(BeTrue())
-			Expect(len(seedNodePayload.StateNodesRlp)).To(Equal(2))
-			Expect(seedNodePayload.StorageNodesRlp).To(Equal(mocks.MockSeeNodePayload.StorageNodesRlp))
+			Expect(superNodePayload.BlockNumber.Int64()).To(Equal(mocks.MockSeeNodePayload.BlockNumber.Int64()))
+			Expect(superNodePayload.HeadersRlp).To(Equal(mocks.MockSeeNodePayload.HeadersRlp))
+			Expect(superNodePayload.UnclesRlp).To(Equal(mocks.MockSeeNodePayload.UnclesRlp))
+			Expect(len(superNodePayload.TransactionsRlp)).To(Equal(2))
+			Expect(super_node.ListContainsBytes(superNodePayload.TransactionsRlp, mocks.MockTransactions.GetRlp(0))).To(BeTrue())
+			Expect(super_node.ListContainsBytes(superNodePayload.TransactionsRlp, mocks.MockTransactions.GetRlp(1))).To(BeTrue())
+			Expect(len(superNodePayload.ReceiptsRlp)).To(Equal(2))
+			Expect(super_node.ListContainsBytes(superNodePayload.ReceiptsRlp, mocks.MockReceipts.GetRlp(0))).To(BeTrue())
+			Expect(super_node.ListContainsBytes(superNodePayload.ReceiptsRlp, mocks.MockReceipts.GetRlp(1))).To(BeTrue())
+			Expect(len(superNodePayload.StateNodesRlp)).To(Equal(2))
+			Expect(superNodePayload.StorageNodesRlp).To(Equal(mocks.MockSeeNodePayload.StorageNodesRlp))
 		})
 	})
 })

pkg/seed_node/filterer_test.go

@@ -1,153 +0,0 @@
-// VulcanizeDB
-// Copyright © 2019 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package seed_node_test
-
-import (
-	"bytes"
-
-	"github.com/ethereum/go-ethereum/core/types"
-	. "github.com/onsi/ginkgo"
-	. "github.com/onsi/gomega"
-
-	"github.com/vulcanize/vulcanizedb/pkg/ipfs/mocks"
-	"github.com/vulcanize/vulcanizedb/pkg/seed_node"
-)
-
-var (
-	filterer                  seed_node.ResponseFilterer
-	expectedRctForStorageRLP1 []byte
-	expectedRctForStorageRLP2 []byte
-)
-
-var _ = Describe("Filterer", func() {
-	Describe("FilterResponse", func() {
-		BeforeEach(func() {
-			filterer = seed_node.NewResponseFilterer()
-			expectedRctForStorageRLP1 = getReceiptForStorageRLP(mocks.MockReceipts, 0)
-			expectedRctForStorageRLP2 = getReceiptForStorageRLP(mocks.MockReceipts, 1)
-		})
-
-		It("Transcribes all the data from the IPLDPayload into the SeedNodePayload if given an open filter", func() {
-			seedNodePayload, err := filterer.FilterResponse(openFilter, *mocks.MockIPLDPayload)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(seedNodePayload.BlockNumber.Int64()).To(Equal(mocks.MockSeeNodePayload.BlockNumber.Int64()))
-			Expect(seedNodePayload.HeadersRlp).To(Equal(mocks.MockSeeNodePayload.HeadersRlp))
-			Expect(seedNodePayload.UnclesRlp).To(Equal(mocks.MockSeeNodePayload.UnclesRlp))
-			Expect(len(seedNodePayload.TransactionsRlp)).To(Equal(2))
-			Expect(seed_node.ListContainsBytes(seedNodePayload.TransactionsRlp, mocks.MockTransactions.GetRlp(0))).To(BeTrue())
-			Expect(seed_node.ListContainsBytes(seedNodePayload.TransactionsRlp, mocks.MockTransactions.GetRlp(1))).To(BeTrue())
-			Expect(len(seedNodePayload.ReceiptsRlp)).To(Equal(2))
-			Expect(seed_node.ListContainsBytes(seedNodePayload.ReceiptsRlp, expectedRctForStorageRLP1)).To(BeTrue())
-			Expect(seed_node.ListContainsBytes(seedNodePayload.ReceiptsRlp, expectedRctForStorageRLP2)).To(BeTrue())
-			Expect(len(seedNodePayload.StateNodesRlp)).To(Equal(2))
-			Expect(seedNodePayload.StateNodesRlp[mocks.ContractLeafKey]).To(Equal(mocks.ValueBytes))
-			Expect(seedNodePayload.StateNodesRlp[mocks.AnotherContractLeafKey]).To(Equal(mocks.AnotherValueBytes))
-			Expect(seedNodePayload.StorageNodesRlp).To(Equal(mocks.MockSeeNodePayload.StorageNodesRlp))
-		})
-
-		It("Applies filters from the provided config.Subscription", func() {
-			seedNodePayload1, err := filterer.FilterResponse(rctContractFilter, *mocks.MockIPLDPayload)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(seedNodePayload1.BlockNumber.Int64()).To(Equal(mocks.MockSeeNodePayload.BlockNumber.Int64()))
-			Expect(len(seedNodePayload1.HeadersRlp)).To(Equal(0))
-			Expect(len(seedNodePayload1.UnclesRlp)).To(Equal(0))
-			Expect(len(seedNodePayload1.TransactionsRlp)).To(Equal(0))
-			Expect(len(seedNodePayload1.StorageNodesRlp)).To(Equal(0))
-			Expect(len(seedNodePayload1.StateNodesRlp)).To(Equal(0))
-			Expect(len(seedNodePayload1.ReceiptsRlp)).To(Equal(1))
-			Expect(seedNodePayload1.ReceiptsRlp[0]).To(Equal(expectedRctForStorageRLP2))
-
-			seedNodePayload2, err := filterer.FilterResponse(rctTopicsFilter, *mocks.MockIPLDPayload)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(seedNodePayload2.BlockNumber.Int64()).To(Equal(mocks.MockSeeNodePayload.BlockNumber.Int64()))
-			Expect(len(seedNodePayload2.HeadersRlp)).To(Equal(0))
-			Expect(len(seedNodePayload2.UnclesRlp)).To(Equal(0))
-			Expect(len(seedNodePayload2.TransactionsRlp)).To(Equal(0))
-			Expect(len(seedNodePayload2.StorageNodesRlp)).To(Equal(0))
-			Expect(len(seedNodePayload2.StateNodesRlp)).To(Equal(0))
-			Expect(len(seedNodePayload2.ReceiptsRlp)).To(Equal(1))
-			Expect(seedNodePayload2.ReceiptsRlp[0]).To(Equal(expectedRctForStorageRLP1))
-
-			seedNodePayload3, err := filterer.FilterResponse(rctTopicsAndContractFilter, *mocks.MockIPLDPayload)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(seedNodePayload3.BlockNumber.Int64()).To(Equal(mocks.MockSeeNodePayload.BlockNumber.Int64()))
-			Expect(len(seedNodePayload3.HeadersRlp)).To(Equal(0))
-			Expect(len(seedNodePayload3.UnclesRlp)).To(Equal(0))
-			Expect(len(seedNodePayload3.TransactionsRlp)).To(Equal(0))
-			Expect(len(seedNodePayload3.StorageNodesRlp)).To(Equal(0))
-			Expect(len(seedNodePayload3.StateNodesRlp)).To(Equal(0))
-			Expect(len(seedNodePayload3.ReceiptsRlp)).To(Equal(1))
-			Expect(seedNodePayload3.ReceiptsRlp[0]).To(Equal(expectedRctForStorageRLP1))
-
-			seedNodePayload4, err := filterer.FilterResponse(rctContractsAndTopicFilter, *mocks.MockIPLDPayload)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(seedNodePayload4.BlockNumber.Int64()).To(Equal(mocks.MockSeeNodePayload.BlockNumber.Int64()))
-			Expect(len(seedNodePayload4.HeadersRlp)).To(Equal(0))
-			Expect(len(seedNodePayload4.UnclesRlp)).To(Equal(0))
-			Expect(len(seedNodePayload4.TransactionsRlp)).To(Equal(0))
-			Expect(len(seedNodePayload4.StorageNodesRlp)).To(Equal(0))
-			Expect(len(seedNodePayload4.StateNodesRlp)).To(Equal(0))
-			Expect(len(seedNodePayload4.ReceiptsRlp)).To(Equal(1))
-			Expect(seedNodePayload4.ReceiptsRlp[0]).To(Equal(expectedRctForStorageRLP2))
-
-			seedNodePayload5, err := filterer.FilterResponse(rctsForAllCollectedTrxs, *mocks.MockIPLDPayload)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(seedNodePayload5.BlockNumber.Int64()).To(Equal(mocks.MockSeeNodePayload.BlockNumber.Int64()))
-			Expect(len(seedNodePayload5.HeadersRlp)).To(Equal(0))
-			Expect(len(seedNodePayload5.UnclesRlp)).To(Equal(0))
-			Expect(len(seedNodePayload5.TransactionsRlp)).To(Equal(2))
-			Expect(seed_node.ListContainsBytes(seedNodePayload5.TransactionsRlp, mocks.MockTransactions.GetRlp(0))).To(BeTrue())
-			Expect(seed_node.ListContainsBytes(seedNodePayload5.TransactionsRlp, mocks.MockTransactions.GetRlp(1))).To(BeTrue())
-			Expect(len(seedNodePayload5.StorageNodesRlp)).To(Equal(0))
-			Expect(len(seedNodePayload5.StateNodesRlp)).To(Equal(0))
-			Expect(len(seedNodePayload5.ReceiptsRlp)).To(Equal(2))
-			Expect(seed_node.ListContainsBytes(seedNodePayload5.ReceiptsRlp, expectedRctForStorageRLP1)).To(BeTrue())
-			Expect(seed_node.ListContainsBytes(seedNodePayload5.ReceiptsRlp, expectedRctForStorageRLP2)).To(BeTrue())
-
-			seedNodePayload6, err := filterer.FilterResponse(rctsForSelectCollectedTrxs, *mocks.MockIPLDPayload)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(seedNodePayload6.BlockNumber.Int64()).To(Equal(mocks.MockSeeNodePayload.BlockNumber.Int64()))
-			Expect(len(seedNodePayload6.HeadersRlp)).To(Equal(0))
-			Expect(len(seedNodePayload6.UnclesRlp)).To(Equal(0))
-			Expect(len(seedNodePayload6.TransactionsRlp)).To(Equal(1))
-			Expect(seed_node.ListContainsBytes(seedNodePayload5.TransactionsRlp, mocks.MockTransactions.GetRlp(1))).To(BeTrue())
-			Expect(len(seedNodePayload6.StorageNodesRlp)).To(Equal(0))
-			Expect(len(seedNodePayload6.StateNodesRlp)).To(Equal(0))
-			Expect(len(seedNodePayload6.ReceiptsRlp)).To(Equal(1))
-			Expect(seedNodePayload4.ReceiptsRlp[0]).To(Equal(expectedRctForStorageRLP2))
-
-			seedNodePayload7, err := filterer.FilterResponse(stateFilter, *mocks.MockIPLDPayload)
-			Expect(err).ToNot(HaveOccurred())
-			Expect(seedNodePayload7.BlockNumber.Int64()).To(Equal(mocks.MockSeeNodePayload.BlockNumber.Int64()))
-			Expect(len(seedNodePayload7.HeadersRlp)).To(Equal(0))
-			Expect(len(seedNodePayload7.UnclesRlp)).To(Equal(0))
-			Expect(len(seedNodePayload7.TransactionsRlp)).To(Equal(0))
-			Expect(len(seedNodePayload7.StorageNodesRlp)).To(Equal(0))
-			Expect(len(seedNodePayload7.ReceiptsRlp)).To(Equal(0))
-			Expect(len(seedNodePayload7.StateNodesRlp)).To(Equal(1))
-			Expect(seedNodePayload7.StateNodesRlp[mocks.ContractLeafKey]).To(Equal(mocks.ValueBytes))
-		})
-	})
-})
-
-func getReceiptForStorageRLP(receipts types.Receipts, i int) []byte {
-	receiptForStorage := (*types.ReceiptForStorage)(receipts[i])
-	receiptBuffer := new(bytes.Buffer)
-	err := receiptForStorage.EncodeRLP(receiptBuffer)
-	Expect(err).ToNot(HaveOccurred())
-	return receiptBuffer.Bytes()
-}

pkg/super_node/api.go

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package seed_node
+package super_node
 
 import (
 	"context"
@@ -28,25 +28,25 @@ import (
 )
 
 // APIName is the namespace used for the state diffing service API
-const APIName = "vulcanizedb"
+const APIName = "vdb"
 
 // APIVersion is the version of the state diffing service API
 const APIVersion = "0.0.1"
 
-// PublicSeedNodeAPI is the public api for the seed node
-type PublicSeedNodeAPI struct {
+// PublicSuperNodeAPI is the public api for the super node
+type PublicSuperNodeAPI struct {
 	sni NodeInterface
 }
 
-// NewPublicSeedNodeAPI creates a new PublicSeedNodeAPI with the provided underlying SyncPublishScreenAndServe process
-func NewPublicSeedNodeAPI(seedNodeInterface NodeInterface) *PublicSeedNodeAPI {
-	return &PublicSeedNodeAPI{
-		sni: seedNodeInterface,
+// NewPublicSuperNodeAPI creates a new PublicSuperNodeAPI with the provided underlying SyncPublishScreenAndServe process
+func NewPublicSuperNodeAPI(superNodeInterface NodeInterface) *PublicSuperNodeAPI {
+	return &PublicSuperNodeAPI{
+		sni: superNodeInterface,
 	}
 }
 
 // Stream is the public method to setup a subscription that fires off SyncPublishScreenAndServe payloads as they are created
-func (api *PublicSeedNodeAPI) Stream(ctx context.Context, streamFilters config.Subscription) (*rpc.Subscription, error) {
+func (api *PublicSuperNodeAPI) Stream(ctx context.Context, streamFilters config.Subscription) (*rpc.Subscription, error) {
 	// ensure that the RPC connection supports subscriptions
 	notifier, supported := rpc.NotifierFromContext(ctx)
 	if !supported {
@@ -58,7 +58,7 @@ func (api *PublicSeedNodeAPI) Stream(ctx context.Context, streamFilters config.S
 
 	go func() {
 		// subscribe to events from the SyncPublishScreenAndServe service
-		payloadChannel := make(chan streamer.SeedNodePayload, payloadChanBufferSize)
+		payloadChannel := make(chan streamer.SuperNodePayload, payloadChanBufferSize)
 		quitChan := make(chan bool, 1)
 		go api.sni.Subscribe(rpcSub.ID, payloadChannel, quitChan, streamFilters)
 
@@ -84,7 +84,7 @@ func (api *PublicSeedNodeAPI) Stream(ctx context.Context, streamFilters config.S
 	return rpcSub, nil
 }
 
-// Node is a public rpc method to allow transformers to fetch the Geth node info for the seed node
-func (api *PublicSeedNodeAPI) Node() core.Node {
+// Node is a public rpc method to allow transformers to fetch the Geth node info for the super node
+func (api *PublicSuperNodeAPI) Node() core.Node {
 	return api.sni.Node()
 }

pkg/super_node/backfiller.go

@@ -0,0 +1,135 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package super_node
+
+import (
+	"sync"
+	"time"
+
+	"github.com/ethereum/go-ethereum/params"
+	log "github.com/sirupsen/logrus"
+
+	"github.com/vulcanize/vulcanizedb/libraries/shared/fetcher"
+	"github.com/vulcanize/vulcanizedb/pkg/core"
+	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
+	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
+)
+
+// BackFillInterface for filling in gaps in the super node
+type BackFillInterface interface {
+	// Method for the super node to periodically check for and fill in gaps in its data using an archival node
+	FillGaps(wg *sync.WaitGroup, quitChan <-chan bool)
+}
+
+// BackFillService for filling in gaps in the super node
+type BackFillService struct {
+	// Interface for converting statediff payloads into ETH-IPLD object payloads
+	Converter ipfs.PayloadConverter
+	// Interface for publishing the ETH-IPLD payloads to IPFS
+	Publisher ipfs.IPLDPublisher
+	// Interface for indexing the CIDs of the published ETH-IPLDs in Postgres
+	Repository CIDRepository
+	// Interface for searching and retrieving CIDs from Postgres index
+	Retriever CIDRetriever
+	// State-diff fetcher; needs to be configured with an archival core.RpcClient
+	StateDiffFetcher fetcher.IStateDiffFetcher
+	// Check frequency
+	GapCheckFrequency time.Duration
+}
+
+// NewBackFillService returns a new BackFillInterface
+func NewBackFillService(ipfsPublisher ipfs.IPLDPublisher, db *postgres.DB, archivalNodeRpcClient core.RpcClient, freq time.Duration) BackFillInterface {
+	return &BackFillService{
+		Repository:        NewCIDRepository(db),
+		Converter:         ipfs.NewPayloadConverter(params.MainnetChainConfig),
+		Publisher:         ipfsPublisher,
+		Retriever:         NewCIDRetriever(db),
+		StateDiffFetcher:  fetcher.NewStateDiffFetcher(archivalNodeRpcClient),
+		GapCheckFrequency: freq,
+	}
+}
+
+// FillGaps periodically checks for and fills in gaps in the super node db
+// this requires a core.RpcClient that is pointed at an archival node with the StateDiffAt method exposed
+func (bfs *BackFillService) FillGaps(wg *sync.WaitGroup, quitChan <-chan bool) {
+	ticker := time.NewTicker(bfs.GapCheckFrequency)
+	wg.Add(1)
+
+	go func() {
+		for {
+			select {
+			case <-quitChan:
+				log.Info("quiting FillGaps process")
+				wg.Done()
+				return
+			case <-ticker.C:
+				log.Info("searching for gaps in the super node database")
+				startingBlock, firstBlockErr := bfs.Retriever.RetrieveFirstBlockNumber()
+				if firstBlockErr != nil {
+					log.Error(firstBlockErr)
+					continue
+				}
+				if startingBlock != 1 {
+					startingGap := [2]int64{
+						1,
+						startingBlock - 1,
+					}
+					log.Info("found gap at the beginning of the sync")
+					bfs.fillGaps(startingGap)
+				}
+
+				gaps, gapErr := bfs.Retriever.RetrieveGapsInData()
+				if gapErr != nil {
+					log.Error(gapErr)
+					continue
+				}
+				for _, gap := range gaps {
+					bfs.fillGaps(gap)
+				}
+			}
+		}
+	}()
+}
+
+func (bfs *BackFillService) fillGaps(gap [2]int64) {
+	log.Infof("filling in gap from block %d to block %d", gap[0], gap[1])
+	blockHeights := make([]uint64, 0, gap[1]-gap[0]+1)
+	for i := gap[0]; i <= gap[1]; i++ {
+		blockHeights = append(blockHeights, uint64(i))
+	}
+	payloads, fetchErr := bfs.StateDiffFetcher.FetchStateDiffsAt(blockHeights)
+	if fetchErr != nil {
+		log.Error(fetchErr)
+		return
+	}
+	for _, payload := range payloads {
+		ipldPayload, convertErr := bfs.Converter.Convert(*payload)
+		if convertErr != nil {
+			log.Error(convertErr)
+			continue
+		}
+		cidPayload, publishErr := bfs.Publisher.Publish(ipldPayload)
+		if publishErr != nil {
+			log.Error(publishErr)
+			continue
+		}
+		indexErr := bfs.Repository.Index(cidPayload)
+		if indexErr != nil {
+			log.Error(indexErr)
+		}
+	}
+}

pkg/super_node/backfiller_test.go

@@ -0,0 +1,187 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package super_node_test
+
+import (
+	"sync"
+	"time"
+
+	"github.com/ethereum/go-ethereum/statediff"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	mocks2 "github.com/vulcanize/vulcanizedb/libraries/shared/mocks"
+	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
+	"github.com/vulcanize/vulcanizedb/pkg/ipfs/mocks"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node"
+	mocks3 "github.com/vulcanize/vulcanizedb/pkg/super_node/mocks"
+)
+
+var _ = Describe("BackFiller", func() {
+	Describe("FillGaps", func() {
+		It("Periodically checks for and fills in gaps in the super node's data", func() {
+			mockCidRepo := &mocks3.CIDRepository{
+				ReturnErr: nil,
+			}
+			mockPublisher := &mocks.IterativeIPLDPublisher{
+				ReturnCIDPayload: []*ipfs.CIDPayload{mocks.MockCIDPayload, mocks.MockCIDPayload},
+				ReturnErr:        nil,
+			}
+			mockConverter := &mocks.IterativePayloadConverter{
+				ReturnIPLDPayload: []*ipfs.IPLDPayload{mocks.MockIPLDPayload, mocks.MockIPLDPayload},
+				ReturnErr:         nil,
+			}
+			mockRetriever := &mocks3.MockCIDRetriever{
+				FirstBlockNumberToReturn: 1,
+				GapsToRetrieve: [][2]int64{
+					{
+						100, 101,
+					},
+				},
+			}
+			mockFetcher := &mocks2.MockStateDiffFetcher{
+				StateDiffsToReturn: map[uint64]*statediff.Payload{
+					100: &mocks.MockStateDiffPayload,
+					101: &mocks.MockStateDiffPayload,
+				},
+			}
+			backfiller := &super_node.BackFillService{
+				Repository:        mockCidRepo,
+				Publisher:         mockPublisher,
+				Converter:         mockConverter,
+				StateDiffFetcher:  mockFetcher,
+				Retriever:         mockRetriever,
+				GapCheckFrequency: time.Second * 10,
+			}
+			wg := &sync.WaitGroup{}
+			quitChan := make(chan bool, 1)
+			backfiller.FillGaps(wg, quitChan)
+			time.Sleep(time.Second * 15)
+			quitChan <- true
+			Expect(len(mockCidRepo.PassedCIDPayload)).To(Equal(2))
+			Expect(mockCidRepo.PassedCIDPayload[0]).To(Equal(mocks.MockCIDPayload))
+			Expect(mockCidRepo.PassedCIDPayload[1]).To(Equal(mocks.MockCIDPayload))
+			Expect(len(mockPublisher.PassedIPLDPayload)).To(Equal(2))
+			Expect(mockPublisher.PassedIPLDPayload[0]).To(Equal(mocks.MockIPLDPayload))
+			Expect(mockPublisher.PassedIPLDPayload[1]).To(Equal(mocks.MockIPLDPayload))
+			Expect(len(mockConverter.PassedStatediffPayload)).To(Equal(2))
+			Expect(mockConverter.PassedStatediffPayload[0]).To(Equal(mocks.MockStateDiffPayload))
+			Expect(mockConverter.PassedStatediffPayload[1]).To(Equal(mocks.MockStateDiffPayload))
+			Expect(mockRetriever.CalledTimes).To(Equal(1))
+			Expect(len(mockFetcher.CalledAtBlockHeights)).To(Equal(1))
+			Expect(mockFetcher.CalledAtBlockHeights[0]).To(Equal([]uint64{100, 101}))
+		})
+
+		It("Works for single block `ranges`", func() {
+			mockCidRepo := &mocks3.CIDRepository{
+				ReturnErr: nil,
+			}
+			mockPublisher := &mocks.IterativeIPLDPublisher{
+				ReturnCIDPayload: []*ipfs.CIDPayload{mocks.MockCIDPayload},
+				ReturnErr:        nil,
+			}
+			mockConverter := &mocks.IterativePayloadConverter{
+				ReturnIPLDPayload: []*ipfs.IPLDPayload{mocks.MockIPLDPayload},
+				ReturnErr:         nil,
+			}
+			mockRetriever := &mocks3.MockCIDRetriever{
+				FirstBlockNumberToReturn: 1,
+				GapsToRetrieve: [][2]int64{
+					{
+						100, 100,
+					},
+				},
+			}
+			mockFetcher := &mocks2.MockStateDiffFetcher{
+				StateDiffsToReturn: map[uint64]*statediff.Payload{
+					100: &mocks.MockStateDiffPayload,
+				},
+			}
+			backfiller := &super_node.BackFillService{
+				Repository:        mockCidRepo,
+				Publisher:         mockPublisher,
+				Converter:         mockConverter,
+				StateDiffFetcher:  mockFetcher,
+				Retriever:         mockRetriever,
+				GapCheckFrequency: time.Second * 10,
+			}
+			wg := &sync.WaitGroup{}
+			quitChan := make(chan bool, 1)
+			backfiller.FillGaps(wg, quitChan)
+			time.Sleep(time.Second * 15)
+			quitChan <- true
+			Expect(len(mockCidRepo.PassedCIDPayload)).To(Equal(1))
+			Expect(mockCidRepo.PassedCIDPayload[0]).To(Equal(mocks.MockCIDPayload))
+			Expect(len(mockPublisher.PassedIPLDPayload)).To(Equal(1))
+			Expect(mockPublisher.PassedIPLDPayload[0]).To(Equal(mocks.MockIPLDPayload))
+			Expect(len(mockConverter.PassedStatediffPayload)).To(Equal(1))
+			Expect(mockConverter.PassedStatediffPayload[0]).To(Equal(mocks.MockStateDiffPayload))
+			Expect(mockRetriever.CalledTimes).To(Equal(1))
+			Expect(len(mockFetcher.CalledAtBlockHeights)).To(Equal(1))
+			Expect(mockFetcher.CalledAtBlockHeights[0]).To(Equal([]uint64{100}))
+		})
+
+		It("Finds beginning gap", func() {
+			mockCidRepo := &mocks3.CIDRepository{
+				ReturnErr: nil,
+			}
+			mockPublisher := &mocks.IterativeIPLDPublisher{
+				ReturnCIDPayload: []*ipfs.CIDPayload{mocks.MockCIDPayload, mocks.MockCIDPayload},
+				ReturnErr:        nil,
+			}
+			mockConverter := &mocks.IterativePayloadConverter{
+				ReturnIPLDPayload: []*ipfs.IPLDPayload{mocks.MockIPLDPayload, mocks.MockIPLDPayload},
+				ReturnErr:         nil,
+			}
+			mockRetriever := &mocks3.MockCIDRetriever{
+				FirstBlockNumberToReturn: 3,
+				GapsToRetrieve:           [][2]int64{},
+			}
+			mockFetcher := &mocks2.MockStateDiffFetcher{
+				StateDiffsToReturn: map[uint64]*statediff.Payload{
+					1: &mocks.MockStateDiffPayload,
+					2: &mocks.MockStateDiffPayload,
+				},
+			}
+			backfiller := &super_node.BackFillService{
+				Repository:        mockCidRepo,
+				Publisher:         mockPublisher,
+				Converter:         mockConverter,
+				StateDiffFetcher:  mockFetcher,
+				Retriever:         mockRetriever,
+				GapCheckFrequency: time.Second * 10,
+			}
+			wg := &sync.WaitGroup{}
+			quitChan := make(chan bool, 1)
+			backfiller.FillGaps(wg, quitChan)
+			time.Sleep(time.Second * 15)
+			quitChan <- true
+			Expect(len(mockCidRepo.PassedCIDPayload)).To(Equal(2))
+			Expect(mockCidRepo.PassedCIDPayload[0]).To(Equal(mocks.MockCIDPayload))
+			Expect(mockCidRepo.PassedCIDPayload[1]).To(Equal(mocks.MockCIDPayload))
+			Expect(len(mockPublisher.PassedIPLDPayload)).To(Equal(2))
+			Expect(mockPublisher.PassedIPLDPayload[0]).To(Equal(mocks.MockIPLDPayload))
+			Expect(mockPublisher.PassedIPLDPayload[1]).To(Equal(mocks.MockIPLDPayload))
+			Expect(len(mockConverter.PassedStatediffPayload)).To(Equal(2))
+			Expect(mockConverter.PassedStatediffPayload[0]).To(Equal(mocks.MockStateDiffPayload))
+			Expect(mockConverter.PassedStatediffPayload[1]).To(Equal(mocks.MockStateDiffPayload))
+			Expect(mockRetriever.CalledTimes).To(Equal(1))
+			Expect(len(mockFetcher.CalledAtBlockHeights)).To(Equal(1))
+			Expect(mockFetcher.CalledAtBlockHeights[0]).To(Equal([]uint64{1, 2}))
+		})
+	})
+})

pkg/super_node/filterer.go

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package seed_node
+package super_node
 
 import (
 	"bytes"
@@ -30,7 +30,7 @@ import (
 
 // ResponseFilterer is the inteface used to screen eth data and package appropriate data into a response payload
 type ResponseFilterer interface {
-	FilterResponse(streamFilters config.Subscription, payload ipfs.IPLDPayload) (streamer.SeedNodePayload, error)
+	FilterResponse(streamFilters config.Subscription, payload ipfs.IPLDPayload) (streamer.SuperNodePayload, error)
 }
 
 // Filterer is the underlying struct for the ResponseFilterer interface
@@ -42,33 +42,33 @@ func NewResponseFilterer() *Filterer {
 }
 
 // FilterResponse is used to filter through eth data to extract and package requested data into a Payload
-func (s *Filterer) FilterResponse(streamFilters config.Subscription, payload ipfs.IPLDPayload) (streamer.SeedNodePayload, error) {
-	response := new(streamer.SeedNodePayload)
+func (s *Filterer) FilterResponse(streamFilters config.Subscription, payload ipfs.IPLDPayload) (streamer.SuperNodePayload, error) {
+	response := new(streamer.SuperNodePayload)
 	err := s.filterHeaders(streamFilters, response, payload)
 	if err != nil {
-		return streamer.SeedNodePayload{}, err
+		return streamer.SuperNodePayload{}, err
 	}
 	txHashes, err := s.filterTransactions(streamFilters, response, payload)
 	if err != nil {
-		return streamer.SeedNodePayload{}, err
+		return streamer.SuperNodePayload{}, err
 	}
 	err = s.filerReceipts(streamFilters, response, payload, txHashes)
 	if err != nil {
-		return streamer.SeedNodePayload{}, err
+		return streamer.SuperNodePayload{}, err
 	}
 	err = s.filterState(streamFilters, response, payload)
 	if err != nil {
-		return streamer.SeedNodePayload{}, err
+		return streamer.SuperNodePayload{}, err
 	}
 	err = s.filterStorage(streamFilters, response, payload)
 	if err != nil {
-		return streamer.SeedNodePayload{}, err
+		return streamer.SuperNodePayload{}, err
 	}
 	response.BlockNumber = payload.BlockNumber
 	return *response, nil
 }
 
-func (s *Filterer) filterHeaders(streamFilters config.Subscription, response *streamer.SeedNodePayload, payload ipfs.IPLDPayload) error {
+func (s *Filterer) filterHeaders(streamFilters config.Subscription, response *streamer.SuperNodePayload, payload ipfs.IPLDPayload) error {
 	if !streamFilters.HeaderFilter.Off && checkRange(streamFilters.StartingBlock.Int64(), streamFilters.EndingBlock.Int64(), payload.BlockNumber.Int64()) {
 		response.HeadersRlp = append(response.HeadersRlp, payload.HeaderRLP)
 		if !streamFilters.HeaderFilter.FinalOnly {
@@ -91,7 +91,7 @@ func checkRange(start, end, actual int64) bool {
 	return false
 }
 
-func (s *Filterer) filterTransactions(streamFilters config.Subscription, response *streamer.SeedNodePayload, payload ipfs.IPLDPayload) ([]common.Hash, error) {
+func (s *Filterer) filterTransactions(streamFilters config.Subscription, response *streamer.SuperNodePayload, payload ipfs.IPLDPayload) ([]common.Hash, error) {
 	trxHashes := make([]common.Hash, 0, len(payload.BlockBody.Transactions))
 	if !streamFilters.TrxFilter.Off && checkRange(streamFilters.StartingBlock.Int64(), streamFilters.EndingBlock.Int64(), payload.BlockNumber.Int64()) {
 		for i, trx := range payload.BlockBody.Transactions {
@@ -127,7 +127,7 @@ func checkTransactions(wantedSrc, wantedDst []string, actualSrc, actualDst strin
 	return false
 }
 
-func (s *Filterer) filerReceipts(streamFilters config.Subscription, response *streamer.SeedNodePayload, payload ipfs.IPLDPayload, trxHashes []common.Hash) error {
+func (s *Filterer) filerReceipts(streamFilters config.Subscription, response *streamer.SuperNodePayload, payload ipfs.IPLDPayload, trxHashes []common.Hash) error {
 	if !streamFilters.ReceiptFilter.Off && checkRange(streamFilters.StartingBlock.Int64(), streamFilters.EndingBlock.Int64(), payload.BlockNumber.Int64()) {
 		for i, receipt := range payload.Receipts {
 			if checkReceipts(receipt, streamFilters.ReceiptFilter.Topic0s, payload.ReceiptMetaData[i].Topic0s, streamFilters.ReceiptFilter.Contracts, payload.ReceiptMetaData[i].ContractAddress, trxHashes) {
@@ -186,7 +186,7 @@ func checkReceipts(rct *types.Receipt, wantedTopics, actualTopics, wantedContrac
 	return false
 }
 
-func (s *Filterer) filterState(streamFilters config.Subscription, response *streamer.SeedNodePayload, payload ipfs.IPLDPayload) error {
+func (s *Filterer) filterState(streamFilters config.Subscription, response *streamer.SuperNodePayload, payload ipfs.IPLDPayload) error {
 	if !streamFilters.StateFilter.Off && checkRange(streamFilters.StartingBlock.Int64(), streamFilters.EndingBlock.Int64(), payload.BlockNumber.Int64()) {
 		response.StateNodesRlp = make(map[common.Hash][]byte)
 		keyFilters := make([]common.Hash, 0, len(streamFilters.StateFilter.Addresses))
@@ -218,7 +218,7 @@ func checkNodeKeys(wantedKeys []common.Hash, actualKey common.Hash) bool {
 	return false
 }
 
-func (s *Filterer) filterStorage(streamFilters config.Subscription, response *streamer.SeedNodePayload, payload ipfs.IPLDPayload) error {
+func (s *Filterer) filterStorage(streamFilters config.Subscription, response *streamer.SuperNodePayload, payload ipfs.IPLDPayload) error {
 	if !streamFilters.StorageFilter.Off && checkRange(streamFilters.StartingBlock.Int64(), streamFilters.EndingBlock.Int64(), payload.BlockNumber.Int64()) {
 		response.StorageNodesRlp = make(map[common.Hash]map[common.Hash][]byte)
 		stateKeyFilters := make([]common.Hash, 0, len(streamFilters.StorageFilter.Addresses))

pkg/super_node/filterer_test.go

@@ -0,0 +1,153 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package super_node_test
+
+import (
+	"bytes"
+
+	"github.com/ethereum/go-ethereum/core/types"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/pkg/ipfs/mocks"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node"
+)
+
+var (
+	filterer                  super_node.ResponseFilterer
+	expectedRctForStorageRLP1 []byte
+	expectedRctForStorageRLP2 []byte
+)
+
+var _ = Describe("Filterer", func() {
+	Describe("FilterResponse", func() {
+		BeforeEach(func() {
+			filterer = super_node.NewResponseFilterer()
+			expectedRctForStorageRLP1 = getReceiptForStorageRLP(mocks.MockReceipts, 0)
+			expectedRctForStorageRLP2 = getReceiptForStorageRLP(mocks.MockReceipts, 1)
+		})
+
+		It("Transcribes all the data from the IPLDPayload into the SuperNodePayload if given an open filter", func() {
+			superNodePayload, err := filterer.FilterResponse(openFilter, *mocks.MockIPLDPayload)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(superNodePayload.BlockNumber.Int64()).To(Equal(mocks.MockSeeNodePayload.BlockNumber.Int64()))
+			Expect(superNodePayload.HeadersRlp).To(Equal(mocks.MockSeeNodePayload.HeadersRlp))
+			Expect(superNodePayload.UnclesRlp).To(Equal(mocks.MockSeeNodePayload.UnclesRlp))
+			Expect(len(superNodePayload.TransactionsRlp)).To(Equal(2))
+			Expect(super_node.ListContainsBytes(superNodePayload.TransactionsRlp, mocks.MockTransactions.GetRlp(0))).To(BeTrue())
+			Expect(super_node.ListContainsBytes(superNodePayload.TransactionsRlp, mocks.MockTransactions.GetRlp(1))).To(BeTrue())
+			Expect(len(superNodePayload.ReceiptsRlp)).To(Equal(2))
+			Expect(super_node.ListContainsBytes(superNodePayload.ReceiptsRlp, expectedRctForStorageRLP1)).To(BeTrue())
+			Expect(super_node.ListContainsBytes(superNodePayload.ReceiptsRlp, expectedRctForStorageRLP2)).To(BeTrue())
+			Expect(len(superNodePayload.StateNodesRlp)).To(Equal(2))
+			Expect(superNodePayload.StateNodesRlp[mocks.ContractLeafKey]).To(Equal(mocks.ValueBytes))
+			Expect(superNodePayload.StateNodesRlp[mocks.AnotherContractLeafKey]).To(Equal(mocks.AnotherValueBytes))
+			Expect(superNodePayload.StorageNodesRlp).To(Equal(mocks.MockSeeNodePayload.StorageNodesRlp))
+		})
+
+		It("Applies filters from the provided config.Subscription", func() {
+			superNodePayload1, err := filterer.FilterResponse(rctContractFilter, *mocks.MockIPLDPayload)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(superNodePayload1.BlockNumber.Int64()).To(Equal(mocks.MockSeeNodePayload.BlockNumber.Int64()))
+			Expect(len(superNodePayload1.HeadersRlp)).To(Equal(0))
+			Expect(len(superNodePayload1.UnclesRlp)).To(Equal(0))
+			Expect(len(superNodePayload1.TransactionsRlp)).To(Equal(0))
+			Expect(len(superNodePayload1.StorageNodesRlp)).To(Equal(0))
+			Expect(len(superNodePayload1.StateNodesRlp)).To(Equal(0))
+			Expect(len(superNodePayload1.ReceiptsRlp)).To(Equal(1))
+			Expect(superNodePayload1.ReceiptsRlp[0]).To(Equal(expectedRctForStorageRLP2))
+
+			superNodePayload2, err := filterer.FilterResponse(rctTopicsFilter, *mocks.MockIPLDPayload)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(superNodePayload2.BlockNumber.Int64()).To(Equal(mocks.MockSeeNodePayload.BlockNumber.Int64()))
+			Expect(len(superNodePayload2.HeadersRlp)).To(Equal(0))
+			Expect(len(superNodePayload2.UnclesRlp)).To(Equal(0))
+			Expect(len(superNodePayload2.TransactionsRlp)).To(Equal(0))
+			Expect(len(superNodePayload2.StorageNodesRlp)).To(Equal(0))
+			Expect(len(superNodePayload2.StateNodesRlp)).To(Equal(0))
+			Expect(len(superNodePayload2.ReceiptsRlp)).To(Equal(1))
+			Expect(superNodePayload2.ReceiptsRlp[0]).To(Equal(expectedRctForStorageRLP1))
+
+			superNodePayload3, err := filterer.FilterResponse(rctTopicsAndContractFilter, *mocks.MockIPLDPayload)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(superNodePayload3.BlockNumber.Int64()).To(Equal(mocks.MockSeeNodePayload.BlockNumber.Int64()))
+			Expect(len(superNodePayload3.HeadersRlp)).To(Equal(0))
+			Expect(len(superNodePayload3.UnclesRlp)).To(Equal(0))
+			Expect(len(superNodePayload3.TransactionsRlp)).To(Equal(0))
+			Expect(len(superNodePayload3.StorageNodesRlp)).To(Equal(0))
+			Expect(len(superNodePayload3.StateNodesRlp)).To(Equal(0))
+			Expect(len(superNodePayload3.ReceiptsRlp)).To(Equal(1))
+			Expect(superNodePayload3.ReceiptsRlp[0]).To(Equal(expectedRctForStorageRLP1))
+
+			superNodePayload4, err := filterer.FilterResponse(rctContractsAndTopicFilter, *mocks.MockIPLDPayload)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(superNodePayload4.BlockNumber.Int64()).To(Equal(mocks.MockSeeNodePayload.BlockNumber.Int64()))
+			Expect(len(superNodePayload4.HeadersRlp)).To(Equal(0))
+			Expect(len(superNodePayload4.UnclesRlp)).To(Equal(0))
+			Expect(len(superNodePayload4.TransactionsRlp)).To(Equal(0))
+			Expect(len(superNodePayload4.StorageNodesRlp)).To(Equal(0))
+			Expect(len(superNodePayload4.StateNodesRlp)).To(Equal(0))
+			Expect(len(superNodePayload4.ReceiptsRlp)).To(Equal(1))
+			Expect(superNodePayload4.ReceiptsRlp[0]).To(Equal(expectedRctForStorageRLP2))
+
+			superNodePayload5, err := filterer.FilterResponse(rctsForAllCollectedTrxs, *mocks.MockIPLDPayload)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(superNodePayload5.BlockNumber.Int64()).To(Equal(mocks.MockSeeNodePayload.BlockNumber.Int64()))
+			Expect(len(superNodePayload5.HeadersRlp)).To(Equal(0))
+			Expect(len(superNodePayload5.UnclesRlp)).To(Equal(0))
+			Expect(len(superNodePayload5.TransactionsRlp)).To(Equal(2))
+			Expect(super_node.ListContainsBytes(superNodePayload5.TransactionsRlp, mocks.MockTransactions.GetRlp(0))).To(BeTrue())
+			Expect(super_node.ListContainsBytes(superNodePayload5.TransactionsRlp, mocks.MockTransactions.GetRlp(1))).To(BeTrue())
+			Expect(len(superNodePayload5.StorageNodesRlp)).To(Equal(0))
+			Expect(len(superNodePayload5.StateNodesRlp)).To(Equal(0))
+			Expect(len(superNodePayload5.ReceiptsRlp)).To(Equal(2))
+			Expect(super_node.ListContainsBytes(superNodePayload5.ReceiptsRlp, expectedRctForStorageRLP1)).To(BeTrue())
+			Expect(super_node.ListContainsBytes(superNodePayload5.ReceiptsRlp, expectedRctForStorageRLP2)).To(BeTrue())
+
+			superNodePayload6, err := filterer.FilterResponse(rctsForSelectCollectedTrxs, *mocks.MockIPLDPayload)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(superNodePayload6.BlockNumber.Int64()).To(Equal(mocks.MockSeeNodePayload.BlockNumber.Int64()))
+			Expect(len(superNodePayload6.HeadersRlp)).To(Equal(0))
+			Expect(len(superNodePayload6.UnclesRlp)).To(Equal(0))
+			Expect(len(superNodePayload6.TransactionsRlp)).To(Equal(1))
+			Expect(super_node.ListContainsBytes(superNodePayload5.TransactionsRlp, mocks.MockTransactions.GetRlp(1))).To(BeTrue())
+			Expect(len(superNodePayload6.StorageNodesRlp)).To(Equal(0))
+			Expect(len(superNodePayload6.StateNodesRlp)).To(Equal(0))
+			Expect(len(superNodePayload6.ReceiptsRlp)).To(Equal(1))
+			Expect(superNodePayload4.ReceiptsRlp[0]).To(Equal(expectedRctForStorageRLP2))
+
+			superNodePayload7, err := filterer.FilterResponse(stateFilter, *mocks.MockIPLDPayload)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(superNodePayload7.BlockNumber.Int64()).To(Equal(mocks.MockSeeNodePayload.BlockNumber.Int64()))
+			Expect(len(superNodePayload7.HeadersRlp)).To(Equal(0))
+			Expect(len(superNodePayload7.UnclesRlp)).To(Equal(0))
+			Expect(len(superNodePayload7.TransactionsRlp)).To(Equal(0))
+			Expect(len(superNodePayload7.StorageNodesRlp)).To(Equal(0))
+			Expect(len(superNodePayload7.ReceiptsRlp)).To(Equal(0))
+			Expect(len(superNodePayload7.StateNodesRlp)).To(Equal(1))
+			Expect(superNodePayload7.StateNodesRlp[mocks.ContractLeafKey]).To(Equal(mocks.ValueBytes))
+		})
+	})
+})
+
+func getReceiptForStorageRLP(receipts types.Receipts, i int) []byte {
+	receiptForStorage := (*types.ReceiptForStorage)(receipts[i])
+	receiptBuffer := new(bytes.Buffer)
+	err := receiptForStorage.EncodeRLP(receiptBuffer)
+	Expect(err).ToNot(HaveOccurred())
+	return receiptBuffer.Bytes()
+}

pkg/super_node/mocks/repository.go

@@ -20,12 +20,12 @@ import "github.com/vulcanize/vulcanizedb/pkg/ipfs"
 
 // CIDRepository is the underlying struct for the Repository interface
 type CIDRepository struct {
-	PassedCIDPayload *ipfs.CIDPayload
+	PassedCIDPayload []*ipfs.CIDPayload
 	ReturnErr        error
 }
 
 // Index indexes a cidPayload in Postgres
 func (repo *CIDRepository) Index(cidPayload *ipfs.CIDPayload) error {
-	repo.PassedCIDPayload = cidPayload
+	repo.PassedCIDPayload = append(repo.PassedCIDPayload, cidPayload)
 	return repo.ReturnErr
 }

pkg/super_node/mocks/retriever.go

@@ -0,0 +1,44 @@
+package mocks
+
+import (
+	"github.com/vulcanize/vulcanizedb/pkg/config"
+	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
+)
+
+// MockCIDRetriever is a mock CID retriever for use in tests
+type MockCIDRetriever struct {
+	GapsToRetrieve              [][2]int64
+	GapsToRetrieveErr           error
+	CalledTimes                 int
+	FirstBlockNumberToReturn    int64
+	RetrieveFirstBlockNumberErr error
+}
+
+// RetrieveCIDs mock method
+func (*MockCIDRetriever) RetrieveCIDs(streamFilters config.Subscription, blockNumber int64) (*ipfs.CIDWrapper, error) {
+	panic("implement me")
+}
+
+// RetrieveLastBlockNumber mock method
+func (*MockCIDRetriever) RetrieveLastBlockNumber() (int64, error) {
+	panic("implement me")
+}
+
+// RetrieveFirstBlockNumber mock method
+func (mcr *MockCIDRetriever) RetrieveFirstBlockNumber() (int64, error) {
+	return mcr.FirstBlockNumberToReturn, mcr.RetrieveFirstBlockNumberErr
+}
+
+// RetrieveGapsInData mock method
+func (mcr *MockCIDRetriever) RetrieveGapsInData() ([][2]int64, error) {
+	mcr.CalledTimes++
+	return mcr.GapsToRetrieve, mcr.GapsToRetrieveErr
+}
+
+// SetGapsToRetrieve mock method
+func (mcr *MockCIDRetriever) SetGapsToRetrieve(gaps [][2]int64) {
+	if mcr.GapsToRetrieve == nil {
+		mcr.GapsToRetrieve = make([][2]int64, 0)
+	}
+	mcr.GapsToRetrieve = append(mcr.GapsToRetrieve, gaps...)
+}

pkg/super_node/repository.go

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package seed_node
+package super_node
 
 import (
 	"github.com/jmoiron/sqlx"

pkg/super_node/repository_test.go

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package seed_node_test
+package super_node_test
 
 import (
 	. "github.com/onsi/ginkgo"
@@ -23,23 +23,23 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
 	"github.com/vulcanize/vulcanizedb/pkg/ipfs/mocks"
-	"github.com/vulcanize/vulcanizedb/pkg/seed_node"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node"
 )
 
 var (
 	db   *postgres.DB
 	err  error
-	repo seed_node.CIDRepository
+	repo super_node.CIDRepository
 )
 
 var _ = Describe("Repository", func() {
 	BeforeEach(func() {
-		db, err = seed_node.SetupDB()
+		db, err = super_node.SetupDB()
 		Expect(err).ToNot(HaveOccurred())
-		repo = seed_node.NewCIDRepository(db)
+		repo = super_node.NewCIDRepository(db)
 	})
 	AfterEach(func() {
-		seed_node.TearDownDB(db)
+		super_node.TearDownDB(db)
 	})
 
 	Describe("Index", func() {
@@ -61,8 +61,8 @@ var _ = Describe("Repository", func() {
 			err = db.Select(&trxs, pgStr, 1)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(trxs)).To(Equal(2))
-			Expect(seed_node.ListContainsString(trxs, "mockTrxCID1")).To(BeTrue())
-			Expect(seed_node.ListContainsString(trxs, "mockTrxCID2")).To(BeTrue())
+			Expect(super_node.ListContainsString(trxs, "mockTrxCID1")).To(BeTrue())
+			Expect(super_node.ListContainsString(trxs, "mockTrxCID2")).To(BeTrue())
 			// check receipts were properly indexed
 			rcts := make([]string, 0)
 			pgStr = `SELECT receipt_cids.cid FROM receipt_cids, transaction_cids, header_cids
@@ -72,8 +72,8 @@ var _ = Describe("Repository", func() {
 			err = db.Select(&rcts, pgStr, 1)
 			Expect(err).ToNot(HaveOccurred())
 			Expect(len(rcts)).To(Equal(2))
-			Expect(seed_node.ListContainsString(rcts, "mockRctCID1")).To(BeTrue())
-			Expect(seed_node.ListContainsString(rcts, "mockRctCID2")).To(BeTrue())
+			Expect(super_node.ListContainsString(rcts, "mockRctCID1")).To(BeTrue())
+			Expect(super_node.ListContainsString(rcts, "mockRctCID2")).To(BeTrue())
 			// check that state nodes were properly indexed
 			stateNodes := make([]ipfs.StateNodeCID, 0)
 			pgStr = `SELECT state_cids.cid, state_cids.state_key, state_cids.leaf FROM state_cids INNER JOIN header_cids ON (state_cids.header_id = header_cids.id)

pkg/super_node/retriever.go

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package seed_node
+package super_node
 
 import (
 	"math/big"
@@ -33,6 +33,7 @@ type CIDRetriever interface {
 	RetrieveCIDs(streamFilters config.Subscription, blockNumber int64) (*ipfs.CIDWrapper, error)
 	RetrieveLastBlockNumber() (int64, error)
 	RetrieveFirstBlockNumber() (int64, error)
+	RetrieveGapsInData() ([][2]int64, error)
 }
 
 // EthCIDRetriever is the underlying struct supporting the CIDRetriever interface
@@ -303,3 +304,26 @@ func (ecr *EthCIDRetriever) retrieveStorageCIDs(tx *sqlx.Tx, streamFilters confi
 	err := tx.Select(&storageNodeCIDs, pgStr, args...)
 	return storageNodeCIDs, err
 }
+
+type gap struct {
+	Start int64 `db:"start"`
+	Stop  int64 `db:"stop"`
+}
+
+func (ecr *EthCIDRetriever) RetrieveGapsInData() ([][2]int64, error) {
+	pgStr := `SELECT header_cids.block_number + 1 AS start, min(fr.block_number) - 1 AS stop FROM header_cids
+				LEFT JOIN header_cids r on header_cids.block_number = r.block_number - 1
+				LEFT JOIN header_cids fr on header_cids.block_number < fr.block_number
+				WHERE r.block_number is NULL and fr.block_number IS NOT NULL
+				GROUP BY header_cids.block_number, r.block_number`
+	gaps := make([]gap, 0)
+	err := ecr.db.Select(&gaps, pgStr)
+	if err != nil {
+		return nil, err
+	}
+	gapRanges := make([][2]int64, 0)
+	for _, gap := range gaps {
+		gapRanges = append(gapRanges, [2]int64{gap.Start, gap.Stop})
+	}
+	return gapRanges, nil
+}

pkg/super_node/retriever_test.go

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package seed_node_test
+package super_node_test
 
 import (
 	"math/big"
@@ -25,11 +25,11 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/config"
 	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
 	"github.com/vulcanize/vulcanizedb/pkg/ipfs/mocks"
-	"github.com/vulcanize/vulcanizedb/pkg/seed_node"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node"
 )
 
 var (
-	retriever  seed_node.CIDRetriever
+	retriever  super_node.CIDRetriever
 	openFilter = config.Subscription{
 		StartingBlock: big.NewInt(0),
 		EndingBlock:   big.NewInt(1),
@@ -178,15 +178,15 @@ var (
 
 var _ = Describe("Retriever", func() {
 	BeforeEach(func() {
-		db, err = seed_node.SetupDB()
+		db, err = super_node.SetupDB()
 		Expect(err).ToNot(HaveOccurred())
-		repo = seed_node.NewCIDRepository(db)
+		repo = super_node.NewCIDRepository(db)
 		err = repo.Index(mocks.MockCIDPayload)
 		Expect(err).ToNot(HaveOccurred())
-		retriever = seed_node.NewCIDRetriever(db)
+		retriever = super_node.NewCIDRetriever(db)
 	})
 	AfterEach(func() {
-		seed_node.TearDownDB(db)
+		super_node.TearDownDB(db)
 	})
 
 	Describe("RetrieveCIDs", func() {
@@ -197,11 +197,11 @@ var _ = Describe("Retriever", func() {
 			Expect(len(cidWrapper.Headers)).To(Equal(1))
 			Expect(cidWrapper.Headers).To(Equal(mocks.MockCIDWrapper.Headers))
 			Expect(len(cidWrapper.Transactions)).To(Equal(2))
-			Expect(seed_node.ListContainsString(cidWrapper.Transactions, mocks.MockCIDWrapper.Transactions[0])).To(BeTrue())
-			Expect(seed_node.ListContainsString(cidWrapper.Transactions, mocks.MockCIDWrapper.Transactions[1])).To(BeTrue())
+			Expect(super_node.ListContainsString(cidWrapper.Transactions, mocks.MockCIDWrapper.Transactions[0])).To(BeTrue())
+			Expect(super_node.ListContainsString(cidWrapper.Transactions, mocks.MockCIDWrapper.Transactions[1])).To(BeTrue())
 			Expect(len(cidWrapper.Receipts)).To(Equal(2))
-			Expect(seed_node.ListContainsString(cidWrapper.Receipts, mocks.MockCIDWrapper.Receipts[0])).To(BeTrue())
-			Expect(seed_node.ListContainsString(cidWrapper.Receipts, mocks.MockCIDWrapper.Receipts[1])).To(BeTrue())
+			Expect(super_node.ListContainsString(cidWrapper.Receipts, mocks.MockCIDWrapper.Receipts[0])).To(BeTrue())
+			Expect(super_node.ListContainsString(cidWrapper.Receipts, mocks.MockCIDWrapper.Receipts[1])).To(BeTrue())
 			Expect(len(cidWrapper.StateNodes)).To(Equal(2))
 			for _, stateNode := range cidWrapper.StateNodes {
 				if stateNode.CID == "mockStateCID1" {
@@ -265,13 +265,13 @@ var _ = Describe("Retriever", func() {
 			Expect(cidWrapper5.BlockNumber).To(Equal(mocks.MockCIDWrapper.BlockNumber))
 			Expect(len(cidWrapper5.Headers)).To(Equal(0))
 			Expect(len(cidWrapper5.Transactions)).To(Equal(2))
-			Expect(seed_node.ListContainsString(cidWrapper5.Transactions, "mockTrxCID1")).To(BeTrue())
-			Expect(seed_node.ListContainsString(cidWrapper5.Transactions, "mockTrxCID2")).To(BeTrue())
+			Expect(super_node.ListContainsString(cidWrapper5.Transactions, "mockTrxCID1")).To(BeTrue())
+			Expect(super_node.ListContainsString(cidWrapper5.Transactions, "mockTrxCID2")).To(BeTrue())
 			Expect(len(cidWrapper5.StateNodes)).To(Equal(0))
 			Expect(len(cidWrapper5.StorageNodes)).To(Equal(0))
 			Expect(len(cidWrapper5.Receipts)).To(Equal(2))
-			Expect(seed_node.ListContainsString(cidWrapper5.Receipts, "mockRctCID1")).To(BeTrue())
-			Expect(seed_node.ListContainsString(cidWrapper5.Receipts, "mockRctCID2")).To(BeTrue())
+			Expect(super_node.ListContainsString(cidWrapper5.Receipts, "mockRctCID1")).To(BeTrue())
+			Expect(super_node.ListContainsString(cidWrapper5.Receipts, "mockRctCID2")).To(BeTrue())
 
 			cidWrapper6, err := retriever.RetrieveCIDs(rctsForSelectCollectedTrxs, 1)
 			Expect(err).ToNot(HaveOccurred())

pkg/super_node/seed_node_suite_test.go

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package seed_node_test
+package super_node_test
 
 import (
 	"io/ioutil"
@@ -25,7 +25,7 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-func TestSeedNode(t *testing.T) {
+func TestSuperNode(t *testing.T) {
 	RegisterFailHandler(Fail)
 	RunSpecs(t, "Seed Node Suite Test")
 }

pkg/super_node/service.go

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package seed_node
+package super_node
 
 import (
 	"sync"
@@ -36,7 +36,9 @@ import (
 	"github.com/vulcanize/vulcanizedb/pkg/ipfs"
 )
 
-const payloadChanBufferSize = 20000 // the max eth sub buffer size
+const (
+	payloadChanBufferSize = 20000 // the max eth sub buffer size
+)
 
 // NodeInterface is the top level interface for streaming, converting to IPLDs, publishing,
 // and indexing all Ethereum data; screening this data; and serving it up to subscribed clients
@@ -49,14 +51,17 @@ type NodeInterface interface {
 	// Main event loop for handling client pub-sub
 	ScreenAndServe(screenAndServePayload <-chan ipfs.IPLDPayload, screenAndServeQuit <-chan bool)
 	// Method to subscribe to receive state diff processing output
-	Subscribe(id rpc.ID, sub chan<- streamer.SeedNodePayload, quitChan chan<- bool, streamFilters config.Subscription)
+	Subscribe(id rpc.ID, sub chan<- streamer.SuperNodePayload, quitChan chan<- bool, streamFilters config.Subscription)
 	// Method to unsubscribe from state diff processing
 	Unsubscribe(id rpc.ID)
 	// Method to access the Geth node info for this service
 	Node() core.Node
+	// Method used to retrieve the underlying IPFS publisher for this service, so that is can be used for backfilling
+	// This is needed because it's not possible to initialize two ipfs nodes at the same path
+	GetPublisher() ipfs.IPLDPublisher
 }
 
-// Service is the underlying struct for the SyncAndPublish interface
+// Service is the underlying struct for the super node
 type Service struct {
 	// Used to sync access to the Subscriptions
 	sync.Mutex
@@ -71,7 +76,7 @@ type Service struct {
 	// Interface for filtering and serving data according to subscribed clients according to their specification
 	Filterer ResponseFilterer
 	// Interface for fetching ETH-IPLD objects from IPFS
-	Fetcher ipfs.IPLDFetcher
+	IPLDFetcher ipfs.IPLDFetcher
 	// Interface for searching and retrieving CIDs from Postgres index
 	Retriever CIDRetriever
 	// Interface for resolving ipfs blocks to their data types
@@ -86,17 +91,17 @@ type Service struct {
 	SubscriptionTypes map[common.Hash]config.Subscription
 	// Number of workers
 	WorkerPoolSize int
-	// Info for the Geth node that this seed node is working with
+	// Info for the Geth node that this super node is working with
 	gethNode core.Node
 }
 
-// NewSeedNode creates a new seed_node.Interface using an underlying seed_node.Service struct
-func NewSeedNode(ipfsPath string, db *postgres.DB, rpcClient core.RpcClient, qc chan bool, workers int, node core.Node) (NodeInterface, error) {
+// NewSuperNode creates a new super_node.Interface using an underlying super_node.Service struct
+func NewSuperNode(ipfsPath string, db *postgres.DB, rpcClient core.RpcClient, qc chan bool, workers int, node core.Node) (NodeInterface, error) {
 	publisher, err := ipfs.NewIPLDPublisher(ipfsPath)
 	if err != nil {
 		return nil, err
 	}
-	fetcher, err := ipfs.NewIPLDFetcher(ipfsPath)
+	ipldFetcher, err := ipfs.NewIPLDFetcher(ipfsPath)
 	if err != nil {
 		return nil, err
 	}
@@ -106,7 +111,7 @@ func NewSeedNode(ipfsPath string, db *postgres.DB, rpcClient core.RpcClient, qc
 		Converter:         ipfs.NewPayloadConverter(params.MainnetChainConfig),
 		Publisher:         publisher,
 		Filterer:          NewResponseFilterer(),
-		Fetcher:           fetcher,
+		IPLDFetcher:       ipldFetcher,
 		Retriever:         NewCIDRetriever(db),
 		Resolver:          ipfs.NewIPLDResolver(),
 		PayloadChan:       make(chan statediff.Payload, payloadChanBufferSize),
@@ -123,13 +128,13 @@ func (sap *Service) Protocols() []p2p.Protocol {
 	return []p2p.Protocol{}
 }
 
-// APIs returns the RPC descriptors the StateDiffingService offers
+// APIs returns the RPC descriptors the super node service offers
 func (sap *Service) APIs() []rpc.API {
 	return []rpc.API{
 		{
 			Namespace: APIName,
 			Version:   APIVersion,
-			Service:   NewPublicSeedNodeAPI(sap),
+			Service:   NewPublicSuperNodeAPI(sap),
 			Public:    true,
 		},
 	}
@@ -256,7 +261,7 @@ func (sap *Service) sendResponse(payload ipfs.IPLDPayload) error {
 		for id, sub := range subs {
 			select {
 			case sub.PayloadChan <- response:
-				log.Infof("sending seed node payload to subscription %s", id)
+				log.Infof("sending super node payload to subscription %s", id)
 			default:
 				log.Infof("unable to send payload to subscription %s; channel has no receiver", id)
 			}
@@ -267,8 +272,8 @@ func (sap *Service) sendResponse(payload ipfs.IPLDPayload) error {
 }
 
 // Subscribe is used by the API to subscribe to the service loop
-func (sap *Service) Subscribe(id rpc.ID, sub chan<- streamer.SeedNodePayload, quitChan chan<- bool, streamFilters config.Subscription) {
-	log.Info("Subscribing to the seed node service")
+func (sap *Service) Subscribe(id rpc.ID, sub chan<- streamer.SuperNodePayload, quitChan chan<- bool, streamFilters config.Subscription) {
+	log.Info("Subscribing to the super node service")
 	// Subscription type is defined as the hash of its content
 	// Group subscriptions by type and screen payloads once for subs of the same type
 	by, err := rlp.EncodeToBytes(streamFilters)
@@ -305,7 +310,7 @@ func (sap *Service) backFill(sub Subscription, id rpc.ID, con config.Subscriptio
 	var err error
 	startingBlock, err = sap.Retriever.RetrieveFirstBlockNumber()
 	if err != nil {
-		sub.PayloadChan <- streamer.SeedNodePayload{
+		sub.PayloadChan <- streamer.SuperNodePayload{
 			ErrMsg: "unable to set block range start; error: " + err.Error(),
 		}
 	}
@@ -314,7 +319,7 @@ func (sap *Service) backFill(sub Subscription, id rpc.ID, con config.Subscriptio
 	}
 	endingBlock, err = sap.Retriever.RetrieveLastBlockNumber()
 	if err != nil {
-		sub.PayloadChan <- streamer.SeedNodePayload{
+		sub.PayloadChan <- streamer.SuperNodePayload{
 			ErrMsg: "unable to set block range end; error: " + err.Error(),
 		}
 	}
@@ -327,10 +332,10 @@ func (sap *Service) backFill(sub Subscription, id rpc.ID, con config.Subscriptio
 	// the blocknumbers in the payloads they receive to keep things in order
 	// TODO: separate backfill into a different rpc subscription method altogether?
 	go func() {
-		for i := con.StartingBlock.Int64(); i <= endingBlock; i++ {
+		for i := startingBlock; i <= endingBlock; i++ {
 			cidWrapper, err := sap.Retriever.RetrieveCIDs(con, i)
 			if err != nil {
-				sub.PayloadChan <- streamer.SeedNodePayload{
+				sub.PayloadChan <- streamer.SuperNodePayload{
 					ErrMsg: "CID retrieval error: " + err.Error(),
 				}
 				continue
@@ -338,10 +343,10 @@ func (sap *Service) backFill(sub Subscription, id rpc.ID, con config.Subscriptio
 			if ipfs.EmptyCIDWrapper(*cidWrapper) {
 				continue
 			}
-			blocksWrapper, err := sap.Fetcher.FetchCIDs(*cidWrapper)
+			blocksWrapper, err := sap.IPLDFetcher.FetchCIDs(*cidWrapper)
 			if err != nil {
 				log.Error(err)
-				sub.PayloadChan <- streamer.SeedNodePayload{
+				sub.PayloadChan <- streamer.SuperNodePayload{
 					ErrMsg: "IPLD fetching error: " + err.Error(),
 				}
 				continue
@@ -349,14 +354,14 @@ func (sap *Service) backFill(sub Subscription, id rpc.ID, con config.Subscriptio
 			backFillIplds, err := sap.Resolver.ResolveIPLDs(*blocksWrapper)
 			if err != nil {
 				log.Error(err)
-				sub.PayloadChan <- streamer.SeedNodePayload{
+				sub.PayloadChan <- streamer.SuperNodePayload{
 					ErrMsg: "IPLD resolving error: " + err.Error(),
 				}
 				continue
 			}
 			select {
 			case sub.PayloadChan <- backFillIplds:
-				log.Infof("sending seed node back-fill payload to subscription %s", id)
+				log.Infof("sending super node back-fill payload to subscription %s", id)
 			default:
 				log.Infof("unable to send back-fill payload to subscription %s; channel has no receiver", id)
 			}
@@ -366,7 +371,7 @@ func (sap *Service) backFill(sub Subscription, id rpc.ID, con config.Subscriptio
 
 // Unsubscribe is used to unsubscribe to the StateDiffingService loop
 func (sap *Service) Unsubscribe(id rpc.ID) {
-	log.Info("Unsubscribing from the seed node service")
+	log.Info("Unsubscribing from the super node service")
 	sap.Lock()
 	for ty := range sap.Subscriptions {
 		delete(sap.Subscriptions[ty], id)
@@ -381,7 +386,7 @@ func (sap *Service) Unsubscribe(id rpc.ID) {
 
 // Start is used to begin the service
 func (sap *Service) Start(*p2p.Server) error {
-	log.Info("Starting seed node service")
+	log.Info("Starting super node service")
 	wg := new(sync.WaitGroup)
 	payloadChan := make(chan ipfs.IPLDPayload, payloadChanBufferSize)
 	quitChan := make(chan bool, 1)
@@ -394,7 +399,7 @@ func (sap *Service) Start(*p2p.Server) error {
 
 // Stop is used to close down the service
 func (sap *Service) Stop() error {
-	log.Info("Stopping seed node service")
+	log.Info("Stopping super node service")
 	close(sap.QuitChan)
 	return nil
 }
@@ -421,3 +426,7 @@ func (sap *Service) close() {
 	}
 	sap.Unlock()
 }
+
+func (sap *Service) GetPublisher() ipfs.IPLDPublisher {
+	return sap.Publisher
+}

pkg/super_node/service_test.go

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package seed_node_test
+package super_node_test
 
 import (
 	"sync"
@@ -27,8 +27,8 @@ import (
 
 	mocks2 "github.com/vulcanize/vulcanizedb/libraries/shared/mocks"
 	"github.com/vulcanize/vulcanizedb/pkg/ipfs/mocks"
-	"github.com/vulcanize/vulcanizedb/pkg/seed_node"
-	mocks3 "github.com/vulcanize/vulcanizedb/pkg/seed_node/mocks"
+	"github.com/vulcanize/vulcanizedb/pkg/super_node"
+	mocks3 "github.com/vulcanize/vulcanizedb/pkg/super_node/mocks"
 )
 
 var _ = Describe("Service", func() {
@@ -55,7 +55,7 @@ var _ = Describe("Service", func() {
 				ReturnIPLDPayload: mocks.MockIPLDPayload,
 				ReturnErr:         nil,
 			}
-			processor := &seed_node.Service{
+			processor := &super_node.Service{
 				Repository:     mockCidRepo,
 				Publisher:      mockPublisher,
 				Streamer:       mockStreamer,
@@ -70,7 +70,8 @@ var _ = Describe("Service", func() {
 			quitChan <- true
 			wg.Wait()
 			Expect(mockConverter.PassedStatediffPayload).To(Equal(mocks.MockStateDiffPayload))
-			Expect(mockCidRepo.PassedCIDPayload).To(Equal(mocks.MockCIDPayload))
+			Expect(len(mockCidRepo.PassedCIDPayload)).To(Equal(1))
+			Expect(mockCidRepo.PassedCIDPayload[0]).To(Equal(mocks.MockCIDPayload))
 			Expect(mockPublisher.PassedIPLDPayload).To(Equal(mocks.MockIPLDPayload))
 			Expect(mockStreamer.PassedPayloadChan).To(Equal(payloadChan))
 		})

pkg/super_node/subscription.go

@@ -14,14 +14,14 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package seed_node
+package super_node
 
 import (
 	"github.com/vulcanize/vulcanizedb/libraries/shared/streamer"
 )
 
-// Subscription holds the information for an individual client subscription to the seed node
+// Subscription holds the information for an individual client subscription to the super node
 type Subscription struct {
-	PayloadChan chan<- streamer.SeedNodePayload
+	PayloadChan chan<- streamer.SuperNodePayload
 	QuitChan    chan<- bool
 }

pkg/super_node/test_helpers.go

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <http://www.gnu.org/licenses/>.
 
-package seed_node
+package super_node
 
 import (
 	"bytes"