split backfill range up into smaller bins and process them concurrently; improve tests; review fixes

libraries/shared/fetcher/geth_rpc_storage_fetcher.go

@@ -26,20 +26,24 @@ import (
 	"github.com/vulcanize/vulcanizedb/libraries/shared/streamer"
 )
 
+const (
+	PayloadChanBufferSize = 20000 // the max eth sub buffer size
+)
+
 type GethRPCStorageFetcher struct {
-	statediffPayloadChan chan statediff.Payload
+	StatediffPayloadChan chan statediff.Payload
 	streamer             streamer.Streamer
 }
 
-func NewGethRPCStorageFetcher(streamer streamer.Streamer, statediffPayloadChan chan statediff.Payload) GethRPCStorageFetcher {
+func NewGethRPCStorageFetcher(streamer streamer.Streamer) GethRPCStorageFetcher {
 	return GethRPCStorageFetcher{
-		statediffPayloadChan: statediffPayloadChan,
+		StatediffPayloadChan: make(chan statediff.Payload, PayloadChanBufferSize),
 		streamer:             streamer,
 	}
 }
 
 func (fetcher GethRPCStorageFetcher) FetchStorageDiffs(out chan<- utils.StorageDiff, errs chan<- error) {
-	ethStatediffPayloadChan := fetcher.statediffPayloadChan
+	ethStatediffPayloadChan := fetcher.StatediffPayloadChan
 	clientSubscription, clientSubErr := fetcher.streamer.Stream(ethStatediffPayloadChan)
 	if clientSubErr != nil {
 		errs <- clientSubErr

libraries/shared/fetcher/geth_rpc_storage_fetcher_test.go

@@ -59,15 +59,13 @@ func (streamer *MockStoragediffStreamer) SetPayloads(payloads []statediff.Payloa
 
 var _ = Describe("Geth RPC Storage Fetcher", func() {
 	var streamer MockStoragediffStreamer
-	var statediffPayloadChan chan statediff.Payload
 	var statediffFetcher fetcher.GethRPCStorageFetcher
 	var storagediffChan chan utils.StorageDiff
 	var errorChan chan error
 
 	BeforeEach(func() {
 		streamer = MockStoragediffStreamer{}
-		statediffPayloadChan = make(chan statediff.Payload, 1)
-		statediffFetcher = fetcher.NewGethRPCStorageFetcher(&streamer, statediffPayloadChan)
+		statediffFetcher = fetcher.NewGethRPCStorageFetcher(&streamer)
 		storagediffChan = make(chan utils.StorageDiff)
 		errorChan = make(chan error)
 	})
@@ -91,9 +89,9 @@ var _ = Describe("Geth RPC Storage Fetcher", func() {
 
 		go statediffFetcher.FetchStorageDiffs(storagediffChan, errorChan)
 
-		streamedPayload := <-statediffPayloadChan
+		streamedPayload := <-statediffFetcher.StatediffPayloadChan
 		Expect(streamedPayload).To(Equal(test_data.MockStatediffPayload))
-		Expect(streamer.PassedPayloadChan).To(Equal(statediffPayloadChan))
+		Expect(streamer.PassedPayloadChan).To(Equal(statediffFetcher.StatediffPayloadChan))
 		close(done)
 	})
 

libraries/shared/fetcher/state_diff_fetcher.go

@@ -25,7 +25,7 @@ import (
 
 // StateDiffFetcher is the state diff fetching interface
 type StateDiffFetcher interface {
-	FetchStateDiffsAt(blockHeights []uint64) ([]*statediff.Payload, error)
+	FetchStateDiffsAt(blockHeights []uint64) ([]statediff.Payload, error)
 }
 
 // BatchClient is an interface to a batch-fetching geth rpc client; created to allow mock insertion
@@ -35,6 +35,8 @@ type BatchClient interface {
 
 // stateDiffFetcher is the state diff fetching struct
 type stateDiffFetcher struct {
+	// stateDiffFetcher is thread-safe as long as the underlying client is thread-safe, since it has/modifies no other state
+	// http.Client is thread-safe
 	client BatchClient
 }
 
@@ -49,7 +51,7 @@ func NewStateDiffFetcher(bc BatchClient) StateDiffFetcher {
 
 // FetchStateDiffsAt fetches the statediff payloads at the given block heights
 // Calls StateDiffAt(ctx context.Context, blockNumber uint64) (*Payload, error)
-func (fetcher *stateDiffFetcher) FetchStateDiffsAt(blockHeights []uint64) ([]*statediff.Payload, error) {
+func (fetcher *stateDiffFetcher) FetchStateDiffsAt(blockHeights []uint64) ([]statediff.Payload, error) {
 	batch := make([]client.BatchElem, 0)
 	for _, height := range blockHeights {
 		batch = append(batch, client.BatchElem{
@@ -62,12 +64,15 @@ func (fetcher *stateDiffFetcher) FetchStateDiffsAt(blockHeights []uint64) ([]*st
 	if batchErr != nil {
 		return nil, fmt.Errorf("stateDiffFetcher err: %s", batchErr.Error())
 	}
-	results := make([]*statediff.Payload, 0, len(blockHeights))
+	results := make([]statediff.Payload, 0, len(blockHeights))
 	for _, batchElem := range batch {
 		if batchElem.Error != nil {
 			return nil, fmt.Errorf("stateDiffFetcher err: %s", batchElem.Error.Error())
 		}
-		results = append(results, batchElem.Result.(*statediff.Payload))
+		payload, ok := batchElem.Result.(*statediff.Payload)
+		if ok {
+			results = append(results, *payload)
+		}
 	}
 	return results, nil
 }

libraries/shared/fetcher/state_diff_fetcher_test.go

@@ -17,10 +17,6 @@
 package fetcher_test
 
 import (
-	"bytes"
-
-	"github.com/ethereum/go-ethereum/rlp"
-	"github.com/ethereum/go-ethereum/statediff"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
@@ -51,26 +47,8 @@ var _ = Describe("StateDiffFetcher", func() {
 			stateDiffPayloads, fetchErr := stateDiffFetcher.FetchStateDiffsAt(blockHeights)
 			Expect(fetchErr).ToNot(HaveOccurred())
 			Expect(len(stateDiffPayloads)).To(Equal(2))
-			// Can only rlp encode the slice of diffs as part of a struct
-			// Rlp encoding allows us to compare content of the slices when the order in the slice may vary
-			expectedPayloadsStruct := struct {
-				payloads []*statediff.Payload
-			}{
-				[]*statediff.Payload{
-					&test_data.MockStatediffPayload,
-					&test_data.MockStatediffPayload2,
-				},
-			}
-			expectedPayloadsBytes, rlpErr1 := rlp.EncodeToBytes(expectedPayloadsStruct)
-			Expect(rlpErr1).ToNot(HaveOccurred())
-			receivedPayloadsStruct := struct {
-				payloads []*statediff.Payload
-			}{
-				stateDiffPayloads,
-			}
-			receivedPayloadsBytes, rlpErr2 := rlp.EncodeToBytes(receivedPayloadsStruct)
-			Expect(rlpErr2).ToNot(HaveOccurred())
-			Expect(bytes.Equal(expectedPayloadsBytes, receivedPayloadsBytes)).To(BeTrue())
+			Expect(stateDiffPayloads[0]).To(Equal(test_data.MockStatediffPayload))
+			Expect(stateDiffPayloads[1]).To(Equal(test_data.MockStatediffPayload2))
 		})
 	})
 })