split backfill range up into smaller bins and process them concurrently; improve tests; review fixes

libraries/shared/mocks/backfiller.go

@@ -16,14 +16,18 @@
 
 package mocks
 
-import "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
+import (
+	"errors"
+
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
+)
 
 // BackFiller mock for tests
 type BackFiller struct {
 	StorageDiffsToReturn []utils.StorageDiff
-	BackFillErr          error
-	PassedStartingBlock  uint64
+	BackFillErrs         []error
 	PassedEndingBlock    uint64
+	StartingBlock        uint64
 }
 
 // SetStorageDiffsToReturn for tests
@@ -32,8 +36,24 @@ func (backFiller *BackFiller) SetStorageDiffsToReturn(diffs []utils.StorageDiff)
 }
 
 // BackFill mock method
-func (backFiller *BackFiller) BackFill(startingBlock, endingBlock uint64) ([]utils.StorageDiff, error) {
-	backFiller.PassedStartingBlock = startingBlock
+func (backFiller *BackFiller) BackFill(endingBlock uint64, backFill chan utils.StorageDiff, errChan chan error, done chan bool) error {
+	if endingBlock < backFiller.StartingBlock {
+		return errors.New("backfill: ending block number needs to be greater than starting block number")
+	}
 	backFiller.PassedEndingBlock = endingBlock
-	return backFiller.StorageDiffsToReturn, backFiller.BackFillErr
+	go func(backFill chan utils.StorageDiff, errChan chan error, done chan bool) {
+		errLen := len(backFiller.BackFillErrs)
+		for i, diff := range backFiller.StorageDiffsToReturn {
+			if i < errLen {
+				err := backFiller.BackFillErrs[i]
+				if err != nil {
+					errChan <- err
+					continue
+				}
+			}
+			backFill <- diff
+		}
+		done <- true
+	}(backFill, errChan, done)
+	return nil
 }

libraries/shared/mocks/state_diff_fetcher.go

@@ -18,31 +18,33 @@ package mocks
 
 import (
 	"errors"
+	"sync/atomic"
 
 	"github.com/ethereum/go-ethereum/statediff"
 )
 
 // StateDiffFetcher mock for tests
 type StateDiffFetcher struct {
-	PayloadsToReturn     map[uint64]*statediff.Payload
-	FetchErr             error
+	PayloadsToReturn     map[uint64]statediff.Payload
+	FetchErrs            map[uint64]error
 	CalledAtBlockHeights [][]uint64
-}
-
-// SetPayloadsToReturn for tests
-func (fetcher *StateDiffFetcher) SetPayloadsToReturn(payloads map[uint64]*statediff.Payload) {
-	fetcher.PayloadsToReturn = payloads
+	CalledTimes          int64
 }
 
 // FetchStateDiffsAt mock method
-func (fetcher *StateDiffFetcher) FetchStateDiffsAt(blockHeights []uint64) ([]*statediff.Payload, error) {
-	fetcher.CalledAtBlockHeights = append(fetcher.CalledAtBlockHeights, blockHeights)
+func (fetcher *StateDiffFetcher) FetchStateDiffsAt(blockHeights []uint64) ([]statediff.Payload, error) {
 	if fetcher.PayloadsToReturn == nil {
-		return nil, errors.New("MockStateDiffFetcher needs to be initialized with payloads to return")
+		return nil, errors.New("mock StateDiffFetcher needs to be initialized with payloads to return")
 	}
-	results := make([]*statediff.Payload, 0, len(blockHeights))
+	atomic.AddInt64(&fetcher.CalledTimes, 1) // thread-safe increment
+	fetcher.CalledAtBlockHeights = append(fetcher.CalledAtBlockHeights, blockHeights)
+	results := make([]statediff.Payload, 0, len(blockHeights))
 	for _, height := range blockHeights {
 		results = append(results, fetcher.PayloadsToReturn[height])
+		err, ok := fetcher.FetchErrs[height]
+		if ok && err != nil {
+			return nil, err
+		}
 	}
 	return results, nil
 }

libraries/shared/mocks/statediff_streamer.go

@@ -0,0 +1,43 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package mocks
+
+import (
+	"github.com/ethereum/go-ethereum/rpc"
+	"github.com/ethereum/go-ethereum/statediff"
+)
+
+// StateDiffStreamer is the underlying struct for the Streamer interface
+type StateDiffStreamer struct {
+	PassedPayloadChan chan statediff.Payload
+	ReturnSub         *rpc.ClientSubscription
+	ReturnErr         error
+	StreamPayloads    []statediff.Payload
+}
+
+// Stream is the main loop for subscribing to data from the Geth state diff process
+func (sds *StateDiffStreamer) Stream(payloadChan chan statediff.Payload) (*rpc.ClientSubscription, error) {
+	sds.PassedPayloadChan = payloadChan
+
+	go func() {
+		for _, payload := range sds.StreamPayloads {
+			sds.PassedPayloadChan <- payload
+		}
+	}()
+
+	return sds.ReturnSub, sds.ReturnErr
+}

libraries/shared/mocks/storage_fetcher.go

@@ -16,30 +16,9 @@
 
 package mocks
 
-import "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
-
-// ClosingStorageFetcher is a mock fetcher for use in tests without backfilling
-type ClosingStorageFetcher struct {
-	DiffsToReturn []utils.StorageDiff
-	ErrsToReturn  []error
-}
-
-// NewClosingStorageFetcher returns a new ClosingStorageFetcher
-func NewClosingStorageFetcher() *ClosingStorageFetcher {
-	return &ClosingStorageFetcher{}
-}
-
-// FetchStorageDiffs mock method
-func (fetcher *ClosingStorageFetcher) FetchStorageDiffs(out chan<- utils.StorageDiff, errs chan<- error) {
-	defer close(out)
-	defer close(errs)
-	for _, err := range fetcher.ErrsToReturn {
-		errs <- err
-	}
-	for _, diff := range fetcher.DiffsToReturn {
-		out <- diff
-	}
-}
+import (
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
+)
 
 // StorageFetcher is a mock fetcher for use in tests with backfilling
 type StorageFetcher struct {

libraries/shared/mocks/storage_queue.go

@@ -29,6 +29,7 @@ type MockStorageQueue struct {
 	DeletePassedIds []int
 	GetAllErr       error
 	DiffsToReturn   []utils.StorageDiff
+	GetAllCalled    bool
 }
 
 // Add mock method
@@ -41,10 +42,18 @@ func (queue *MockStorageQueue) Add(diff utils.StorageDiff) error {
 // Delete mock method
 func (queue *MockStorageQueue) Delete(id int) error {
 	queue.DeletePassedIds = append(queue.DeletePassedIds, id)
+	diffsToReturn := make([]utils.StorageDiff, 0)
+	for _, diff := range queue.DiffsToReturn {
+		if diff.Id != id {
+			diffsToReturn = append(diffsToReturn, diff)
+		}
+	}
+	queue.DiffsToReturn = diffsToReturn
 	return queue.DeleteErr
 }
 
 // GetAll mock method
 func (queue *MockStorageQueue) GetAll() ([]utils.StorageDiff, error) {
+	queue.GetAllCalled = true
 	return queue.DiffsToReturn, queue.GetAllErr
 }