integrate backfill into storage watcher; documentation for storage backfill

libraries/shared/mocks/backfiller.go

@@ -0,0 +1,39 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package mocks
+
+import "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
+
+// BackFiller mock for tests
+type BackFiller struct {
+	StorageDiffsToReturn []utils.StorageDiff
+	BackFillErr          error
+	PassedStartingBlock  uint64
+	PassedEndingBlock    uint64
+}
+
+// SetStorageDiffsToReturn for tests
+func (backFiller *BackFiller) SetStorageDiffsToReturn(diffs []utils.StorageDiff) {
+	backFiller.StorageDiffsToReturn = diffs
+}
+
+// BackFill mock method
+func (backFiller *BackFiller) BackFill(startingBlock, endingBlock uint64) ([]utils.StorageDiff, error) {
+	backFiller.PassedStartingBlock = startingBlock
+	backFiller.PassedEndingBlock = endingBlock
+	return backFiller.StorageDiffsToReturn, backFiller.BackFillErr
+}

libraries/shared/mocks/batch_client.go

@@ -0,0 +1,64 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package mocks
+
+import (
+	"encoding/json"
+	"errors"
+
+	"github.com/ethereum/go-ethereum/statediff"
+	"github.com/vulcanize/vulcanizedb/pkg/eth/client"
+)
+
+// BackFillerClient is a mock client for use in backfiller tests
+type BackFillerClient struct {
+	MappedStateDiffAt map[uint64][]byte
+}
+
+// SetReturnDiffAt method to set what statediffs the mock client returns
+func (mc *BackFillerClient) SetReturnDiffAt(height uint64, diffPayload statediff.Payload) error {
+	if mc.MappedStateDiffAt == nil {
+		mc.MappedStateDiffAt = make(map[uint64][]byte)
+	}
+	by, err := json.Marshal(diffPayload)
+	if err != nil {
+		return err
+	}
+	mc.MappedStateDiffAt[height] = by
+	return nil
+}
+
+// BatchCall mockClient method to simulate batch call to geth
+func (mc *BackFillerClient) BatchCall(batch []client.BatchElem) error {
+	if mc.MappedStateDiffAt == nil {
+		return errors.New("mockclient needs to be initialized with statediff payloads and errors")
+	}
+	for _, batchElem := range batch {
+		if len(batchElem.Args) != 1 {
+			return errors.New("expected batch elem to contain single argument")
+		}
+		blockHeight, ok := batchElem.Args[0].(uint64)
+		if !ok {
+			return errors.New("expected batch elem argument to be a uint64")
+		}
+		err := json.Unmarshal(mc.MappedStateDiffAt[blockHeight], batchElem.Result)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}

libraries/shared/mocks/state_diff_fetcher.go

@@ -0,0 +1,48 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package mocks
+
+import (
+	"errors"
+
+	"github.com/ethereum/go-ethereum/statediff"
+)
+
+// StateDiffFetcher mock for tests
+type StateDiffFetcher struct {
+	PayloadsToReturn     map[uint64]*statediff.Payload
+	FetchErr             error
+	CalledAtBlockHeights [][]uint64
+}
+
+// SetPayloadsToReturn for tests
+func (fetcher *StateDiffFetcher) SetPayloadsToReturn(payloads map[uint64]*statediff.Payload) {
+	fetcher.PayloadsToReturn = payloads
+}
+
+// FetchStateDiffsAt mock method
+func (fetcher *StateDiffFetcher) FetchStateDiffsAt(blockHeights []uint64) ([]*statediff.Payload, error) {
+	fetcher.CalledAtBlockHeights = append(fetcher.CalledAtBlockHeights, blockHeights)
+	if fetcher.PayloadsToReturn == nil {
+		return nil, errors.New("MockStateDiffFetcher needs to be initialized with payloads to return")
+	}
+	results := make([]*statediff.Payload, 0, len(blockHeights))
+	for _, height := range blockHeights {
+		results = append(results, fetcher.PayloadsToReturn[height])
+	}
+	return results, nil
+}

libraries/shared/mocks/storage_fetcher.go

@@ -18,16 +18,19 @@ package mocks
 
 import "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 
-type MockStorageFetcher struct {
+// ClosingStorageFetcher is a mock fetcher for use in tests without backfilling
+type ClosingStorageFetcher struct {
 	DiffsToReturn []utils.StorageDiff
 	ErrsToReturn  []error
 }
 
-func NewMockStorageFetcher() *MockStorageFetcher {
-	return &MockStorageFetcher{}
+// NewClosingStorageFetcher returns a new ClosingStorageFetcher
+func NewClosingStorageFetcher() *ClosingStorageFetcher {
+	return &ClosingStorageFetcher{}
 }
 
-func (fetcher *MockStorageFetcher) FetchStorageDiffs(out chan<- utils.StorageDiff, errs chan<- error) {
+// FetchStorageDiffs mock method
+func (fetcher *ClosingStorageFetcher) FetchStorageDiffs(out chan<- utils.StorageDiff, errs chan<- error) {
 	defer close(out)
 	defer close(errs)
 	for _, err := range fetcher.ErrsToReturn {
@@ -37,3 +40,24 @@ func (fetcher *MockStorageFetcher) FetchStorageDiffs(out chan<- utils.StorageDif
 		out <- diff
 	}
 }
+
+// StorageFetcher is a mock fetcher for use in tests with backfilling
+type StorageFetcher struct {
+	DiffsToReturn []utils.StorageDiff
+	ErrsToReturn  []error
+}
+
+// NewStorageFetcher returns a new StorageFetcher
+func NewStorageFetcher() *StorageFetcher {
+	return &StorageFetcher{}
+}
+
+// FetchStorageDiffs mock method
+func (fetcher *StorageFetcher) FetchStorageDiffs(out chan<- utils.StorageDiff, errs chan<- error) {
+	for _, err := range fetcher.ErrsToReturn {
+		errs <- err
+	}
+	for _, diff := range fetcher.DiffsToReturn {
+		out <- diff
+	}
+}

libraries/shared/mocks/storage_queue.go

@@ -20,27 +20,31 @@ import (
 	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 )
 
+// MockStorageQueue for tests
 type MockStorageQueue struct {
-	AddCalled      bool
-	AddError       error
-	AddPassedDiff  utils.StorageDiff
-	DeleteErr      error
-	DeletePassedID int
-	GetAllErr      error
-	DiffsToReturn  []utils.StorageDiff
+	AddCalled       bool
+	AddError        error
+	AddPassedDiffs  []utils.StorageDiff
+	DeleteErr       error
+	DeletePassedIds []int
+	GetAllErr       error
+	DiffsToReturn   []utils.StorageDiff
 }
 
+// Add mock method
 func (queue *MockStorageQueue) Add(diff utils.StorageDiff) error {
 	queue.AddCalled = true
-	queue.AddPassedDiff = diff
+	queue.AddPassedDiffs = append(queue.AddPassedDiffs, diff)
 	return queue.AddError
 }
 
+// Delete mock method
 func (queue *MockStorageQueue) Delete(id int) error {
-	queue.DeletePassedID = id
+	queue.DeletePassedIds = append(queue.DeletePassedIds, id)
 	return queue.DeleteErr
 }
 
+// GetAll mock method
 func (queue *MockStorageQueue) GetAll() ([]utils.StorageDiff, error) {
 	return queue.DiffsToReturn, queue.GetAllErr
 }

libraries/shared/mocks/storage_transformer.go

@@ -18,26 +18,31 @@ package mocks
 
 import (
 	"github.com/ethereum/go-ethereum/common"
+
 	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/transformer"
 	"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
 )
 
+// MockStorageTransformer for tests
 type MockStorageTransformer struct {
 	KeccakOfAddress common.Hash
 	ExecuteErr      error
-	PassedDiff      utils.StorageDiff
+	PassedDiffs     []utils.StorageDiff
 }
 
+// Execute mock method
 func (transformer *MockStorageTransformer) Execute(diff utils.StorageDiff) error {
-	transformer.PassedDiff = diff
+	transformer.PassedDiffs = append(transformer.PassedDiffs, diff)
 	return transformer.ExecuteErr
 }
 
+// KeccakContractAddress mock method
 func (transformer *MockStorageTransformer) KeccakContractAddress() common.Hash {
 	return transformer.KeccakOfAddress
 }
 
+// FakeTransformerInitializer mock method
 func (transformer *MockStorageTransformer) FakeTransformerInitializer(db *postgres.DB) transformer.StorageTransformer {
 	return transformer
 }