storage backfiller

libraries/shared/fetcher/geth_rpc_storage_fetcher.go

@@ -16,9 +16,11 @@ package fetcher
 
 import (
 	"fmt"
+
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/statediff"
 	"github.com/sirupsen/logrus"
+
 	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/streamer"
 )
@@ -54,7 +56,7 @@ func (fetcher GethRPCStorageFetcher) FetchStorageDiffs(out chan<- utils.StorageD
 			errs <- decodeErr
 		}
 
-		accounts := getAccountsFromDiff(*stateDiff)
+		accounts := utils.GetAccountsFromDiff(*stateDiff)
 		logrus.Trace(fmt.Sprintf("iterating through %d accounts on stateDiff for block %d", len(accounts), stateDiff.BlockNumber))
 		for _, account := range accounts {
 			logrus.Trace(fmt.Sprintf("iterating through %d Storage values on account", len(account.Storage)))
@@ -74,8 +76,3 @@ func (fetcher GethRPCStorageFetcher) FetchStorageDiffs(out chan<- utils.StorageD
 		}
 	}
 }
-
-func getAccountsFromDiff(stateDiff statediff.StateDiff) []statediff.AccountDiff {
-	accounts := append(stateDiff.CreatedAccounts, stateDiff.UpdatedAccounts...)
-	return append(accounts, stateDiff.DeletedAccounts...)
-}

libraries/shared/storage/backfill.go

@@ -0,0 +1,135 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package storage
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/statediff"
+	"github.com/sirupsen/logrus"
+
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
+	"github.com/vulcanize/vulcanizedb/pkg/geth/client"
+)
+
+type IBackFiller interface {
+	BackFill(bfa BackFillerArgs) (map[common.Hash][]utils.StorageDiff, error)
+}
+
+type BatchClient interface {
+	BatchCall(batch []client.BatchElem) error
+}
+
+type BackFiller struct {
+	client BatchClient
+}
+
+type BackFillerArgs struct {
+	// mapping of hashed addresses to a list of the storage key hashes we want to collect at that address
+	WantedStorage map[common.Hash][]common.Hash
+	StartingBlock uint64
+	EndingBlock   uint64
+}
+
+const method = "statediff_stateDiffAt"
+
+func NewStorageBackFiller(bc BatchClient) IBackFiller {
+	return &BackFiller{
+		client: bc,
+	}
+}
+
+// BackFill uses the provided config to fetch and return the state diff at the specified blocknumber
+// StateDiffAt(ctx context.Context, blockNumber uint64) (*Payload, error)
+func (bf *BackFiller) BackFill(bfa BackFillerArgs) (map[common.Hash][]utils.StorageDiff, error) {
+	results := make(map[common.Hash][]utils.StorageDiff, len(bfa.WantedStorage))
+	if bfa.EndingBlock < bfa.StartingBlock {
+		return nil, errors.New("backfill: ending block number needs to be greater than starting block number")
+	}
+	batch := make([]client.BatchElem, 0)
+	for i := bfa.StartingBlock; i <= bfa.EndingBlock; i++ {
+		batch = append(batch, client.BatchElem{
+			Method: method,
+			Args:   []interface{}{i},
+			Result: new(statediff.Payload),
+		})
+	}
+	batchErr := bf.client.BatchCall(batch)
+	if batchErr != nil {
+		return nil, batchErr
+	}
+	for _, batchElem := range batch {
+		payload := batchElem.Result.(*statediff.Payload)
+		if batchElem.Error != nil {
+			return nil, batchElem.Error
+		}
+		block := new(types.Block)
+		blockDecodeErr := rlp.DecodeBytes(payload.BlockRlp, block)
+		if blockDecodeErr != nil {
+			return nil, blockDecodeErr
+		}
+		stateDiff := new(statediff.StateDiff)
+		stateDiffDecodeErr := rlp.DecodeBytes(payload.StateDiffRlp, stateDiff)
+		if stateDiffDecodeErr != nil {
+			return nil, stateDiffDecodeErr
+		}
+		accounts := utils.GetAccountsFromDiff(*stateDiff)
+		for _, account := range accounts {
+			if wantedHashedAddress(bfa.WantedStorage, common.BytesToHash(account.Key)) {
+				logrus.Trace(fmt.Sprintf("iterating through %d Storage values on account", len(account.Storage)))
+				for _, storage := range account.Storage {
+					if wantedHashedStorageKey(bfa.WantedStorage[common.BytesToHash(account.Key)], storage.Key) {
+						diff, formatErr := utils.FromGethStateDiff(account, stateDiff, storage)
+						logrus.Trace("adding storage diff to out channel",
+							"keccak of address: ", diff.HashedAddress.Hex(),
+							"block height: ", diff.BlockHeight,
+							"storage key: ", diff.StorageKey.Hex(),
+							"storage value: ", diff.StorageValue.Hex())
+						if formatErr != nil {
+							return nil, formatErr
+						}
+						results[diff.HashedAddress] = append(results[diff.HashedAddress], diff)
+					}
+				}
+			}
+		}
+	}
+	return results, nil
+}
+
+func wantedHashedAddress(wantedStorage map[common.Hash][]common.Hash, hashedKey common.Hash) bool {
+	for addrHash := range wantedStorage {
+		if bytes.Equal(addrHash.Bytes(), hashedKey.Bytes()) {
+			return true
+		}
+	}
+	return false
+}
+
+func wantedHashedStorageKey(wantedKeys []common.Hash, keyBytes []byte) bool {
+	for _, key := range wantedKeys {
+		if bytes.Equal(key.Bytes(), keyBytes) {
+			return true
+		}
+	}
+	return false
+}

libraries/shared/storage/backfill_test.go

@@ -0,0 +1,174 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package storage_test
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/statediff"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/test_data"
+	"github.com/vulcanize/vulcanizedb/pkg/geth/client"
+)
+
+type MockClient struct {
+	MappedStateDiffAt map[uint64][]byte
+	MappedErrors      map[uint64]error
+}
+
+func (mc *MockClient) SetReturnDiffAt(height uint64, diffPayload statediff.Payload) error {
+	if mc.MappedStateDiffAt == nil {
+		mc.MappedStateDiffAt = make(map[uint64][]byte)
+	}
+	by, err := json.Marshal(diffPayload)
+	if err != nil {
+		return err
+	}
+	mc.MappedStateDiffAt[height] = by
+	return nil
+}
+
+func (mc *MockClient) SetReturnErrorAt(height uint64, err error) {
+	if mc.MappedErrors == nil {
+		mc.MappedErrors = make(map[uint64]error)
+	}
+	mc.MappedErrors[height] = err
+}
+
+func (mc *MockClient) BatchCall(batch []client.BatchElem) error {
+	if mc.MappedStateDiffAt == nil || mc.MappedErrors == nil {
+		return errors.New("mockclient needs to be initialized with statediff payloads and errors")
+	}
+	for _, batchElem := range batch {
+		if len(batchElem.Args) != 1 {
+			return errors.New("expected batch elem to contain single argument")
+		}
+		blockHeight, ok := batchElem.Args[0].(uint64)
+		if !ok {
+			return errors.New("expected batch elem argument to be a uint64")
+		}
+		err := json.Unmarshal(mc.MappedStateDiffAt[blockHeight], batchElem.Result)
+		if err != nil {
+			return err
+		}
+		batchElem.Error = mc.MappedErrors[blockHeight]
+	}
+	return nil
+}
+
+var _ = Describe("BackFiller", func() {
+	Describe("BackFill", func() {
+		It("Batch calls statediff_stateDiffAt", func() {
+			mc := new(MockClient)
+			setDiffAtErr1 := mc.SetReturnDiffAt(test_data.BlockNumber.Uint64(), test_data.MockStatediffPayload)
+			Expect(setDiffAtErr1).ToNot(HaveOccurred())
+			setDiffAtErr2 := mc.SetReturnDiffAt(test_data.BlockNumber2.Uint64(), test_data.MockStatediffPayload2)
+			Expect(setDiffAtErr2).ToNot(HaveOccurred())
+			mc.SetReturnErrorAt(test_data.BlockNumber.Uint64(), nil)
+			mc.SetReturnErrorAt(test_data.BlockNumber2.Uint64(), nil)
+			backFiller := storage.NewStorageBackFiller(mc)
+			backFillArgs := storage.BackFillerArgs{
+				WantedStorage: map[common.Hash][]common.Hash{
+					test_data.ContractLeafKey:        {common.BytesToHash(test_data.StorageKey)},
+					test_data.AnotherContractLeafKey: {common.BytesToHash(test_data.StorageKey)},
+				},
+				StartingBlock: test_data.BlockNumber.Uint64(),
+				EndingBlock:   test_data.BlockNumber2.Uint64(),
+			}
+			backFillStorage, backFillErr := backFiller.BackFill(backFillArgs)
+			Expect(backFillErr).ToNot(HaveOccurred())
+			Expect(len(backFillStorage)).To(Equal(2))
+			Expect(len(backFillStorage[test_data.ContractLeafKey])).To(Equal(1))
+			Expect(len(backFillStorage[test_data.AnotherContractLeafKey])).To(Equal(3))
+			Expect(backFillStorage[test_data.ContractLeafKey][0]).To(Equal(test_data.CreatedExpectedStorageDiff))
+			// Can only rlp encode the slice of diffs as part of a struct
+			// Rlp encoding allows us to compare content of the slices when the order in the slice may vary
+			expectedDiffStruct := struct {
+				diffs []utils.StorageDiff
+			}{
+				[]utils.StorageDiff{
+					test_data.UpdatedExpectedStorageDiff,
+					test_data.UpdatedExpectedStorageDiff2,
+					test_data.DeletedExpectedStorageDiff,
+				},
+			}
+			expectedDiffBytes, rlpErr1 := rlp.EncodeToBytes(expectedDiffStruct)
+			Expect(rlpErr1).ToNot(HaveOccurred())
+			receivedDiffStruct := struct {
+				diffs []utils.StorageDiff
+			}{
+				backFillStorage[test_data.AnotherContractLeafKey],
+			}
+			receivedDiffBytes, rlpErr2 := rlp.EncodeToBytes(receivedDiffStruct)
+			Expect(rlpErr2).ToNot(HaveOccurred())
+			Expect(bytes.Equal(expectedDiffBytes, receivedDiffBytes)).To(BeTrue())
+		})
+
+		It("Only returns storage for provided addresses (address hashes)", func() {
+			mc := new(MockClient)
+			setDiffAtErr1 := mc.SetReturnDiffAt(test_data.BlockNumber.Uint64(), test_data.MockStatediffPayload)
+			Expect(setDiffAtErr1).ToNot(HaveOccurred())
+			setDiffAtErr2 := mc.SetReturnDiffAt(test_data.BlockNumber2.Uint64(), test_data.MockStatediffPayload2)
+			Expect(setDiffAtErr2).ToNot(HaveOccurred())
+			mc.SetReturnErrorAt(test_data.BlockNumber.Uint64(), nil)
+			mc.SetReturnErrorAt(test_data.BlockNumber2.Uint64(), nil)
+			backFiller := storage.NewStorageBackFiller(mc)
+			backFillArgs := storage.BackFillerArgs{
+				WantedStorage: map[common.Hash][]common.Hash{
+					test_data.ContractLeafKey: {common.BytesToHash(test_data.StorageKey)},
+				},
+				StartingBlock: test_data.BlockNumber.Uint64(),
+				EndingBlock:   test_data.BlockNumber2.Uint64(),
+			}
+			backFillStorage, backFillErr := backFiller.BackFill(backFillArgs)
+			Expect(backFillErr).ToNot(HaveOccurred())
+			Expect(len(backFillStorage)).To(Equal(1))
+			Expect(len(backFillStorage[test_data.ContractLeafKey])).To(Equal(1))
+			Expect(len(backFillStorage[test_data.AnotherContractLeafKey])).To(Equal(0))
+			Expect(backFillStorage[test_data.ContractLeafKey][0]).To(Equal(test_data.CreatedExpectedStorageDiff))
+		})
+
+		It("Only returns storage for provided storage keys", func() {
+			mc := new(MockClient)
+			setDiffAtErr1 := mc.SetReturnDiffAt(test_data.BlockNumber.Uint64(), test_data.MockStatediffPayload)
+			Expect(setDiffAtErr1).ToNot(HaveOccurred())
+			setDiffAtErr2 := mc.SetReturnDiffAt(test_data.BlockNumber2.Uint64(), test_data.MockStatediffPayload2)
+			Expect(setDiffAtErr2).ToNot(HaveOccurred())
+			mc.SetReturnErrorAt(test_data.BlockNumber.Uint64(), nil)
+			mc.SetReturnErrorAt(test_data.BlockNumber2.Uint64(), nil)
+			backFiller := storage.NewStorageBackFiller(mc)
+			backFillArgs := storage.BackFillerArgs{
+				WantedStorage: map[common.Hash][]common.Hash{
+					test_data.ContractLeafKey: nil,
+				},
+				StartingBlock: test_data.BlockNumber.Uint64(),
+				EndingBlock:   test_data.BlockNumber2.Uint64(),
+			}
+			backFillStorage, backFillErr := backFiller.BackFill(backFillArgs)
+			Expect(backFillErr).ToNot(HaveOccurred())
+			Expect(len(backFillStorage)).To(Equal(0))
+		})
+	})
+})

libraries/shared/storage/utils/diff.go

@@ -17,11 +17,12 @@
 package utils
 
 import (
+	"strconv"
+
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/statediff"
-	"strconv"
 )
 
 const ExpectedRowLength = 5
@@ -71,3 +72,8 @@ func FromGethStateDiff(account statediff.AccountDiff, stateDiff *statediff.State
 func HexToKeccak256Hash(hex string) common.Hash {
 	return crypto.Keccak256Hash(common.FromHex(hex))
 }
+
+func GetAccountsFromDiff(stateDiff statediff.StateDiff) []statediff.AccountDiff {
+	accounts := append(stateDiff.CreatedAccounts, stateDiff.UpdatedAccounts...)
+	return append(accounts, stateDiff.DeletedAccounts...)
+}

libraries/shared/test_data/statediff.go

@@ -15,20 +15,23 @@
 package test_data
 
 import (
-	"errors"
+	"math/big"
+	"math/rand"
+
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/state"
 	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/statediff"
-	"math/big"
-	"math/rand"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
 )
 
 var (
 	BlockNumber             = big.NewInt(rand.Int63())
+	BlockNumber2            = big.NewInt(0).Add(BlockNumber, big.NewInt(1))
 	BlockHash               = "0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"
+	BlockHash2              = "0xaa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f72"
 	CodeHash                = common.Hex2Bytes("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470")
 	NewNonceValue           = rand.Uint64()
 	NewBalanceValue         = rand.Int63()
@@ -43,15 +46,14 @@ var (
 		Path:  StoragePath,
 		Proof: [][]byte{},
 	}}
-	LargeStorageValue            = common.Hex2Bytes("00191b53778c567b14b50ba0000")
-	LargeStorageValueRlp, rlpErr = rlp.EncodeToBytes(LargeStorageValue)
-	storageWithLargeValue        = []statediff.StorageDiff{{
+	LargeStorageValue       = common.Hex2Bytes("00191b53778c567b14b50ba0000")
+	LargeStorageValueRlp, _ = rlp.EncodeToBytes(LargeStorageValue)
+	storageWithLargeValue   = []statediff.StorageDiff{{
 		Key:   StorageKey,
 		Value: LargeStorageValueRlp,
 		Path:  StoragePath,
 		Proof: [][]byte{},
 	}}
-	EmptyStorage        = make([]statediff.StorageDiff, 0)
 	StorageWithBadValue = statediff.StorageDiff{
 		Key:   StorageKey,
 		Value: []byte{0, 1, 2},
@@ -83,6 +85,11 @@ var (
 		Value:   valueBytes,
 		Storage: storageWithLargeValue,
 	}}
+	UpdatedAccountDiffs2 = []statediff.AccountDiff{{
+		Key:     AnotherContractLeafKey.Bytes(),
+		Value:   valueBytes,
+		Storage: storageWithSmallValue,
+	}}
 
 	DeletedAccountDiffs = []statediff.AccountDiff{{
 		Key:     AnotherContractLeafKey.Bytes(),
@@ -97,7 +104,15 @@ var (
 		DeletedAccounts: DeletedAccountDiffs,
 		UpdatedAccounts: UpdatedAccountDiffs,
 	}
-	MockStateDiffBytes, _ = rlp.EncodeToBytes(MockStateDiff)
+	MockStateDiff2 = statediff.StateDiff{
+		BlockNumber:     BlockNumber2,
+		BlockHash:       common.HexToHash(BlockHash2),
+		CreatedAccounts: nil,
+		DeletedAccounts: nil,
+		UpdatedAccounts: UpdatedAccountDiffs2,
+	}
+	MockStateDiffBytes, _  = rlp.EncodeToBytes(MockStateDiff)
+	MockStateDiff2Bytes, _ = rlp.EncodeToBytes(MockStateDiff2)
 
 	mockTransaction1 = types.NewTransaction(0, common.HexToAddress("0x0"), big.NewInt(1000), 50, big.NewInt(100), nil)
 	mockTransaction2 = types.NewTransaction(1, common.HexToAddress("0x1"), big.NewInt(2000), 100, big.NewInt(200), nil)
@@ -114,24 +129,53 @@ var (
 		TxHash:      common.HexToHash("0x0"),
 		ReceiptHash: common.HexToHash("0x0"),
 	}
-	MockBlock       = types.NewBlock(&MockHeader, MockTransactions, nil, MockReceipts)
-	MockBlockRlp, _ = rlp.EncodeToBytes(MockBlock)
+	MockHeader2 = types.Header{
+		Time:        0,
+		Number:      BlockNumber2,
+		Root:        common.HexToHash("0x1"),
+		TxHash:      common.HexToHash("0x1"),
+		ReceiptHash: common.HexToHash("0x1"),
+	}
+	MockBlock        = types.NewBlock(&MockHeader, MockTransactions, nil, MockReceipts)
+	MockBlock2       = types.NewBlock(&MockHeader2, MockTransactions, nil, MockReceipts)
+	MockBlockRlp, _  = rlp.EncodeToBytes(MockBlock)
+	MockBlockRlp2, _ = rlp.EncodeToBytes(MockBlock2)
 
 	MockStatediffPayload = statediff.Payload{
 		BlockRlp:     MockBlockRlp,
 		StateDiffRlp: MockStateDiffBytes,
-		Err:          nil,
+	}
+	MockStatediffPayload2 = statediff.Payload{
+		BlockRlp:     MockBlockRlp2,
+		StateDiffRlp: MockStateDiff2Bytes,
 	}
 
-	EmptyStatediffPayload = statediff.Payload{
-		BlockRlp:     []byte{},
-		StateDiffRlp: []byte{},
-		Err:          nil,
+	CreatedExpectedStorageDiff = utils.StorageDiff{
+		HashedAddress: common.BytesToHash(ContractLeafKey[:]),
+		BlockHash:     common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"),
+		BlockHeight:   int(BlockNumber.Int64()),
+		StorageKey:    common.BytesToHash(StorageKey),
+		StorageValue:  common.BytesToHash(SmallStorageValue),
 	}
-
-	ErrStatediffPayload = statediff.Payload{
-		BlockRlp:     []byte{},
-		StateDiffRlp: []byte{},
-		Err:          errors.New("mock error"),
+	UpdatedExpectedStorageDiff = utils.StorageDiff{
+		HashedAddress: common.BytesToHash(AnotherContractLeafKey[:]),
+		BlockHash:     common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"),
+		BlockHeight:   int(BlockNumber.Int64()),
+		StorageKey:    common.BytesToHash(StorageKey),
+		StorageValue:  common.BytesToHash(LargeStorageValue),
+	}
+	UpdatedExpectedStorageDiff2 = utils.StorageDiff{
+		HashedAddress: common.BytesToHash(AnotherContractLeafKey[:]),
+		BlockHash:     common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"),
+		BlockHeight:   int(BlockNumber2.Int64()),
+		StorageKey:    common.BytesToHash(StorageKey),
+		StorageValue:  common.BytesToHash(SmallStorageValue),
+	}
+	DeletedExpectedStorageDiff = utils.StorageDiff{
+		HashedAddress: common.BytesToHash(AnotherContractLeafKey[:]),
+		BlockHash:     common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"),
+		BlockHeight:   int(BlockNumber.Int64()),
+		StorageKey:    common.BytesToHash(StorageKey),
+		StorageValue:  common.BytesToHash(SmallStorageValue),
 	}
 )