storage backfiller

libraries/shared/storage/backfill.go

@@ -0,0 +1,135 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package storage
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/types"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/statediff"
+	"github.com/sirupsen/logrus"
+
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
+	"github.com/vulcanize/vulcanizedb/pkg/geth/client"
+)
+
+type IBackFiller interface {
+	BackFill(bfa BackFillerArgs) (map[common.Hash][]utils.StorageDiff, error)
+}
+
+type BatchClient interface {
+	BatchCall(batch []client.BatchElem) error
+}
+
+type BackFiller struct {
+	client BatchClient
+}
+
+type BackFillerArgs struct {
+	// mapping of hashed addresses to a list of the storage key hashes we want to collect at that address
+	WantedStorage map[common.Hash][]common.Hash
+	StartingBlock uint64
+	EndingBlock   uint64
+}
+
+const method = "statediff_stateDiffAt"
+
+func NewStorageBackFiller(bc BatchClient) IBackFiller {
+	return &BackFiller{
+		client: bc,
+	}
+}
+
+// BackFill uses the provided config to fetch and return the state diff at the specified blocknumber
+// StateDiffAt(ctx context.Context, blockNumber uint64) (*Payload, error)
+func (bf *BackFiller) BackFill(bfa BackFillerArgs) (map[common.Hash][]utils.StorageDiff, error) {
+	results := make(map[common.Hash][]utils.StorageDiff, len(bfa.WantedStorage))
+	if bfa.EndingBlock < bfa.StartingBlock {
+		return nil, errors.New("backfill: ending block number needs to be greater than starting block number")
+	}
+	batch := make([]client.BatchElem, 0)
+	for i := bfa.StartingBlock; i <= bfa.EndingBlock; i++ {
+		batch = append(batch, client.BatchElem{
+			Method: method,
+			Args:   []interface{}{i},
+			Result: new(statediff.Payload),
+		})
+	}
+	batchErr := bf.client.BatchCall(batch)
+	if batchErr != nil {
+		return nil, batchErr
+	}
+	for _, batchElem := range batch {
+		payload := batchElem.Result.(*statediff.Payload)
+		if batchElem.Error != nil {
+			return nil, batchElem.Error
+		}
+		block := new(types.Block)
+		blockDecodeErr := rlp.DecodeBytes(payload.BlockRlp, block)
+		if blockDecodeErr != nil {
+			return nil, blockDecodeErr
+		}
+		stateDiff := new(statediff.StateDiff)
+		stateDiffDecodeErr := rlp.DecodeBytes(payload.StateDiffRlp, stateDiff)
+		if stateDiffDecodeErr != nil {
+			return nil, stateDiffDecodeErr
+		}
+		accounts := utils.GetAccountsFromDiff(*stateDiff)
+		for _, account := range accounts {
+			if wantedHashedAddress(bfa.WantedStorage, common.BytesToHash(account.Key)) {
+				logrus.Trace(fmt.Sprintf("iterating through %d Storage values on account", len(account.Storage)))
+				for _, storage := range account.Storage {
+					if wantedHashedStorageKey(bfa.WantedStorage[common.BytesToHash(account.Key)], storage.Key) {
+						diff, formatErr := utils.FromGethStateDiff(account, stateDiff, storage)
+						logrus.Trace("adding storage diff to out channel",
+							"keccak of address: ", diff.HashedAddress.Hex(),
+							"block height: ", diff.BlockHeight,
+							"storage key: ", diff.StorageKey.Hex(),
+							"storage value: ", diff.StorageValue.Hex())
+						if formatErr != nil {
+							return nil, formatErr
+						}
+						results[diff.HashedAddress] = append(results[diff.HashedAddress], diff)
+					}
+				}
+			}
+		}
+	}
+	return results, nil
+}
+
+func wantedHashedAddress(wantedStorage map[common.Hash][]common.Hash, hashedKey common.Hash) bool {
+	for addrHash := range wantedStorage {
+		if bytes.Equal(addrHash.Bytes(), hashedKey.Bytes()) {
+			return true
+		}
+	}
+	return false
+}
+
+func wantedHashedStorageKey(wantedKeys []common.Hash, keyBytes []byte) bool {
+	for _, key := range wantedKeys {
+		if bytes.Equal(key.Bytes(), keyBytes) {
+			return true
+		}
+	}
+	return false
+}

libraries/shared/storage/backfill_test.go

@@ -0,0 +1,174 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package storage_test
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/rlp"
+	"github.com/ethereum/go-ethereum/statediff"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
+
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/test_data"
+	"github.com/vulcanize/vulcanizedb/pkg/geth/client"
+)
+
+type MockClient struct {
+	MappedStateDiffAt map[uint64][]byte
+	MappedErrors      map[uint64]error
+}
+
+func (mc *MockClient) SetReturnDiffAt(height uint64, diffPayload statediff.Payload) error {
+	if mc.MappedStateDiffAt == nil {
+		mc.MappedStateDiffAt = make(map[uint64][]byte)
+	}
+	by, err := json.Marshal(diffPayload)
+	if err != nil {
+		return err
+	}
+	mc.MappedStateDiffAt[height] = by
+	return nil
+}
+
+func (mc *MockClient) SetReturnErrorAt(height uint64, err error) {
+	if mc.MappedErrors == nil {
+		mc.MappedErrors = make(map[uint64]error)
+	}
+	mc.MappedErrors[height] = err
+}
+
+func (mc *MockClient) BatchCall(batch []client.BatchElem) error {
+	if mc.MappedStateDiffAt == nil || mc.MappedErrors == nil {
+		return errors.New("mockclient needs to be initialized with statediff payloads and errors")
+	}
+	for _, batchElem := range batch {
+		if len(batchElem.Args) != 1 {
+			return errors.New("expected batch elem to contain single argument")
+		}
+		blockHeight, ok := batchElem.Args[0].(uint64)
+		if !ok {
+			return errors.New("expected batch elem argument to be a uint64")
+		}
+		err := json.Unmarshal(mc.MappedStateDiffAt[blockHeight], batchElem.Result)
+		if err != nil {
+			return err
+		}
+		batchElem.Error = mc.MappedErrors[blockHeight]
+	}
+	return nil
+}
+
+var _ = Describe("BackFiller", func() {
+	Describe("BackFill", func() {
+		It("Batch calls statediff_stateDiffAt", func() {
+			mc := new(MockClient)
+			setDiffAtErr1 := mc.SetReturnDiffAt(test_data.BlockNumber.Uint64(), test_data.MockStatediffPayload)
+			Expect(setDiffAtErr1).ToNot(HaveOccurred())
+			setDiffAtErr2 := mc.SetReturnDiffAt(test_data.BlockNumber2.Uint64(), test_data.MockStatediffPayload2)
+			Expect(setDiffAtErr2).ToNot(HaveOccurred())
+			mc.SetReturnErrorAt(test_data.BlockNumber.Uint64(), nil)
+			mc.SetReturnErrorAt(test_data.BlockNumber2.Uint64(), nil)
+			backFiller := storage.NewStorageBackFiller(mc)
+			backFillArgs := storage.BackFillerArgs{
+				WantedStorage: map[common.Hash][]common.Hash{
+					test_data.ContractLeafKey:        {common.BytesToHash(test_data.StorageKey)},
+					test_data.AnotherContractLeafKey: {common.BytesToHash(test_data.StorageKey)},
+				},
+				StartingBlock: test_data.BlockNumber.Uint64(),
+				EndingBlock:   test_data.BlockNumber2.Uint64(),
+			}
+			backFillStorage, backFillErr := backFiller.BackFill(backFillArgs)
+			Expect(backFillErr).ToNot(HaveOccurred())
+			Expect(len(backFillStorage)).To(Equal(2))
+			Expect(len(backFillStorage[test_data.ContractLeafKey])).To(Equal(1))
+			Expect(len(backFillStorage[test_data.AnotherContractLeafKey])).To(Equal(3))
+			Expect(backFillStorage[test_data.ContractLeafKey][0]).To(Equal(test_data.CreatedExpectedStorageDiff))
+			// Can only rlp encode the slice of diffs as part of a struct
+			// Rlp encoding allows us to compare content of the slices when the order in the slice may vary
+			expectedDiffStruct := struct {
+				diffs []utils.StorageDiff
+			}{
+				[]utils.StorageDiff{
+					test_data.UpdatedExpectedStorageDiff,
+					test_data.UpdatedExpectedStorageDiff2,
+					test_data.DeletedExpectedStorageDiff,
+				},
+			}
+			expectedDiffBytes, rlpErr1 := rlp.EncodeToBytes(expectedDiffStruct)
+			Expect(rlpErr1).ToNot(HaveOccurred())
+			receivedDiffStruct := struct {
+				diffs []utils.StorageDiff
+			}{
+				backFillStorage[test_data.AnotherContractLeafKey],
+			}
+			receivedDiffBytes, rlpErr2 := rlp.EncodeToBytes(receivedDiffStruct)
+			Expect(rlpErr2).ToNot(HaveOccurred())
+			Expect(bytes.Equal(expectedDiffBytes, receivedDiffBytes)).To(BeTrue())
+		})
+
+		It("Only returns storage for provided addresses (address hashes)", func() {
+			mc := new(MockClient)
+			setDiffAtErr1 := mc.SetReturnDiffAt(test_data.BlockNumber.Uint64(), test_data.MockStatediffPayload)
+			Expect(setDiffAtErr1).ToNot(HaveOccurred())
+			setDiffAtErr2 := mc.SetReturnDiffAt(test_data.BlockNumber2.Uint64(), test_data.MockStatediffPayload2)
+			Expect(setDiffAtErr2).ToNot(HaveOccurred())
+			mc.SetReturnErrorAt(test_data.BlockNumber.Uint64(), nil)
+			mc.SetReturnErrorAt(test_data.BlockNumber2.Uint64(), nil)
+			backFiller := storage.NewStorageBackFiller(mc)
+			backFillArgs := storage.BackFillerArgs{
+				WantedStorage: map[common.Hash][]common.Hash{
+					test_data.ContractLeafKey: {common.BytesToHash(test_data.StorageKey)},
+				},
+				StartingBlock: test_data.BlockNumber.Uint64(),
+				EndingBlock:   test_data.BlockNumber2.Uint64(),
+			}
+			backFillStorage, backFillErr := backFiller.BackFill(backFillArgs)
+			Expect(backFillErr).ToNot(HaveOccurred())
+			Expect(len(backFillStorage)).To(Equal(1))
+			Expect(len(backFillStorage[test_data.ContractLeafKey])).To(Equal(1))
+			Expect(len(backFillStorage[test_data.AnotherContractLeafKey])).To(Equal(0))
+			Expect(backFillStorage[test_data.ContractLeafKey][0]).To(Equal(test_data.CreatedExpectedStorageDiff))
+		})
+
+		It("Only returns storage for provided storage keys", func() {
+			mc := new(MockClient)
+			setDiffAtErr1 := mc.SetReturnDiffAt(test_data.BlockNumber.Uint64(), test_data.MockStatediffPayload)
+			Expect(setDiffAtErr1).ToNot(HaveOccurred())
+			setDiffAtErr2 := mc.SetReturnDiffAt(test_data.BlockNumber2.Uint64(), test_data.MockStatediffPayload2)
+			Expect(setDiffAtErr2).ToNot(HaveOccurred())
+			mc.SetReturnErrorAt(test_data.BlockNumber.Uint64(), nil)
+			mc.SetReturnErrorAt(test_data.BlockNumber2.Uint64(), nil)
+			backFiller := storage.NewStorageBackFiller(mc)
+			backFillArgs := storage.BackFillerArgs{
+				WantedStorage: map[common.Hash][]common.Hash{
+					test_data.ContractLeafKey: nil,
+				},
+				StartingBlock: test_data.BlockNumber.Uint64(),
+				EndingBlock:   test_data.BlockNumber2.Uint64(),
+			}
+			backFillStorage, backFillErr := backFiller.BackFill(backFillArgs)
+			Expect(backFillErr).ToNot(HaveOccurred())
+			Expect(len(backFillStorage)).To(Equal(0))
+		})
+	})
+})

libraries/shared/storage/utils/diff.go

@@ -17,11 +17,12 @@
 package utils
 
 import (
+	"strconv"
+
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/statediff"
-	"strconv"
 )
 
 const ExpectedRowLength = 5
@@ -71,3 +72,8 @@ func FromGethStateDiff(account statediff.AccountDiff, stateDiff *statediff.State
 func HexToKeccak256Hash(hex string) common.Hash {
 	return crypto.Keccak256Hash(common.FromHex(hex))
 }
+
+func GetAccountsFromDiff(stateDiff statediff.StateDiff) []statediff.AccountDiff {
+	accounts := append(stateDiff.CreatedAccounts, stateDiff.UpdatedAccounts...)
+	return append(accounts, stateDiff.DeletedAccounts...)
+}