diff --git a/libraries/shared/fetcher/geth_rpc_storage_fetcher.go b/libraries/shared/fetcher/geth_rpc_storage_fetcher.go index 6168b81c..27d505a0 100644 --- a/libraries/shared/fetcher/geth_rpc_storage_fetcher.go +++ b/libraries/shared/fetcher/geth_rpc_storage_fetcher.go @@ -16,9 +16,11 @@ package fetcher import ( "fmt" + "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/statediff" "github.com/sirupsen/logrus" + "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils" "github.com/vulcanize/vulcanizedb/libraries/shared/streamer" ) @@ -54,7 +56,7 @@ func (fetcher GethRPCStorageFetcher) FetchStorageDiffs(out chan<- utils.StorageD errs <- decodeErr } - accounts := getAccountsFromDiff(*stateDiff) + accounts := utils.GetAccountsFromDiff(*stateDiff) logrus.Trace(fmt.Sprintf("iterating through %d accounts on stateDiff for block %d", len(accounts), stateDiff.BlockNumber)) for _, account := range accounts { logrus.Trace(fmt.Sprintf("iterating through %d Storage values on account", len(account.Storage))) @@ -74,8 +76,3 @@ func (fetcher GethRPCStorageFetcher) FetchStorageDiffs(out chan<- utils.StorageD } } } - -func getAccountsFromDiff(stateDiff statediff.StateDiff) []statediff.AccountDiff { - accounts := append(stateDiff.CreatedAccounts, stateDiff.UpdatedAccounts...) - return append(accounts, stateDiff.DeletedAccounts...) -} diff --git a/libraries/shared/storage/backfill.go b/libraries/shared/storage/backfill.go new file mode 100644 index 00000000..bde07288 --- /dev/null +++ b/libraries/shared/storage/backfill.go @@ -0,0 +1,135 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package storage + +import ( + "bytes" + "errors" + "fmt" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/statediff" + "github.com/sirupsen/logrus" + + "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils" + "github.com/vulcanize/vulcanizedb/pkg/geth/client" +) + +type IBackFiller interface { + BackFill(bfa BackFillerArgs) (map[common.Hash][]utils.StorageDiff, error) +} + +type BatchClient interface { + BatchCall(batch []client.BatchElem) error +} + +type BackFiller struct { + client BatchClient +} + +type BackFillerArgs struct { + // mapping of hashed addresses to a list of the storage key hashes we want to collect at that address + WantedStorage map[common.Hash][]common.Hash + StartingBlock uint64 + EndingBlock uint64 +} + +const method = "statediff_stateDiffAt" + +func NewStorageBackFiller(bc BatchClient) IBackFiller { + return &BackFiller{ + client: bc, + } +} + +// BackFill uses the provided config to fetch and return the state diff at the specified blocknumber +// StateDiffAt(ctx context.Context, blockNumber uint64) (*Payload, error) +func (bf *BackFiller) BackFill(bfa BackFillerArgs) (map[common.Hash][]utils.StorageDiff, error) { + results := make(map[common.Hash][]utils.StorageDiff, len(bfa.WantedStorage)) + if bfa.EndingBlock < bfa.StartingBlock { + return nil, errors.New("backfill: ending block number needs to be greater than starting block number") + } + batch := make([]client.BatchElem, 0) + for i := bfa.StartingBlock; i <= bfa.EndingBlock; i++ { + batch = append(batch, client.BatchElem{ + Method: method, + Args: []interface{}{i}, + Result: new(statediff.Payload), + }) + } + batchErr := bf.client.BatchCall(batch) + if batchErr != nil { + return nil, batchErr + } + for _, batchElem := range batch { + payload := batchElem.Result.(*statediff.Payload) + if batchElem.Error != nil { + return nil, batchElem.Error + } + block := new(types.Block) + blockDecodeErr := rlp.DecodeBytes(payload.BlockRlp, block) + if blockDecodeErr != nil { + return nil, blockDecodeErr + } + stateDiff := new(statediff.StateDiff) + stateDiffDecodeErr := rlp.DecodeBytes(payload.StateDiffRlp, stateDiff) + if stateDiffDecodeErr != nil { + return nil, stateDiffDecodeErr + } + accounts := utils.GetAccountsFromDiff(*stateDiff) + for _, account := range accounts { + if wantedHashedAddress(bfa.WantedStorage, common.BytesToHash(account.Key)) { + logrus.Trace(fmt.Sprintf("iterating through %d Storage values on account", len(account.Storage))) + for _, storage := range account.Storage { + if wantedHashedStorageKey(bfa.WantedStorage[common.BytesToHash(account.Key)], storage.Key) { + diff, formatErr := utils.FromGethStateDiff(account, stateDiff, storage) + logrus.Trace("adding storage diff to out channel", + "keccak of address: ", diff.HashedAddress.Hex(), + "block height: ", diff.BlockHeight, + "storage key: ", diff.StorageKey.Hex(), + "storage value: ", diff.StorageValue.Hex()) + if formatErr != nil { + return nil, formatErr + } + results[diff.HashedAddress] = append(results[diff.HashedAddress], diff) + } + } + } + } + } + return results, nil +} + +func wantedHashedAddress(wantedStorage map[common.Hash][]common.Hash, hashedKey common.Hash) bool { + for addrHash := range wantedStorage { + if bytes.Equal(addrHash.Bytes(), hashedKey.Bytes()) { + return true + } + } + return false +} + +func wantedHashedStorageKey(wantedKeys []common.Hash, keyBytes []byte) bool { + for _, key := range wantedKeys { + if bytes.Equal(key.Bytes(), keyBytes) { + return true + } + } + return false +} diff --git a/libraries/shared/storage/backfill_test.go b/libraries/shared/storage/backfill_test.go new file mode 100644 index 00000000..7f243ff4 --- /dev/null +++ b/libraries/shared/storage/backfill_test.go @@ -0,0 +1,174 @@ +// VulcanizeDB +// Copyright © 2019 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package storage_test + +import ( + "bytes" + "encoding/json" + "errors" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/statediff" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + + "github.com/vulcanize/vulcanizedb/libraries/shared/storage" + "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils" + "github.com/vulcanize/vulcanizedb/libraries/shared/test_data" + "github.com/vulcanize/vulcanizedb/pkg/geth/client" +) + +type MockClient struct { + MappedStateDiffAt map[uint64][]byte + MappedErrors map[uint64]error +} + +func (mc *MockClient) SetReturnDiffAt(height uint64, diffPayload statediff.Payload) error { + if mc.MappedStateDiffAt == nil { + mc.MappedStateDiffAt = make(map[uint64][]byte) + } + by, err := json.Marshal(diffPayload) + if err != nil { + return err + } + mc.MappedStateDiffAt[height] = by + return nil +} + +func (mc *MockClient) SetReturnErrorAt(height uint64, err error) { + if mc.MappedErrors == nil { + mc.MappedErrors = make(map[uint64]error) + } + mc.MappedErrors[height] = err +} + +func (mc *MockClient) BatchCall(batch []client.BatchElem) error { + if mc.MappedStateDiffAt == nil || mc.MappedErrors == nil { + return errors.New("mockclient needs to be initialized with statediff payloads and errors") + } + for _, batchElem := range batch { + if len(batchElem.Args) != 1 { + return errors.New("expected batch elem to contain single argument") + } + blockHeight, ok := batchElem.Args[0].(uint64) + if !ok { + return errors.New("expected batch elem argument to be a uint64") + } + err := json.Unmarshal(mc.MappedStateDiffAt[blockHeight], batchElem.Result) + if err != nil { + return err + } + batchElem.Error = mc.MappedErrors[blockHeight] + } + return nil +} + +var _ = Describe("BackFiller", func() { + Describe("BackFill", func() { + It("Batch calls statediff_stateDiffAt", func() { + mc := new(MockClient) + setDiffAtErr1 := mc.SetReturnDiffAt(test_data.BlockNumber.Uint64(), test_data.MockStatediffPayload) + Expect(setDiffAtErr1).ToNot(HaveOccurred()) + setDiffAtErr2 := mc.SetReturnDiffAt(test_data.BlockNumber2.Uint64(), test_data.MockStatediffPayload2) + Expect(setDiffAtErr2).ToNot(HaveOccurred()) + mc.SetReturnErrorAt(test_data.BlockNumber.Uint64(), nil) + mc.SetReturnErrorAt(test_data.BlockNumber2.Uint64(), nil) + backFiller := storage.NewStorageBackFiller(mc) + backFillArgs := storage.BackFillerArgs{ + WantedStorage: map[common.Hash][]common.Hash{ + test_data.ContractLeafKey: {common.BytesToHash(test_data.StorageKey)}, + test_data.AnotherContractLeafKey: {common.BytesToHash(test_data.StorageKey)}, + }, + StartingBlock: test_data.BlockNumber.Uint64(), + EndingBlock: test_data.BlockNumber2.Uint64(), + } + backFillStorage, backFillErr := backFiller.BackFill(backFillArgs) + Expect(backFillErr).ToNot(HaveOccurred()) + Expect(len(backFillStorage)).To(Equal(2)) + Expect(len(backFillStorage[test_data.ContractLeafKey])).To(Equal(1)) + Expect(len(backFillStorage[test_data.AnotherContractLeafKey])).To(Equal(3)) + Expect(backFillStorage[test_data.ContractLeafKey][0]).To(Equal(test_data.CreatedExpectedStorageDiff)) + // Can only rlp encode the slice of diffs as part of a struct + // Rlp encoding allows us to compare content of the slices when the order in the slice may vary + expectedDiffStruct := struct { + diffs []utils.StorageDiff + }{ + []utils.StorageDiff{ + test_data.UpdatedExpectedStorageDiff, + test_data.UpdatedExpectedStorageDiff2, + test_data.DeletedExpectedStorageDiff, + }, + } + expectedDiffBytes, rlpErr1 := rlp.EncodeToBytes(expectedDiffStruct) + Expect(rlpErr1).ToNot(HaveOccurred()) + receivedDiffStruct := struct { + diffs []utils.StorageDiff + }{ + backFillStorage[test_data.AnotherContractLeafKey], + } + receivedDiffBytes, rlpErr2 := rlp.EncodeToBytes(receivedDiffStruct) + Expect(rlpErr2).ToNot(HaveOccurred()) + Expect(bytes.Equal(expectedDiffBytes, receivedDiffBytes)).To(BeTrue()) + }) + + It("Only returns storage for provided addresses (address hashes)", func() { + mc := new(MockClient) + setDiffAtErr1 := mc.SetReturnDiffAt(test_data.BlockNumber.Uint64(), test_data.MockStatediffPayload) + Expect(setDiffAtErr1).ToNot(HaveOccurred()) + setDiffAtErr2 := mc.SetReturnDiffAt(test_data.BlockNumber2.Uint64(), test_data.MockStatediffPayload2) + Expect(setDiffAtErr2).ToNot(HaveOccurred()) + mc.SetReturnErrorAt(test_data.BlockNumber.Uint64(), nil) + mc.SetReturnErrorAt(test_data.BlockNumber2.Uint64(), nil) + backFiller := storage.NewStorageBackFiller(mc) + backFillArgs := storage.BackFillerArgs{ + WantedStorage: map[common.Hash][]common.Hash{ + test_data.ContractLeafKey: {common.BytesToHash(test_data.StorageKey)}, + }, + StartingBlock: test_data.BlockNumber.Uint64(), + EndingBlock: test_data.BlockNumber2.Uint64(), + } + backFillStorage, backFillErr := backFiller.BackFill(backFillArgs) + Expect(backFillErr).ToNot(HaveOccurred()) + Expect(len(backFillStorage)).To(Equal(1)) + Expect(len(backFillStorage[test_data.ContractLeafKey])).To(Equal(1)) + Expect(len(backFillStorage[test_data.AnotherContractLeafKey])).To(Equal(0)) + Expect(backFillStorage[test_data.ContractLeafKey][0]).To(Equal(test_data.CreatedExpectedStorageDiff)) + }) + + It("Only returns storage for provided storage keys", func() { + mc := new(MockClient) + setDiffAtErr1 := mc.SetReturnDiffAt(test_data.BlockNumber.Uint64(), test_data.MockStatediffPayload) + Expect(setDiffAtErr1).ToNot(HaveOccurred()) + setDiffAtErr2 := mc.SetReturnDiffAt(test_data.BlockNumber2.Uint64(), test_data.MockStatediffPayload2) + Expect(setDiffAtErr2).ToNot(HaveOccurred()) + mc.SetReturnErrorAt(test_data.BlockNumber.Uint64(), nil) + mc.SetReturnErrorAt(test_data.BlockNumber2.Uint64(), nil) + backFiller := storage.NewStorageBackFiller(mc) + backFillArgs := storage.BackFillerArgs{ + WantedStorage: map[common.Hash][]common.Hash{ + test_data.ContractLeafKey: nil, + }, + StartingBlock: test_data.BlockNumber.Uint64(), + EndingBlock: test_data.BlockNumber2.Uint64(), + } + backFillStorage, backFillErr := backFiller.BackFill(backFillArgs) + Expect(backFillErr).ToNot(HaveOccurred()) + Expect(len(backFillStorage)).To(Equal(0)) + }) + }) +}) diff --git a/libraries/shared/storage/utils/diff.go b/libraries/shared/storage/utils/diff.go index 991c9828..45e8ffa2 100644 --- a/libraries/shared/storage/utils/diff.go +++ b/libraries/shared/storage/utils/diff.go @@ -17,11 +17,12 @@ package utils import ( + "strconv" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/statediff" - "strconv" ) const ExpectedRowLength = 5 @@ -71,3 +72,8 @@ func FromGethStateDiff(account statediff.AccountDiff, stateDiff *statediff.State func HexToKeccak256Hash(hex string) common.Hash { return crypto.Keccak256Hash(common.FromHex(hex)) } + +func GetAccountsFromDiff(stateDiff statediff.StateDiff) []statediff.AccountDiff { + accounts := append(stateDiff.CreatedAccounts, stateDiff.UpdatedAccounts...) + return append(accounts, stateDiff.DeletedAccounts...) +} diff --git a/libraries/shared/test_data/statediff.go b/libraries/shared/test_data/statediff.go index 3e0a2219..d749b773 100644 --- a/libraries/shared/test_data/statediff.go +++ b/libraries/shared/test_data/statediff.go @@ -15,20 +15,23 @@ package test_data import ( - "errors" + "math/big" + "math/rand" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/state" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/statediff" - "math/big" - "math/rand" + "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils" ) var ( BlockNumber = big.NewInt(rand.Int63()) + BlockNumber2 = big.NewInt(0).Add(BlockNumber, big.NewInt(1)) BlockHash = "0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73" + BlockHash2 = "0xaa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f72" CodeHash = common.Hex2Bytes("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470") NewNonceValue = rand.Uint64() NewBalanceValue = rand.Int63() @@ -43,15 +46,14 @@ var ( Path: StoragePath, Proof: [][]byte{}, }} - LargeStorageValue = common.Hex2Bytes("00191b53778c567b14b50ba0000") - LargeStorageValueRlp, rlpErr = rlp.EncodeToBytes(LargeStorageValue) - storageWithLargeValue = []statediff.StorageDiff{{ + LargeStorageValue = common.Hex2Bytes("00191b53778c567b14b50ba0000") + LargeStorageValueRlp, _ = rlp.EncodeToBytes(LargeStorageValue) + storageWithLargeValue = []statediff.StorageDiff{{ Key: StorageKey, Value: LargeStorageValueRlp, Path: StoragePath, Proof: [][]byte{}, }} - EmptyStorage = make([]statediff.StorageDiff, 0) StorageWithBadValue = statediff.StorageDiff{ Key: StorageKey, Value: []byte{0, 1, 2}, @@ -83,6 +85,11 @@ var ( Value: valueBytes, Storage: storageWithLargeValue, }} + UpdatedAccountDiffs2 = []statediff.AccountDiff{{ + Key: AnotherContractLeafKey.Bytes(), + Value: valueBytes, + Storage: storageWithSmallValue, + }} DeletedAccountDiffs = []statediff.AccountDiff{{ Key: AnotherContractLeafKey.Bytes(), @@ -97,7 +104,15 @@ var ( DeletedAccounts: DeletedAccountDiffs, UpdatedAccounts: UpdatedAccountDiffs, } - MockStateDiffBytes, _ = rlp.EncodeToBytes(MockStateDiff) + MockStateDiff2 = statediff.StateDiff{ + BlockNumber: BlockNumber2, + BlockHash: common.HexToHash(BlockHash2), + CreatedAccounts: nil, + DeletedAccounts: nil, + UpdatedAccounts: UpdatedAccountDiffs2, + } + MockStateDiffBytes, _ = rlp.EncodeToBytes(MockStateDiff) + MockStateDiff2Bytes, _ = rlp.EncodeToBytes(MockStateDiff2) mockTransaction1 = types.NewTransaction(0, common.HexToAddress("0x0"), big.NewInt(1000), 50, big.NewInt(100), nil) mockTransaction2 = types.NewTransaction(1, common.HexToAddress("0x1"), big.NewInt(2000), 100, big.NewInt(200), nil) @@ -114,24 +129,53 @@ var ( TxHash: common.HexToHash("0x0"), ReceiptHash: common.HexToHash("0x0"), } - MockBlock = types.NewBlock(&MockHeader, MockTransactions, nil, MockReceipts) - MockBlockRlp, _ = rlp.EncodeToBytes(MockBlock) + MockHeader2 = types.Header{ + Time: 0, + Number: BlockNumber2, + Root: common.HexToHash("0x1"), + TxHash: common.HexToHash("0x1"), + ReceiptHash: common.HexToHash("0x1"), + } + MockBlock = types.NewBlock(&MockHeader, MockTransactions, nil, MockReceipts) + MockBlock2 = types.NewBlock(&MockHeader2, MockTransactions, nil, MockReceipts) + MockBlockRlp, _ = rlp.EncodeToBytes(MockBlock) + MockBlockRlp2, _ = rlp.EncodeToBytes(MockBlock2) MockStatediffPayload = statediff.Payload{ BlockRlp: MockBlockRlp, StateDiffRlp: MockStateDiffBytes, - Err: nil, + } + MockStatediffPayload2 = statediff.Payload{ + BlockRlp: MockBlockRlp2, + StateDiffRlp: MockStateDiff2Bytes, } - EmptyStatediffPayload = statediff.Payload{ - BlockRlp: []byte{}, - StateDiffRlp: []byte{}, - Err: nil, + CreatedExpectedStorageDiff = utils.StorageDiff{ + HashedAddress: common.BytesToHash(ContractLeafKey[:]), + BlockHash: common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"), + BlockHeight: int(BlockNumber.Int64()), + StorageKey: common.BytesToHash(StorageKey), + StorageValue: common.BytesToHash(SmallStorageValue), } - - ErrStatediffPayload = statediff.Payload{ - BlockRlp: []byte{}, - StateDiffRlp: []byte{}, - Err: errors.New("mock error"), + UpdatedExpectedStorageDiff = utils.StorageDiff{ + HashedAddress: common.BytesToHash(AnotherContractLeafKey[:]), + BlockHash: common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"), + BlockHeight: int(BlockNumber.Int64()), + StorageKey: common.BytesToHash(StorageKey), + StorageValue: common.BytesToHash(LargeStorageValue), + } + UpdatedExpectedStorageDiff2 = utils.StorageDiff{ + HashedAddress: common.BytesToHash(AnotherContractLeafKey[:]), + BlockHash: common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"), + BlockHeight: int(BlockNumber2.Int64()), + StorageKey: common.BytesToHash(StorageKey), + StorageValue: common.BytesToHash(SmallStorageValue), + } + DeletedExpectedStorageDiff = utils.StorageDiff{ + HashedAddress: common.BytesToHash(AnotherContractLeafKey[:]), + BlockHash: common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"), + BlockHeight: int(BlockNumber.Int64()), + StorageKey: common.BytesToHash(StorageKey), + StorageValue: common.BytesToHash(SmallStorageValue), } )