From 6869330bd359bffbee407b9f3694869ddf258141 Mon Sep 17 00:00:00 2001 From: Elizabeth Engelman Date: Fri, 30 Aug 2019 14:17:26 -0500 Subject: [PATCH] Decode storage value RLP after fetching from statediffs --- .../fetcher/geth_rpc_storage_fetcher.go | 7 +- .../fetcher/geth_rpc_storage_fetcher_test.go | 104 +++++++++++++----- libraries/shared/storage/utils/diff.go | 13 ++- libraries/shared/storage/utils/diff_test.go | 48 +++++++- libraries/shared/test_data/statediff.go | 42 +++---- 5 files changed, 157 insertions(+), 57 deletions(-) diff --git a/libraries/shared/fetcher/geth_rpc_storage_fetcher.go b/libraries/shared/fetcher/geth_rpc_storage_fetcher.go index 7713a093..a6de437a 100644 --- a/libraries/shared/fetcher/geth_rpc_storage_fetcher.go +++ b/libraries/shared/fetcher/geth_rpc_storage_fetcher.go @@ -60,7 +60,12 @@ func (fetcher GethRpcStorageFetcher) FetchStorageDiffs(out chan<- utils.StorageD logrus.Trace(fmt.Sprintf("iterating through %d Storage values on account", len(account.Storage))) for _, storage := range account.Storage { logrus.Trace("adding storage diff to out channel") - out <- utils.FromGethStateDiff(account, stateDiff, storage) + diff, formatErr := utils.FromGethStateDiff(account, stateDiff, storage) + if formatErr != nil { + errs <- formatErr + } + + out <- diff } } } diff --git a/libraries/shared/fetcher/geth_rpc_storage_fetcher_test.go b/libraries/shared/fetcher/geth_rpc_storage_fetcher_test.go index 0e8be2f8..c4b35b12 100644 --- a/libraries/shared/fetcher/geth_rpc_storage_fetcher_test.go +++ b/libraries/shared/fetcher/geth_rpc_storage_fetcher_test.go @@ -16,6 +16,7 @@ package fetcher_test import ( "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/rpc" "github.com/ethereum/go-ethereum/statediff" . "github.com/onsi/ginkgo" @@ -93,34 +94,7 @@ var _ = Describe("Geth RPC Storage Fetcher", func() { close(done) }) - It("adds parsed statediff payloads to the rows channel", func(done Done) { - streamer.SetPayloads([]statediff.Payload{test_data.MockStatediffPayload}) - - go statediffFetcher.FetchStorageDiffs(storagediffChan, errorChan) - - height := test_data.BlockNumber - intHeight := int(height.Int64()) - expectedStorageDiff := utils.StorageDiff{ - KeccakOfContractAddress: common.BytesToHash(test_data.ContractLeafKey[:]), - BlockHash: common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"), - BlockHeight: intHeight, - StorageKey: common.BytesToHash(test_data.StorageKey), - StorageValue: common.BytesToHash(test_data.StorageValue), - } - anotherExpectedStorageDiff := utils.StorageDiff{ - KeccakOfContractAddress: common.BytesToHash(test_data.AnotherContractLeafKey[:]), - BlockHash: common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"), - BlockHeight: intHeight, - StorageKey: common.BytesToHash(test_data.StorageKey), - StorageValue: common.BytesToHash(test_data.StorageValue), - } - Expect(<-storagediffChan).To(Equal(expectedStorageDiff)) - Expect(<-storagediffChan).To(Equal(anotherExpectedStorageDiff)) - - close(done) - }) - - It("adds errors to error channel if parsing the diff fails", func(done Done) { + It("adds errors to error channel if decoding the state diff RLP fails", func(done Done) { badStatediffPayload := statediff.Payload{} streamer.SetPayloads([]statediff.Payload{badStatediffPayload}) @@ -130,4 +104,78 @@ var _ = Describe("Geth RPC Storage Fetcher", func() { close(done) }) + + It("adds parsed statediff payloads to the rows channel", func(done Done) { + streamer.SetPayloads([]statediff.Payload{test_data.MockStatediffPayload}) + + go statediffFetcher.FetchStorageDiffs(storagediffChan, errorChan) + + height := test_data.BlockNumber + intHeight := int(height.Int64()) + createdExpectedStorageDiff := utils.StorageDiff{ + KeccakOfContractAddress: common.BytesToHash(test_data.ContractLeafKey[:]), + BlockHash: common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"), + BlockHeight: intHeight, + StorageKey: common.BytesToHash(test_data.StorageKey), + StorageValue: common.BytesToHash(test_data.SmallStorageValue), + } + updatedExpectedStorageDiff := utils.StorageDiff{ + KeccakOfContractAddress: common.BytesToHash(test_data.AnotherContractLeafKey[:]), + BlockHash: common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"), + BlockHeight: intHeight, + StorageKey: common.BytesToHash(test_data.StorageKey), + StorageValue: common.BytesToHash(test_data.LargeStorageValue), + } + deletedExpectedStorageDiff := utils.StorageDiff{ + KeccakOfContractAddress: common.BytesToHash(test_data.AnotherContractLeafKey[:]), + BlockHash: common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"), + BlockHeight: intHeight, + StorageKey: common.BytesToHash(test_data.StorageKey), + StorageValue: common.BytesToHash(test_data.SmallStorageValue), + } + + createdStateDiff := <-storagediffChan + updatedStateDiff := <-storagediffChan + deletedStateDiff := <-storagediffChan + + Expect(createdStateDiff).To(Equal(createdExpectedStorageDiff)) + Expect(updatedStateDiff).To(Equal(updatedExpectedStorageDiff)) + Expect(deletedStateDiff).To(Equal(deletedExpectedStorageDiff)) + + close(done) + }) + + It("adds errors to error channel if formatting the diff as a StateDiff object fails", func(done Done) { + badStorageDiffs := []statediff.StorageDiff{{ + Key: test_data.StorageKey, + Value: []byte{1, 2, 3}, + // this storage value will fail to be decoded as an RLP with the following error message: + // "input contains more than one value" + Path: test_data.StoragePath, + Proof: [][]byte{}, + }} + + accountDiffs := test_data.CreatedAccountDiffs + accountDiffs[0].Storage = badStorageDiffs + + stateDiff := statediff.StateDiff{ + BlockNumber: test_data.BlockNumber, + BlockHash: common.HexToHash(test_data.BlockHash), + CreatedAccounts: accountDiffs, + } + + stateDiffRlp, err := rlp.EncodeToBytes(stateDiff) + Expect(err).NotTo(HaveOccurred()) + + badStatediffPayload := statediff.Payload{ + StateDiffRlp: stateDiffRlp, + } + streamer.SetPayloads([]statediff.Payload{badStatediffPayload}) + + go statediffFetcher.FetchStorageDiffs(storagediffChan, errorChan) + + Expect(<-errorChan).To(MatchError("rlp: input contains more than one value")) + + close(done) + }) }) diff --git a/libraries/shared/storage/utils/diff.go b/libraries/shared/storage/utils/diff.go index d7bdbc97..afa75215 100644 --- a/libraries/shared/storage/utils/diff.go +++ b/libraries/shared/storage/utils/diff.go @@ -19,6 +19,7 @@ package utils import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/statediff" "strconv" ) @@ -51,14 +52,20 @@ func FromParityCsvRow(csvRow []string) (StorageDiff, error) { }, nil } -func FromGethStateDiff(account statediff.AccountDiff, stateDiff *statediff.StateDiff, storage statediff.StorageDiff) StorageDiff { +func FromGethStateDiff(account statediff.AccountDiff, stateDiff *statediff.StateDiff, storage statediff.StorageDiff) (StorageDiff, error) { + var decodedValue []byte + err := rlp.DecodeBytes(storage.Value, &decodedValue) + if err != nil { + return StorageDiff{}, err + } + return StorageDiff{ KeccakOfContractAddress: common.BytesToHash(account.Key), BlockHash: stateDiff.BlockHash, BlockHeight: int(stateDiff.BlockNumber.Int64()), StorageKey: common.BytesToHash(storage.Key), - StorageValue: common.BytesToHash(storage.Value), - } + StorageValue: common.BytesToHash(decodedValue), + }, nil } func HexToKeccak256Hash(hex string) common.Hash { diff --git a/libraries/shared/storage/utils/diff_test.go b/libraries/shared/storage/utils/diff_test.go index 24df7451..13c5b4a6 100644 --- a/libraries/shared/storage/utils/diff_test.go +++ b/libraries/shared/storage/utils/diff_test.go @@ -18,10 +18,12 @@ package utils_test import ( "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/statediff" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils" + "github.com/vulcanize/vulcanizedb/libraries/shared/test_data" "github.com/vulcanize/vulcanizedb/pkg/fakes" "math/big" "math/rand" @@ -63,18 +65,26 @@ var _ = Describe("Storage row parsing", func() { }) Describe("FromGethStateDiff", func() { - It("adds relevant fields to diff", func() { - accountDiff := statediff.AccountDiff{Key: []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 0}} - stateDiff := &statediff.StateDiff{ + var ( + accountDiff = statediff.AccountDiff{Key: []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 0}} + stateDiff = &statediff.StateDiff{ BlockNumber: big.NewInt(rand.Int63()), BlockHash: fakes.FakeHash, } + ) + + It("adds relevant fields to diff", func() { + storageValueBytes := []byte{3} + storageValueRlp, encodeErr := rlp.EncodeToBytes(storageValueBytes) + Expect(encodeErr).NotTo(HaveOccurred()) + storageDiff := statediff.StorageDiff{ Key: []byte{0, 9, 8, 7, 6, 5, 4, 3, 2, 1}, - Value: []byte{1, 2, 3, 4, 5, 0, 9, 8, 7, 6}, + Value: storageValueRlp, } - result := utils.FromGethStateDiff(accountDiff, stateDiff, storageDiff) + result, err := utils.FromGethStateDiff(accountDiff, stateDiff, storageDiff) + Expect(err).NotTo(HaveOccurred()) expectedAddress := common.BytesToHash(accountDiff.Key) Expect(result.KeccakOfContractAddress).To(Equal(expectedAddress)) @@ -83,8 +93,34 @@ var _ = Describe("Storage row parsing", func() { Expect(result.BlockHeight).To(Equal(expectedBlockHeight)) expectedStorageKey := common.BytesToHash(storageDiff.Key) Expect(result.StorageKey).To(Equal(expectedStorageKey)) - expectedStorageValue := common.BytesToHash(storageDiff.Value) + expectedStorageValue := common.BytesToHash(storageValueBytes) Expect(result.StorageValue).To(Equal(expectedStorageValue)) }) + + It("handles decoding large storage values from their RLP", func() { + storageValueBytes := []byte{1, 2, 3, 4, 5, 0, 9, 8, 7, 6} + storageValueRlp, encodeErr := rlp.EncodeToBytes(storageValueBytes) + Expect(encodeErr).NotTo(HaveOccurred()) + + storageDiff := statediff.StorageDiff{ + Key: []byte{0, 9, 8, 7, 6, 5, 4, 3, 2, 1}, + Value: storageValueRlp, + } + + result, err := utils.FromGethStateDiff(accountDiff, stateDiff, storageDiff) + Expect(err).NotTo(HaveOccurred()) + Expect(result.StorageValue).To(Equal(common.BytesToHash(storageValueBytes))) + }) + + It("returns an err if decoding the storage value Rlp fails", func() { + storageDiff := statediff.StorageDiff{ + Key: []byte{0, 9, 8, 7, 6, 5, 4, 3, 2, 1}, + Value: test_data.StorageKey, + } + + _, err := utils.FromGethStateDiff(accountDiff, stateDiff, storageDiff) + Expect(err).To(HaveOccurred()) + Expect(err).To(MatchError("rlp: input contains more than one value")) + }) }) }) diff --git a/libraries/shared/test_data/statediff.go b/libraries/shared/test_data/statediff.go index d846cbb8..fc8e9e69 100644 --- a/libraries/shared/test_data/statediff.go +++ b/libraries/shared/test_data/statediff.go @@ -29,20 +29,29 @@ import ( var ( BlockNumber = big.NewInt(rand.Int63()) BlockHash = "0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73" - CodeHash = common.Hex2Bytes("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470") - NewNonceValue = rand.Uint64() - NewBalanceValue = rand.Int63() - ContractRoot = common.HexToHash("0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421") - StoragePath = common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes() - StorageKey = common.HexToHash("0000000000000000000000000000000000000000000000000000000000000001").Bytes() - StorageValue = common.Hex2Bytes("0x03") - storage = []statediff.StorageDiff{{ + CodeHash = common.Hex2Bytes("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470") + NewNonceValue = rand.Uint64() + NewBalanceValue = rand.Int63() + ContractRoot = common.HexToHash("0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421") + StoragePath = common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes() + StorageKey = common.HexToHash("0000000000000000000000000000000000000000000000000000000000000001").Bytes() + SmallStorageValue = common.Hex2Bytes("03") + SmallStorageValueRlp, _ = rlp.EncodeToBytes(SmallStorageValue) + storageWithSmallValue = []statediff.StorageDiff{{ Key: StorageKey, - Value: StorageValue, + Value: SmallStorageValueRlp, Path: StoragePath, Proof: [][]byte{}, }} - emptyStorage = make([]statediff.StorageDiff, 0) + LargeStorageValue = common.Hex2Bytes("00191b53778c567b14b50ba0000") + LargeStorageValueRlp, rlpErr = rlp.EncodeToBytes(LargeStorageValue) + storageWithLargeValue = []statediff.StorageDiff{{ + Key: StorageKey, + Value: LargeStorageValueRlp, + Path: StoragePath, + Proof: [][]byte{}, + }} + EmptyStorage = make([]statediff.StorageDiff, 0) contractAddress = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592") ContractLeafKey = crypto.Keccak256Hash(contractAddress[:]) anotherContractAddress = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476593") @@ -59,25 +68,20 @@ var ( { Key: ContractLeafKey.Bytes(), Value: valueBytes, - Storage: storage, - }, - { - Key: AnotherContractLeafKey.Bytes(), - Value: valueBytes, - Storage: emptyStorage, + Storage: storageWithSmallValue, }, } UpdatedAccountDiffs = []statediff.AccountDiff{{ Key: AnotherContractLeafKey.Bytes(), Value: valueBytes, - Storage: storage, + Storage: storageWithLargeValue, }} DeletedAccountDiffs = []statediff.AccountDiff{{ - Key: ContractLeafKey.Bytes(), + Key: AnotherContractLeafKey.Bytes(), Value: valueBytes, - Storage: storage, + Storage: storageWithSmallValue, }} MockStateDiff = statediff.StateDiff{