Decode storage value RLP after fetching from statediffs

libraries/shared/fetcher/geth_rpc_storage_fetcher.go

@@ -60,7 +60,12 @@ func (fetcher GethRpcStorageFetcher) FetchStorageDiffs(out chan<- utils.StorageD
 			logrus.Trace(fmt.Sprintf("iterating through %d Storage values on account", len(account.Storage)))
 			for _, storage := range account.Storage {
 				logrus.Trace("adding storage diff to out channel")
-				out <- utils.FromGethStateDiff(account, stateDiff, storage)
+				diff, formatErr := utils.FromGethStateDiff(account, stateDiff, storage)
+				if formatErr != nil {
+					errs <- formatErr
+				}
+
+				out <- diff
 			}
 		}
 	}

libraries/shared/fetcher/geth_rpc_storage_fetcher_test.go

@@ -16,6 +16,7 @@ package fetcher_test
 
 import (
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/rpc"
 	"github.com/ethereum/go-ethereum/statediff"
 	. "github.com/onsi/ginkgo"
@@ -93,34 +94,7 @@ var _ = Describe("Geth RPC Storage Fetcher", func() {
 		close(done)
 	})
 
-	It("adds parsed statediff payloads to the rows channel", func(done Done) {
+	It("adds errors to error channel if decoding the state diff RLP fails", func(done Done) {
-		streamer.SetPayloads([]statediff.Payload{test_data.MockStatediffPayload})
-
-		go statediffFetcher.FetchStorageDiffs(storagediffChan, errorChan)
-
-		height := test_data.BlockNumber
-		intHeight := int(height.Int64())
-		expectedStorageDiff := utils.StorageDiff{
-			KeccakOfContractAddress: common.BytesToHash(test_data.ContractLeafKey[:]),
-			BlockHash:               common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"),
-			BlockHeight:             intHeight,
-			StorageKey:              common.BytesToHash(test_data.StorageKey),
-			StorageValue:            common.BytesToHash(test_data.StorageValue),
-		}
-		anotherExpectedStorageDiff := utils.StorageDiff{
-			KeccakOfContractAddress: common.BytesToHash(test_data.AnotherContractLeafKey[:]),
-			BlockHash:               common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"),
-			BlockHeight:             intHeight,
-			StorageKey:              common.BytesToHash(test_data.StorageKey),
-			StorageValue:            common.BytesToHash(test_data.StorageValue),
-		}
-		Expect(<-storagediffChan).To(Equal(expectedStorageDiff))
-		Expect(<-storagediffChan).To(Equal(anotherExpectedStorageDiff))
-
-		close(done)
-	})
-
-	It("adds errors to error channel if parsing the diff fails", func(done Done) {
 		badStatediffPayload := statediff.Payload{}
 		streamer.SetPayloads([]statediff.Payload{badStatediffPayload})
 
@@ -130,4 +104,78 @@ var _ = Describe("Geth RPC Storage Fetcher", func() {
 
 		close(done)
 	})
+
+	It("adds parsed statediff payloads to the rows channel", func(done Done) {
+		streamer.SetPayloads([]statediff.Payload{test_data.MockStatediffPayload})
+
+		go statediffFetcher.FetchStorageDiffs(storagediffChan, errorChan)
+
+		height := test_data.BlockNumber
+		intHeight := int(height.Int64())
+		createdExpectedStorageDiff := utils.StorageDiff{
+			KeccakOfContractAddress: common.BytesToHash(test_data.ContractLeafKey[:]),
+			BlockHash:               common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"),
+			BlockHeight:             intHeight,
+			StorageKey:              common.BytesToHash(test_data.StorageKey),
+			StorageValue:            common.BytesToHash(test_data.SmallStorageValue),
+		}
+		updatedExpectedStorageDiff := utils.StorageDiff{
+			KeccakOfContractAddress: common.BytesToHash(test_data.AnotherContractLeafKey[:]),
+			BlockHash:               common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"),
+			BlockHeight:             intHeight,
+			StorageKey:              common.BytesToHash(test_data.StorageKey),
+			StorageValue:            common.BytesToHash(test_data.LargeStorageValue),
+		}
+		deletedExpectedStorageDiff := utils.StorageDiff{
+			KeccakOfContractAddress: common.BytesToHash(test_data.AnotherContractLeafKey[:]),
+			BlockHash:               common.HexToHash("0xfa40fbe2d98d98b3363a778d52f2bcd29d6790b9b3f3cab2b167fd12d3550f73"),
+			BlockHeight:             intHeight,
+			StorageKey:              common.BytesToHash(test_data.StorageKey),
+			StorageValue:            common.BytesToHash(test_data.SmallStorageValue),
+		}
+
+		createdStateDiff := <-storagediffChan
+		updatedStateDiff := <-storagediffChan
+		deletedStateDiff := <-storagediffChan
+
+		Expect(createdStateDiff).To(Equal(createdExpectedStorageDiff))
+		Expect(updatedStateDiff).To(Equal(updatedExpectedStorageDiff))
+		Expect(deletedStateDiff).To(Equal(deletedExpectedStorageDiff))
+
+		close(done)
+	})
+
+	It("adds errors to error channel if formatting the diff as a StateDiff object fails", func(done Done) {
+		badStorageDiffs := []statediff.StorageDiff{{
+			Key:   test_data.StorageKey,
+			Value: []byte{1, 2, 3},
+			// this storage value will fail to be decoded as an RLP with the following error message:
+			// "input contains more than one value"
+			Path:  test_data.StoragePath,
+			Proof: [][]byte{},
+		}}
+
+		accountDiffs := test_data.CreatedAccountDiffs
+		accountDiffs[0].Storage = badStorageDiffs
+
+		stateDiff := statediff.StateDiff{
+			BlockNumber:     test_data.BlockNumber,
+			BlockHash:       common.HexToHash(test_data.BlockHash),
+			CreatedAccounts: accountDiffs,
+		}
+
+		stateDiffRlp, err := rlp.EncodeToBytes(stateDiff)
+		Expect(err).NotTo(HaveOccurred())
+
+		badStatediffPayload := statediff.Payload{
+			StateDiffRlp: stateDiffRlp,
+		}
+		streamer.SetPayloads([]statediff.Payload{badStatediffPayload})
+
+		go statediffFetcher.FetchStorageDiffs(storagediffChan, errorChan)
+
+		Expect(<-errorChan).To(MatchError("rlp: input contains more than one value"))
+
+		close(done)
+	})
 })

libraries/shared/storage/utils/diff.go

@@ -19,6 +19,7 @@ package utils
 import (
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/crypto"
+	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/statediff"
 	"strconv"
 )
@@ -51,14 +52,20 @@ func FromParityCsvRow(csvRow []string) (StorageDiff, error) {
 	}, nil
 }
 
-func FromGethStateDiff(account statediff.AccountDiff, stateDiff *statediff.StateDiff, storage statediff.StorageDiff) StorageDiff {
+func FromGethStateDiff(account statediff.AccountDiff, stateDiff *statediff.StateDiff, storage statediff.StorageDiff) (StorageDiff, error) {
+	var decodedValue []byte
+	err := rlp.DecodeBytes(storage.Value, &decodedValue)
+	if err != nil {
+		return StorageDiff{}, err
+	}
+
 	return StorageDiff{
 		KeccakOfContractAddress: common.BytesToHash(account.Key),
 		BlockHash:               stateDiff.BlockHash,
 		BlockHeight:             int(stateDiff.BlockNumber.Int64()),
 		StorageKey:              common.BytesToHash(storage.Key),
-		StorageValue:            common.BytesToHash(storage.Value),
+		StorageValue:            common.BytesToHash(decodedValue),
-	}
+	}, nil
 }
 
 func HexToKeccak256Hash(hex string) common.Hash {

libraries/shared/storage/utils/diff_test.go

@@ -18,10 +18,12 @@ package utils_test
 
 import (
 	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/rlp"
 	"github.com/ethereum/go-ethereum/statediff"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
+	"github.com/vulcanize/vulcanizedb/libraries/shared/test_data"
 	"github.com/vulcanize/vulcanizedb/pkg/fakes"
 	"math/big"
 	"math/rand"
@@ -63,18 +65,26 @@ var _ = Describe("Storage row parsing", func() {
 	})
 
 	Describe("FromGethStateDiff", func() {
-		It("adds relevant fields to diff", func() {
+		var (
-			accountDiff := statediff.AccountDiff{Key: []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 0}}
+			accountDiff = statediff.AccountDiff{Key: []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 0}}
-			stateDiff := &statediff.StateDiff{
+			stateDiff = &statediff.StateDiff{
 				BlockNumber: big.NewInt(rand.Int63()),
 				BlockHash:   fakes.FakeHash,
 			}
+		)
+
+		It("adds relevant fields to diff", func() {
+			storageValueBytes := []byte{3}
+			storageValueRlp, encodeErr := rlp.EncodeToBytes(storageValueBytes)
+			Expect(encodeErr).NotTo(HaveOccurred())
+
 			storageDiff := statediff.StorageDiff{
 				Key:   []byte{0, 9, 8, 7, 6, 5, 4, 3, 2, 1},
-				Value: []byte{1, 2, 3, 4, 5, 0, 9, 8, 7, 6},
+				Value: storageValueRlp,
 			}
 
-			result := utils.FromGethStateDiff(accountDiff, stateDiff, storageDiff)
+			result, err := utils.FromGethStateDiff(accountDiff, stateDiff, storageDiff)
+			Expect(err).NotTo(HaveOccurred())
 
 			expectedAddress := common.BytesToHash(accountDiff.Key)
 			Expect(result.KeccakOfContractAddress).To(Equal(expectedAddress))
@@ -83,8 +93,34 @@ var _ = Describe("Storage row parsing", func() {
 			Expect(result.BlockHeight).To(Equal(expectedBlockHeight))
 			expectedStorageKey := common.BytesToHash(storageDiff.Key)
 			Expect(result.StorageKey).To(Equal(expectedStorageKey))
-			expectedStorageValue := common.BytesToHash(storageDiff.Value)
+			expectedStorageValue := common.BytesToHash(storageValueBytes)
 			Expect(result.StorageValue).To(Equal(expectedStorageValue))
 		})
+
+		It("handles decoding large storage values from their RLP", func() {
+			storageValueBytes := []byte{1, 2, 3, 4, 5, 0, 9, 8, 7, 6}
+			storageValueRlp, encodeErr := rlp.EncodeToBytes(storageValueBytes)
+			Expect(encodeErr).NotTo(HaveOccurred())
+
+			storageDiff := statediff.StorageDiff{
+				Key:   []byte{0, 9, 8, 7, 6, 5, 4, 3, 2, 1},
+				Value: storageValueRlp,
+			}
+
+			result, err := utils.FromGethStateDiff(accountDiff, stateDiff, storageDiff)
+			Expect(err).NotTo(HaveOccurred())
+			Expect(result.StorageValue).To(Equal(common.BytesToHash(storageValueBytes)))
+		})
+
+		It("returns an err if decoding the storage value Rlp fails", func() {
+			storageDiff := statediff.StorageDiff{
+				Key:   []byte{0, 9, 8, 7, 6, 5, 4, 3, 2, 1},
+				Value: test_data.StorageKey,
+			}
+
+			_, err := utils.FromGethStateDiff(accountDiff, stateDiff, storageDiff)
+			Expect(err).To(HaveOccurred())
+			Expect(err).To(MatchError("rlp: input contains more than one value"))
+		})
 	})
 })

libraries/shared/test_data/statediff.go

@@ -35,14 +35,23 @@ var (
 	ContractRoot      = common.HexToHash("0x56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421")
 	StoragePath       = common.HexToHash("0xc5d2460186f7233c927e7db2dcc703c0e500b653ca82273b7bfad8045d85a470").Bytes()
 	StorageKey        = common.HexToHash("0000000000000000000000000000000000000000000000000000000000000001").Bytes()
-	StorageValue    = common.Hex2Bytes("0x03")
+	SmallStorageValue = common.Hex2Bytes("03")
-	storage         = []statediff.StorageDiff{{
+	SmallStorageValueRlp, _ = rlp.EncodeToBytes(SmallStorageValue)
+	storageWithSmallValue = []statediff.StorageDiff{{
 		Key:   StorageKey,
-		Value: StorageValue,
+		Value: SmallStorageValueRlp,
 		Path:  StoragePath,
 		Proof: [][]byte{},
 	}}
-	emptyStorage           = make([]statediff.StorageDiff, 0)
+	LargeStorageValue = common.Hex2Bytes("00191b53778c567b14b50ba0000")
+	LargeStorageValueRlp, rlpErr = rlp.EncodeToBytes(LargeStorageValue)
+	storageWithLargeValue = []statediff.StorageDiff{{
+		Key:   StorageKey,
+		Value: LargeStorageValueRlp,
+		Path:  StoragePath,
+		Proof: [][]byte{},
+	}}
+	EmptyStorage           = make([]statediff.StorageDiff, 0)
 	contractAddress        = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592")
 	ContractLeafKey        = crypto.Keccak256Hash(contractAddress[:])
 	anotherContractAddress = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476593")
@@ -59,25 +68,20 @@ var (
 		{
 			Key:     ContractLeafKey.Bytes(),
 			Value:   valueBytes,
-			Storage: storage,
+			Storage: storageWithSmallValue,
-		},
-		{
-			Key:     AnotherContractLeafKey.Bytes(),
-			Value:   valueBytes,
-			Storage: emptyStorage,
 		},
 	}
 
 	UpdatedAccountDiffs = []statediff.AccountDiff{{
 		Key:     AnotherContractLeafKey.Bytes(),
 		Value:   valueBytes,
-		Storage: storage,
+		Storage: storageWithLargeValue,
 	}}
 
 	DeletedAccountDiffs = []statediff.AccountDiff{{
-		Key:     ContractLeafKey.Bytes(),
+		Key:     AnotherContractLeafKey.Bytes(),
 		Value:   valueBytes,
-		Storage: storage,
+		Storage: storageWithSmallValue,
 	}}
 
 	MockStateDiff = statediff.StateDiff{