cerc-io/ipld-eth-server
12
0
Fork 1
Code Issues 10 Pull Requests 2 Actions Packages Projects Releases 37 Wiki Activity

Always hash storage diff contract addresses

Browse Source 
- Enables syncing Geth and Parity diffs with same transformer lookup
- Maybe worth always hashing the storage key so we don't need a hashed
  and not-hashed version in the key lookups?
This commit is contained in:
Rob Mulholand
2019-09-25 16:36:08 -05:00
committed by Elizabeth Engelman
parent f574407bb6
commit d06dddbfaa
19 changed files with 183 additions and 522 deletions
Download Patch File Download Diff File Expand all files Collapse all files
libraries/shared/storage/storage_queue.go
+1 -1
View File
@@ -38,7 +38,7 @@ func NewStorageQueue(db *postgres.DB) StorageQueue {
func (queue StorageQueue) Add(diff utils.StorageDiff) error {
_, err := queue.db.Exec(`INSERT INTO public.queued_storage (contract,
block_hash, block_height, storage_key, storage_value) VALUES
($1, $2, $3, $4, $5) ON CONFLICT DO NOTHING`, diff.Contract.Bytes(), diff.BlockHash.Bytes(),
($1, $2, $3, $4, $5) ON CONFLICT DO NOTHING`, diff.KeccakOfContractAddress.Bytes(), diff.BlockHash.Bytes(),
diff.BlockHeight, diff.StorageKey.Bytes(), diff.StorageValue.Bytes())
return err
}
libraries/shared/storage/storage_queue_test.go
+17 -13
View File
@@ -18,9 +18,9 @@ package storage_test
import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/crypto"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/libraries/shared/storage"
"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
"github.com/vulcanize/vulcanizedb/pkg/datastore/postgres"
@@ -35,12 +35,14 @@ var _ = Describe("Storage queue", func() {
)
BeforeEach(func() {
fakeAddr := common.FromHex("0x123456")
hashedFakeAddr := crypto.Keccak256(fakeAddr)
diff = utils.StorageDiff{
Contract: common.HexToAddress("0x123456"),
BlockHash: common.HexToHash("0x678901"),
BlockHeight: 987,
StorageKey: common.HexToHash("0x654321"),
StorageValue: common.HexToHash("0x198765"),
KeccakOfContractAddress: common.BytesToHash(hashedFakeAddr),
BlockHash: common.HexToHash("0x678901"),
BlockHeight: 987,
StorageKey: common.HexToHash("0x654321"),
StorageValue: common.HexToHash("0x198765"),
}
db = test_config.NewTestDB(test_config.NewTestNode())
test_config.CleanTestDB(db)
@@ -81,12 +83,14 @@ var _ = Describe("Storage queue", func() {
})
It("gets all storage diffs from db", func() {
fakeAddr := common.FromHex("0x234567")
hashedFakeAddr := crypto.Keccak256(fakeAddr)
diffTwo := utils.StorageDiff{
Contract: common.HexToAddress("0x123456"),
BlockHash: common.HexToHash("0x678902"),
BlockHeight: 988,
StorageKey: common.HexToHash("0x654322"),
StorageValue: common.HexToHash("0x198766"),
KeccakOfContractAddress: common.BytesToHash(hashedFakeAddr),
BlockHash: common.HexToHash("0x678902"),
BlockHeight: 988,
StorageKey: common.HexToHash("0x654322"),
StorageValue: common.HexToHash("0x198766"),
}
addErr := queue.Add(diffTwo)
Expect(addErr).NotTo(HaveOccurred())
@@ -97,13 +101,13 @@ var _ = Describe("Storage queue", func() {
Expect(len(diffs)).To(Equal(2))
Expect(diffs[0]).NotTo(Equal(diffs[1]))
Expect(diffs[0].Id).NotTo(BeZero())
Expect(diffs[0].Contract).To(Or(Equal(diff.Contract), Equal(diffTwo.Contract)))
Expect(diffs[0].KeccakOfContractAddress).To(Or(Equal(diff.KeccakOfContractAddress), Equal(diffTwo.KeccakOfContractAddress)))
Expect(diffs[0].BlockHash).To(Or(Equal(diff.BlockHash), Equal(diffTwo.BlockHash)))
Expect(diffs[0].BlockHeight).To(Or(Equal(diff.BlockHeight), Equal(diffTwo.BlockHeight)))
Expect(diffs[0].StorageKey).To(Or(Equal(diff.StorageKey), Equal(diffTwo.StorageKey)))
Expect(diffs[0].StorageValue).To(Or(Equal(diff.StorageValue), Equal(diffTwo.StorageValue)))
Expect(diffs[1].Id).NotTo(BeZero())
Expect(diffs[1].Contract).To(Or(Equal(diff.Contract), Equal(diffTwo.Contract)))
Expect(diffs[1].KeccakOfContractAddress).To(Or(Equal(diff.KeccakOfContractAddress), Equal(diffTwo.KeccakOfContractAddress)))
Expect(diffs[1].BlockHash).To(Or(Equal(diff.BlockHash), Equal(diffTwo.BlockHash)))
Expect(diffs[1].BlockHeight).To(Or(Equal(diff.BlockHeight), Equal(diffTwo.BlockHeight)))
Expect(diffs[1].StorageKey).To(Or(Equal(diff.StorageKey), Equal(diffTwo.StorageKey)))
libraries/shared/storage/utils/diff.go
+21 -10
View File
@@ -17,24 +17,24 @@
package utils
import (
"strconv"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/statediff"
"strconv"
)
const ExpectedRowLength = 5
type StorageDiff struct {
Id int
Contract common.Address
KeccakOfContractAddress common.Hash
KeccakOfContractAddress common.Hash `db:"contract"`
BlockHash common.Hash `db:"block_hash"`
BlockHeight int `db:"block_height"`
StorageKey common.Hash `db:"storage_key"`
StorageValue common.Hash `db:"storage_value"`
}
func FromStrings(csvRow []string) (StorageDiff, error) {
func FromParityCsvRow(csvRow []string) (StorageDiff, error) {
if len(csvRow) != ExpectedRowLength {
return StorageDiff{}, ErrRowMalformed{Length: len(csvRow)}
}
@@ -42,11 +42,22 @@ func FromStrings(csvRow []string) (StorageDiff, error) {
if err != nil {
return StorageDiff{}, err
}
hashedAddr := crypto.Keccak256(common.FromHex(csvRow[0]))
return StorageDiff{
Contract: common.HexToAddress(csvRow[0]),
BlockHash: common.HexToHash(csvRow[1]),
BlockHeight: height,
StorageKey: common.HexToHash(csvRow[3]),
StorageValue: common.HexToHash(csvRow[4]),
KeccakOfContractAddress: common.BytesToHash(hashedAddr),
BlockHash: common.HexToHash(csvRow[1]),
BlockHeight: height,
StorageKey: common.HexToHash(csvRow[3]),
StorageValue: common.HexToHash(csvRow[4]),
}, nil
}
func FromGethStateDiff(account statediff.AccountDiff, stateDiff *statediff.StateDiff, storage statediff.StorageDiff) StorageDiff {
return StorageDiff{
KeccakOfContractAddress: common.BytesToHash(account.Key),
BlockHash: stateDiff.BlockHash,
BlockHeight: int(stateDiff.BlockNumber.Int64()),
StorageKey: common.BytesToHash(storage.Key),
StorageValue: common.BytesToHash(storage.Value),
}
}
libraries/shared/storage/utils/diff_test.go
+57 -24
View File
@@ -18,41 +18,74 @@ package utils_test
import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/statediff"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/libraries/shared/storage/utils"
"github.com/vulcanize/vulcanizedb/pkg/fakes"
"math/big"
"math/rand"
)
var _ = Describe("Storage row parsing", func() {
It("converts an array of strings to a row struct", func() {
contract := "0x123"
blockHash := "0x456"
blockHeight := "789"
storageKey := "0x987"
storageValue := "0x654"
data := []string{contract, blockHash, blockHeight, storageKey, storageValue}
Describe("FromParityCsvRow", func() {
It("converts an array of strings to a row struct", func() {
contract := "0x123"
blockHash := "0x456"
blockHeight := "789"
storageKey := "0x987"
storageValue := "0x654"
data := []string{contract, blockHash, blockHeight, storageKey, storageValue}
result, err := utils.FromStrings(data)
result, err := utils.FromParityCsvRow(data)
Expect(err).NotTo(HaveOccurred())
Expect(result.Contract).To(Equal(common.HexToAddress(contract)))
Expect(result.BlockHash).To(Equal(common.HexToHash(blockHash)))
Expect(result.BlockHeight).To(Equal(789))
Expect(result.StorageKey).To(Equal(common.HexToHash(storageKey)))
Expect(result.StorageValue).To(Equal(common.HexToHash(storageValue)))
Expect(err).NotTo(HaveOccurred())
expectedKeccakOfContractAddress := common.BytesToHash(crypto.Keccak256(common.FromHex(contract)))
Expect(result.KeccakOfContractAddress).To(Equal(expectedKeccakOfContractAddress))
Expect(result.BlockHash).To(Equal(common.HexToHash(blockHash)))
Expect(result.BlockHeight).To(Equal(789))
Expect(result.StorageKey).To(Equal(common.HexToHash(storageKey)))
Expect(result.StorageValue).To(Equal(common.HexToHash(storageValue)))
})
It("returns an error if row is missing data", func() {
_, err := utils.FromParityCsvRow([]string{"0x123"})
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(utils.ErrRowMalformed{Length: 1}))
})
It("returns error if block height malformed", func() {
_, err := utils.FromParityCsvRow([]string{"", "", "", "", ""})
Expect(err).To(HaveOccurred())
})
})
It("returns an error if row is missing data", func() {
_, err := utils.FromStrings([]string{"0x123"})
Describe("FromGethStateDiff", func() {
It("adds relevant fields to diff", func() {
accountDiff := statediff.AccountDiff{Key: []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 0}}
stateDiff := &statediff.StateDiff{
BlockNumber: big.NewInt(rand.Int63()),
BlockHash: fakes.FakeHash,
}
storageDiff := statediff.StorageDiff{
Key: []byte{0, 9, 8, 7, 6, 5, 4, 3, 2, 1},
Value: []byte{1, 2, 3, 4, 5, 0, 9, 8, 7, 6},
}
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(utils.ErrRowMalformed{Length: 1}))
})
result := utils.FromGethStateDiff(accountDiff, stateDiff, storageDiff)
It("returns error if block height malformed", func() {
_, err := utils.FromStrings([]string{"", "", "", "", ""})
Expect(err).To(HaveOccurred())
expectedAddress := common.BytesToHash(accountDiff.Key)
Expect(result.KeccakOfContractAddress).To(Equal(expectedAddress))
Expect(result.BlockHash).To(Equal(fakes.FakeHash))
expectedBlockHeight := int(stateDiff.BlockNumber.Int64())
Expect(result.BlockHeight).To(Equal(expectedBlockHeight))
expectedStorageKey := common.BytesToHash(storageDiff.Key)
Expect(result.StorageKey).To(Equal(expectedStorageKey))
expectedStorageValue := common.BytesToHash(storageDiff.Value)
Expect(result.StorageValue).To(Equal(expectedStorageValue))
})
})
})