deep-stack/ipld-eth-server
4
0
Fork 0
forked from cerc-io/ipld-eth-server
Code Pull Requests Activity

(VDB-950) Write raw diffs before transforming

Browse Source 
- Raw field we can reference by FK for related data
- Enables replay for unwatched or mistransformed diffs
This commit is contained in:
Rob Mulholand
2019-12-03 14:51:17 -06:00
committed by Ian Norden
parent 1178940047
commit 56ce8bdb41
45 changed files with 543 additions and 1815 deletions
Download Patch File Download Diff File Expand all files Collapse all files
libraries/shared/storage/utils/decoder.go
+1 -1
View File
@@ -27,7 +27,7 @@ const (
bitsPerByte = 8
)
func Decode(diff StorageDiff, metadata StorageValueMetadata) (interface{}, error) {
func Decode(diff PersistedStorageDiff, metadata StorageValueMetadata) (interface{}, error) {
switch metadata.Type {
case Uint256:
return decodeInteger(diff.StorageValue.Bytes()), nil
libraries/shared/storage/utils/decoder_test.go
+8 -8
View File
@@ -29,7 +29,7 @@ import (
var _ = Describe("Storage decoder", func() {
It("decodes uint256", func() {
fakeInt := common.HexToHash("0000000000000000000000000000000000000000000000000000000000000539")
diff := utils.StorageDiff{StorageValue: fakeInt}
diff := utils.PersistedStorageDiff{StorageDiffInput: utils.StorageDiffInput{StorageValue: fakeInt}}
metadata := utils.StorageValueMetadata{Type: utils.Uint256}
result, err := utils.Decode(diff, metadata)
@@ -40,7 +40,7 @@ var _ = Describe("Storage decoder", func() {
It("decodes uint128", func() {
fakeInt := common.HexToHash("0000000000000000000000000000000000000000000000000000000000011123")
diff := utils.StorageDiff{StorageValue: fakeInt}
diff := utils.PersistedStorageDiff{StorageDiffInput: utils.StorageDiffInput{StorageValue: fakeInt}}
metadata := utils.StorageValueMetadata{Type: utils.Uint128}
result, err := utils.Decode(diff, metadata)
@@ -51,7 +51,7 @@ var _ = Describe("Storage decoder", func() {
It("decodes uint48", func() {
fakeInt := common.HexToHash("0000000000000000000000000000000000000000000000000000000000000123")
diff := utils.StorageDiff{StorageValue: fakeInt}
diff := utils.PersistedStorageDiff{StorageDiffInput: utils.StorageDiffInput{StorageValue: fakeInt}}
metadata := utils.StorageValueMetadata{Type: utils.Uint48}
result, err := utils.Decode(diff, metadata)
@@ -62,7 +62,7 @@ var _ = Describe("Storage decoder", func() {
It("decodes address", func() {
fakeAddress := common.HexToAddress("0x12345")
diff := utils.StorageDiff{StorageValue: fakeAddress.Hash()}
diff := utils.PersistedStorageDiff{StorageDiffInput: utils.StorageDiffInput{StorageValue: fakeAddress.Hash()}}
metadata := utils.StorageValueMetadata{Type: utils.Address}
result, err := utils.Decode(diff, metadata)
@@ -75,7 +75,7 @@ var _ = Describe("Storage decoder", func() {
It("decodes uint48 items", func() {
//this is a real storage data example
packedStorage := common.HexToHash("000000000000000000000000000000000000000000000002a300000000002a30")
diff := utils.StorageDiff{StorageValue: packedStorage}
diff := utils.PersistedStorageDiff{StorageDiffInput: utils.StorageDiffInput{StorageValue: packedStorage}}
packedTypes := map[int]utils.ValueType{}
packedTypes[0] = utils.Uint48
packedTypes[1] = utils.Uint48
@@ -99,7 +99,7 @@ var _ = Describe("Storage decoder", func() {
packedStorageHex := "0000000A5D1AFFFFFFFFFFFE00000009F3C600000002A300000000002A30"
packedStorage := common.HexToHash(packedStorageHex)
diff := utils.StorageDiff{StorageValue: packedStorage}
diff := utils.PersistedStorageDiff{StorageDiffInput: utils.StorageDiffInput{StorageValue: packedStorage}}
packedTypes := map[int]utils.ValueType{}
packedTypes[0] = utils.Uint48
packedTypes[1] = utils.Uint48
@@ -129,7 +129,7 @@ var _ = Describe("Storage decoder", func() {
packedStorageHex := "000000038D7EA4C67FF8E502B6730000" +
"0000000000000000AB54A98CEB1F0AD2"
packedStorage := common.HexToHash(packedStorageHex)
diff := utils.StorageDiff{StorageValue: packedStorage}
diff := utils.PersistedStorageDiff{StorageDiffInput: utils.StorageDiffInput{StorageValue: packedStorage}}
packedTypes := map[int]utils.ValueType{}
packedTypes[0] = utils.Uint128
packedTypes[1] = utils.Uint128
@@ -151,7 +151,7 @@ var _ = Describe("Storage decoder", func() {
//TODO: replace with real data when available
addressHex := "0000000000000000000000000000000000012345"
packedStorage := common.HexToHash("00000002a300" + "000000002a30" + addressHex)
row := utils.StorageDiff{StorageValue: packedStorage}
row := utils.PersistedStorageDiff{StorageDiffInput: utils.StorageDiffInput{StorageValue: packedStorage}}
packedTypes := map[int]utils.ValueType{}
packedTypes[0] = utils.Address
packedTypes[1] = utils.Uint48
libraries/shared/storage/utils/diff.go
+21 -10
View File
@@ -27,24 +27,28 @@ import (
const ExpectedRowLength = 5
type StorageDiff struct {
ID int
HashedAddress common.Hash `db:"contract"`
type StorageDiffInput struct {
HashedAddress common.Hash `db:"hashed_address"`
BlockHash common.Hash `db:"block_hash"`
BlockHeight int `db:"block_height"`
StorageKey common.Hash `db:"storage_key"`
StorageValue common.Hash `db:"storage_value"`
}
func FromParityCsvRow(csvRow []string) (StorageDiff, error) {
type PersistedStorageDiff struct {
StorageDiffInput
ID int64
}
func FromParityCsvRow(csvRow []string) (StorageDiffInput, error) {
if len(csvRow) != ExpectedRowLength {
return StorageDiff{}, ErrRowMalformed{Length: len(csvRow)}
return StorageDiffInput{}, ErrRowMalformed{Length: len(csvRow)}
}
height, err := strconv.Atoi(csvRow[2])
if err != nil {
return StorageDiff{}, err
return StorageDiffInput{}, err
}
return StorageDiff{
return StorageDiffInput{
HashedAddress: HexToKeccak256Hash(csvRow[0]),
BlockHash: common.HexToHash(csvRow[1]),
BlockHeight: height,
@@ -53,14 +57,14 @@ func FromParityCsvRow(csvRow []string) (StorageDiff, error) {
}, nil
}
func FromGethStateDiff(account statediff.AccountDiff, stateDiff *statediff.StateDiff, storage statediff.StorageDiff) (StorageDiff, error) {
func FromGethStateDiff(account statediff.AccountDiff, stateDiff *statediff.StateDiff, storage statediff.StorageDiff) (StorageDiffInput, error) {
var decodedValue []byte
err := rlp.DecodeBytes(storage.Value, &decodedValue)
if err != nil {
return StorageDiff{}, err
return StorageDiffInput{}, err
}
return StorageDiff{
return StorageDiffInput{
HashedAddress: common.BytesToHash(account.Key),
BlockHash: stateDiff.BlockHash,
BlockHeight: int(stateDiff.BlockNumber.Int64()),
@@ -69,6 +73,13 @@ func FromGethStateDiff(account statediff.AccountDiff, stateDiff *statediff.State
}, nil
}
func ToPersistedDiff(raw StorageDiffInput, id int64) PersistedStorageDiff {
return PersistedStorageDiff{
StorageDiffInput: raw,
ID: id,
}
}
func HexToKeccak256Hash(hex string) common.Hash {
return crypto.Keccak256Hash(common.FromHex(hex))
}