cerc-io/ipld-eth-server
12
0
Fork 1
Code Issues 10 Pull Requests 2 Actions Packages Projects Releases 37 Wiki Activity

(VDB-298) Consume Pit contract storage diffs

Browse Source 
- Continuously parse storage diffs CSV data to read Pit contract state
- Convert ilks in database to raw bytes32 value for use in generating
  storage keys dynamically
- Persist storage diffs with block number and hash for validation
This commit is contained in:
Rob Mulholand
2019-02-06 10:31:46 -06:00
parent c474933432
commit 867f92c431
85 changed files with 2143 additions and 79 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/transformers/storage_diffs/shared/decoder.go
+42
View File
@@ -0,0 +1,42 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package shared
import (
"github.com/ethereum/go-ethereum/common"
"math/big"
)
func Decode(row StorageDiffRow, metadata StorageValueMetadata) (interface{}, error) {
switch metadata.Type {
case Uint256:
return decodeUint256(row.StorageValue.Bytes()), nil
case Address:
return decodeAddress(row.StorageValue.Bytes()), nil
default:
return nil, ErrTypeNotFound{}
}
}
func decodeUint256(raw []byte) string {
n := big.NewInt(0).SetBytes(raw)
return n.String()
}
func decodeAddress(raw []byte) string {
return common.BytesToAddress(raw).Hex()
}
pkg/transformers/storage_diffs/shared/decoder_test.go
+58
View File
@@ -0,0 +1,58 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package shared_test
import (
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared"
"math/big"
)
var _ = Describe("Storage decoder", func() {
It("decodes uint256", func() {
fakeInt := common.HexToHash("0000000000000000000000000000000000000000000000000000000000000539")
row := shared.StorageDiffRow{StorageValue: fakeInt}
metadata := shared.StorageValueMetadata{Type: shared.Uint256}
result, err := shared.Decode(row, metadata)
Expect(err).NotTo(HaveOccurred())
Expect(result).To(Equal(big.NewInt(0).SetBytes(fakeInt.Bytes()).String()))
})
It("decodes address", func() {
fakeAddress := common.HexToAddress("0x12345")
row := shared.StorageDiffRow{StorageValue: fakeAddress.Hash()}
metadata := shared.StorageValueMetadata{Type: shared.Address}
result, err := shared.Decode(row, metadata)
Expect(err).NotTo(HaveOccurred())
Expect(result).To(Equal(fakeAddress.Hex()))
})
It("returns error if attempting to decode unknown type", func() {
metadata := shared.StorageValueMetadata{Type: 100}
_, err := shared.Decode(shared.StorageDiffRow{}, metadata)
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(shared.ErrTypeNotFound{}))
})
})
pkg/transformers/storage_diffs/shared/errors.go
+61
View File
@@ -0,0 +1,61 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package shared
import "fmt"
type ErrContractNotFound struct {
Contract string
}
func (e ErrContractNotFound) Error() string {
return fmt.Sprintf("transformer not found for contract: %s", e.Contract)
}
type ErrHeaderMismatch struct {
BlockHeight int
DbHash string
DiffHash string
}
func (e ErrHeaderMismatch) Error() string {
return fmt.Sprintf("header hash in row does not match db at height %d - row: %s, db: %s", e.BlockHeight, e.DbHash, e.DiffHash)
}
type ErrRowMalformed struct {
Length int
}
func (e ErrRowMalformed) Error() string {
return fmt.Sprintf("storage row malformed: length %d, expected %d", e.Length, ExpectedRowLength)
}
type ErrStorageKeyNotFound struct {
Key string
}
func (e ErrStorageKeyNotFound) Error() string {
return fmt.Sprintf("unknown storage key: %s", e.Key)
}
type ErrTypeNotFound struct {
Type int
}
func (e ErrTypeNotFound) Error() string {
return fmt.Sprintf("no decoder for type: %d", e.Type)
}
pkg/transformers/storage_diffs/shared/row.go
+49
View File
@@ -0,0 +1,49 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package shared
import (
"github.com/ethereum/go-ethereum/common"
"strconv"
)
const ExpectedRowLength = 5
type StorageDiffRow struct {
Contract common.Address
BlockHash common.Hash
BlockHeight int
StorageKey common.Hash
StorageValue common.Hash
}
func FromStrings(csvRow []string) (StorageDiffRow, error) {
if len(csvRow) != ExpectedRowLength {
return StorageDiffRow{}, ErrRowMalformed{Length: len(csvRow)}
}
height, err := strconv.Atoi(csvRow[2])
if err != nil {
return StorageDiffRow{}, err
}
return StorageDiffRow{
Contract: common.HexToAddress(csvRow[0]),
BlockHash: common.HexToHash(csvRow[1]),
BlockHeight: height,
StorageKey: common.HexToHash(csvRow[3]),
StorageValue: common.HexToHash(csvRow[4]),
}, nil
}
pkg/transformers/storage_diffs/shared/row_test.go
+57
View File
@@ -0,0 +1,57 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package shared_test
import (
"github.com/ethereum/go-ethereum/common"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/vulcanizedb/pkg/transformers/storage_diffs/shared"
)
var _ = Describe("Storage row parsing", func() {
It("converts an array of strings to a row struct", func() {
contract := "0x123"
blockHash := "0x456"
blockHeight := "789"
storageKey := "0x987"
storageValue := "0x654"
data := []string{contract, blockHash, blockHeight, storageKey, storageValue}
result, err := shared.FromStrings(data)
Expect(err).NotTo(HaveOccurred())
Expect(result.Contract).To(Equal(common.HexToAddress(contract)))
Expect(result.BlockHash).To(Equal(common.HexToHash(blockHash)))
Expect(result.BlockHeight).To(Equal(789))
Expect(result.StorageKey).To(Equal(common.HexToHash(storageKey)))
Expect(result.StorageValue).To(Equal(common.HexToHash(storageValue)))
})
It("returns an error if row is missing data", func() {
_, err := shared.FromStrings([]string{"0x123"})
Expect(err).To(HaveOccurred())
Expect(err).To(MatchError(shared.ErrRowMalformed{Length: 1}))
})
It("returns error if block height malformed", func() {
_, err := shared.FromStrings([]string{"", "", "", "", ""})
Expect(err).To(HaveOccurred())
})
})
pkg/transformers/storage_diffs/shared/shared_suite_test.go
+29
View File
@@ -0,0 +1,29 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package shared_test
import (
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
func TestShared(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "Shared Suite")
}
pkg/transformers/storage_diffs/shared/value.go
+31
View File
@@ -0,0 +1,31 @@
// VulcanizeDB
// Copyright © 2018 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package shared
type ValueType int
const (
Uint256 ValueType = iota
Bytes32
Address
)
type StorageValueMetadata struct {
Name string
Key string
Type ValueType
}