cerc-io/eth-ipfs-state-validator
16
0
Fork 1
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #3 from vulcanize/validateTrie

Browse Source 
Unit tests and block-service based type
This commit is contained in:
Ian Norden 2020-07-12 23:20:23 -05:00 committed by GitHub
commit ca51cd14d3
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 1312 additions and 29 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
README.md
View File

@ -1,6 +1,8 @@
# eth-ipfs-state-validator
Uses [pg-ipfs-ethdb](https://github.com/vulcanize/pg-ipfs-ethdb) to validate completeness of Ethereum state data on IPFS
[![Go Report Card](https://goreportcard.com/badge/github.com/vulcanize/eth-ipfs-state-validator)](https://goreportcard.com/report/github.com/vulcanize/eth-ipfs-state-validator)
> Uses [ipfs-ethdb](https://github.com/vulcanize/ipfs-ethdb/postgres) to validate completeness of IPFS Ethereum state data
## Background
@ -16,20 +18,27 @@ in a database- requires traversing the entire trie (or linked set of tries) and
`full` validates completeness of the entire state corresponding to a provided state root, including both state and storage tries
`./eth-ipfs-state-validator validateTrie --config={path to db config} --type=full --state-root={state root hex string}`
`./eth-ipfs-state-validator validateTrie --ipfs-path={path to ipfs repo} --type=full --state-root={state root hex string}`
`state` validates completeness of the state trie corresponding to a provided state root, excluding the storage tries
`./eth-ipfs-state-validator validateTrie --config={path to db config} --type=state --state-root={state root hex string}`
`./eth-ipfs-state-validator validateTrie --ipfs-path={path to ipfs repo} --type=state --state-root={state root hex string}`
`storage` validates completeness of only the storage trie corresponding to a provided storage root and contract address
`./eth-ipfs-state-validator validateTrie --config={path to db config} --type=storage --storage-root={state root hex string} --address={contract address hex string}`
`./eth-ipfs-state-validator validateTrie --ipfs-path={path to ipfs repo} --type=storage --storage-root={state root hex string} --address={contract address hex string}`
The config file holds the parameters for connecting to an [IPFS-backing Postgres database](https://github.com/ipfs/go-ds-sql).
If an IPFS path is provided with the `--ipfs-path` flag, the validator operates through an IPFS block-service and expects a configured IPFS repository at
the provided path. In this case, the validator will vie for contention on the lockfile located at the ipfs path.
Alternatively, if no IPFS path is provided, the `--config` flag can be used to provide a path to a .toml config file with
Postgres database connection parameters. In this case, the validator interfaces directly with the Postgres database and the
database is assumed to be [IPFS-backing](https://github.com/ipfs/go-ds-sql).
Postgres DB config:
```toml
[database]

1
cmd/root.go
View File

@ -31,6 +31,7 @@ var (
validationType string
contractAddrStr string
cfgFile string
ipfsPath string
)
// rootCmd represents the base command when called without any subcommands

21
cmd/validateTrie.go
View File

@ -32,6 +32,8 @@ var validateTrieCmd = &cobra.Command{
Short: "Validate completeness of state data on IPFS",
Long: `This command is used to validate the completeness of state data corresponding specific to a specific root
If an ipfs-path is provided it will use a blockservice, otherwise it expects Postgres db configuration in a linked config file.
It can operate at three levels:
"full" validates completeness of the entire state corresponding to a provided state root, including both state and storage tries
@ -56,11 +58,10 @@ It can operate at three levels:
}
func validateTrie() {
db, err := validator.NewDB()
v, err := newValidator()
if err != nil {
logWithCommand.Fatal(err)
}
v := validator.NewValidator(db)
switch strings.ToLower(validationType) {
case "f", "full":
if stateRootStr == "" {
@ -96,10 +97,26 @@ func validateTrie() {
}
}
func newValidator() (*validator.Validator, error) {
if ipfsPath == "" {
db, err := validator.NewDB()
if err != nil {
logWithCommand.Fatal(err)
}
return validator.NewPGIPFSValidator(db), nil
}
bs, err := validator.InitIPFSBlockService(ipfsPath)
if err != nil {
return nil, err
}
return validator.NewIPFSValidator(bs), nil
}
func init() {
rootCmd.AddCommand(validateTrieCmd)
validateTrieCmd.Flags().StringVarP(&stateRootStr, "state-root", "s", "", "Root of the state trie we wish to validate; for full or state validation")
validateTrieCmd.Flags().StringVarP(&validationType, "type", "t", "full", "Type of validations: full, state, storage")
validateTrieCmd.Flags().StringVarP(&storageRootStr, "storage-root", "o", "", "Root of the storage trie we wish to validate; for storage validation")
validateTrieCmd.Flags().StringVarP(&contractAddrStr, "address", "a", "", "Contract address for the storage trie we wish to validate; for storage validation")
validateTrieCmd.Flags().StringVarP(&ipfsPath, "ipfs-path", "i", "", "Path to IPFS repository")
}

12
go.mod
View File

@ -4,12 +4,22 @@ go 1.13
require (
github.com/ethereum/go-ethereum v1.9.15
github.com/hashicorp/golang-lru v0.5.4
github.com/ipfs/go-block-format v0.0.2
github.com/ipfs/go-blockservice v0.1.3
github.com/ipfs/go-cid v0.0.5
github.com/ipfs/go-filestore v1.0.0 //indirect
github.com/ipfs/go-ipfs v0.5.1
github.com/ipfs/go-ipfs-blockstore v1.0.0
github.com/ipfs/go-ipfs-ds-help v1.0.0
github.com/ipfs/go-ipfs-exchange-interface v0.0.1
github.com/jmoiron/sqlx v1.2.0
github.com/lib/pq v1.5.2
github.com/multiformats/go-multihash v0.0.13
github.com/onsi/ginkgo v1.12.1
github.com/onsi/gomega v1.10.1
github.com/sirupsen/logrus v1.6.0
github.com/spf13/cobra v1.0.0
github.com/spf13/viper v1.7.0
github.com/vulcanize/pg-ipfs-ethdb v0.0.2-alpha
github.com/vulcanize/ipfs-ethdb v0.0.2
)

848
go.sum
View File

File diff suppressed because it is too large Load Diff

2
pkg/database.go
View File

@ -1,5 +1,5 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// Copyright © 2020 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by

29
pkg/suite_test.go Normal file
View File

@ -0,0 +1,29 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package validator_test
import (
"testing"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
func TestTrieValidator(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "IPFS ETH trie validator test")
}

80
pkg/util.go Normal file
View File

@ -0,0 +1,80 @@
// VulcanizeDB
// Copyright © 2020 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package validator
import (
"context"
"github.com/ipfs/go-blockservice"
"github.com/ipfs/go-cid"
"github.com/ipfs/go-ipfs-blockstore"
"github.com/ipfs/go-ipfs-ds-help"
"github.com/ipfs/go-ipfs/core"
"github.com/ipfs/go-ipfs/repo/fsrepo"
"github.com/jmoiron/sqlx"
)
// PublishRaw derives a cid from raw bytes and provided codec and multihash type, and writes it to the db tx
func PublishRaw(tx *sqlx.Tx, codec, mh uint64, raw []byte) (string, error) {
c, err := RawdataToCid(codec, raw, mh)
if err != nil {
return "", err
}
dbKey := dshelp.MultihashToDsKey(c.Hash())
prefixedKey := blockstore.BlockPrefix.String() + dbKey.String()
_, err = tx.Exec(`INSERT INTO public.blocks (key, data) VALUES ($1, $2) ON CONFLICT (key) DO NOTHING`, prefixedKey, raw)
return c.String(), err
}
// RawdataToCid takes the desired codec, multihash type, and a slice of bytes
// and returns the proper cid of the object.
func RawdataToCid(codec uint64, rawdata []byte, multiHash uint64) (cid.Cid, error) {
c, err := cid.Prefix{
Codec: codec,
Version: 1,
MhType: multiHash,
MhLength: -1,
}.Sum(rawdata)
if err != nil {
return cid.Cid{}, err
}
return c, nil
}
// InitIPFSBlockService is used to configure and return a BlockService using an ipfs repo path (e.g. ~/.ipfs)
func InitIPFSBlockService(ipfsPath string) (blockservice.BlockService, error) {
r, openErr := fsrepo.Open(ipfsPath)
if openErr != nil {
return nil, openErr
}
ctx := context.Background()
cfg := &core.BuildCfg{
Online: false,
Repo: r,
}
ipfsNode, newNodeErr := core.NewNode(ctx, cfg)
if newNodeErr != nil {
return nil, newNodeErr
}
return ipfsNode.Blocks, nil
}
// ResetTestDB drops all rows in the test db public.blocks table
func ResetTestDB(db *sqlx.DB) error {
_, err := db.Exec("DELETE FROM public.blocks")
return err
}

30
pkg/validator.go
View File

@ -24,9 +24,11 @@ import (
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/trie"
"github.com/ipfs/go-blockservice"
"github.com/jmoiron/sqlx"
"github.com/vulcanize/pg-ipfs-ethdb"
"github.com/vulcanize/ipfs-ethdb"
"github.com/vulcanize/ipfs-ethdb/postgres"
)
// Validator is used for validating Ethereum state and storage tries on PG-IPFS
@ -36,12 +38,32 @@ type Validator struct {
stateDatabase state.Database
}
// NewPGIPFSValidator returns a new trie validator ontop of a connection pool for an IPFS backing Postgres database
func NewPGIPFSValidator(db *sqlx.DB) *Validator {
kvs := pgipfsethdb.NewKeyValueStore(db)
database := pgipfsethdb.NewDatabase(db)
return &Validator{
kvs: kvs,
trieDB: trie.NewDatabase(kvs),
stateDatabase: state.NewDatabase(database),
}
}
// NewIPFSValidator returns a new trie validator ontop of an IPFS blockservice
func NewIPFSValidator(bs blockservice.BlockService) *Validator {
kvs := ipfsethdb.NewKeyValueStore(bs)
database := ipfsethdb.NewDatabase(bs)
return &Validator{
kvs: kvs,
trieDB: trie.NewDatabase(kvs),
stateDatabase: state.NewDatabase(database),
}
}
// NewValidator returns a new trie validator
// Validating the completeness of a modified merkle patricia tries requires traversing the entire trie and verifying that
// every node is present, this is an expensive operation
func NewValidator(db *sqlx.DB) *Validator {
kvs := ipfsethdb.NewKeyValueStore(db)
database := ipfsethdb.NewDatabase(db)
func NewValidator(kvs ethdb.KeyValueStore, database ethdb.Database) *Validator {
return &Validator{
kvs: kvs,
trieDB: trie.NewDatabase(kvs),

299
pkg/validator_test.go Normal file
View File

@ -0,0 +1,299 @@
// VulcanizeDB
// Copyright © 2020 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package validator_test
import (
"math/big"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ipfs/go-cid/_rsrch/cidiface"
"github.com/jmoiron/sqlx"
"github.com/multiformats/go-multihash"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
"github.com/vulcanize/eth-ipfs-state-validator/pkg"
"github.com/vulcanize/ipfs-ethdb/postgres"
)
var (
contractAddr = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592")
slot0StorageValue = common.Hex2Bytes("94703c4b2bd70c169f5717101caee543299fc946c7")
slot1StorageValue = common.Hex2Bytes("01")
nullCodeHash = crypto.Keccak256Hash([]byte{})
emptyRootNode, _ = rlp.EncodeToBytes([]byte{})
emptyContractRoot = crypto.Keccak256Hash(emptyRootNode)
stateBranchRootNode, _ = rlp.EncodeToBytes([]interface{}{
crypto.Keccak256(bankAccountLeafNode),
[]byte{},
[]byte{},
[]byte{},
[]byte{},
crypto.Keccak256(minerAccountLeafNode),
crypto.Keccak256(contractAccountLeafNode),
[]byte{},
[]byte{},
[]byte{},
[]byte{},
[]byte{},
crypto.Keccak256(account2LeafNode),
[]byte{},
crypto.Keccak256(account1LeafNode),
[]byte{},
[]byte{},
})
stateRoot = crypto.Keccak256Hash(stateBranchRootNode)
contractAccount, _ = rlp.EncodeToBytes(state.Account{
Nonce: 1,
Balance: big.NewInt(0),
CodeHash: common.HexToHash("0xaaea5efba4fd7b45d7ec03918ac5d8b31aa93b48986af0e6b591f0f087c80127").Bytes(),
Root: crypto.Keccak256Hash(storageBranchRootNode),
})
contractAccountLeafNode, _ = rlp.EncodeToBytes([]interface{}{
common.Hex2Bytes("3114658a74d9cc9f7acf2c5cd696c3494d7c344d78bfec3add0d91ec4e8d1c45"),
contractAccount,
})
minerAccount, _ = rlp.EncodeToBytes(state.Account{
Nonce: 0,
Balance: big.NewInt(1000),
CodeHash: nullCodeHash.Bytes(),
Root: emptyContractRoot,
})
minerAccountLeafNode, _ = rlp.EncodeToBytes([]interface{}{
common.Hex2Bytes("3380c7b7ae81a58eb98d9c78de4a1fd7fd9535fc953ed2be602daaa41767312a"),
minerAccount,
})
account1, _ = rlp.EncodeToBytes(state.Account{
Nonce: 2,
Balance: big.NewInt(1000),
CodeHash: nullCodeHash.Bytes(),
Root: emptyContractRoot,
})
account1LeafNode, _ = rlp.EncodeToBytes([]interface{}{
common.Hex2Bytes("3926db69aaced518e9b9f0f434a473e7174109c943548bb8f23be41ca76d9ad2"),
account1,
})
account2, _ = rlp.EncodeToBytes(state.Account{
Nonce: 0,
Balance: big.NewInt(1000),
CodeHash: nullCodeHash.Bytes(),
Root: emptyContractRoot,
})
account2LeafNode, _ = rlp.EncodeToBytes([]interface{}{
common.Hex2Bytes("3957f3e2f04a0764c3a0491b175f69926da61efbcc8f61fa1455fd2d2b4cdd45"),
account2,
})
bankAccount, _ = rlp.EncodeToBytes(state.Account{
Nonce: 2,
Balance: big.NewInt(1000),
CodeHash: nullCodeHash.Bytes(),
Root: emptyContractRoot,
})
bankAccountLeafNode, _ = rlp.EncodeToBytes([]interface{}{
common.Hex2Bytes("30bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2a"),
bankAccount,
})
storageBranchRootNode, _ = rlp.EncodeToBytes([]interface{}{
[]byte{},
[]byte{},
crypto.Keccak256(slot0StorageLeafNode),
[]byte{},
[]byte{},
[]byte{},
[]byte{},
[]byte{},
[]byte{},
[]byte{},
[]byte{},
crypto.Keccak256(slot1StorageLeafNode),
[]byte{},
[]byte{},
[]byte{},
[]byte{},
[]byte{},
})
storageRoot = crypto.Keccak256Hash(storageBranchRootNode)
slot0StorageLeafNode, _ = rlp.EncodeToBytes([]interface{}{
common.Hex2Bytes("390decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563"),
slot0StorageValue,
})
slot1StorageLeafNode, _ = rlp.EncodeToBytes([]interface{}{
common.Hex2Bytes("310e2d527612073b26eecdfd717e6a320cf44b4afac2b0732d9fcbe2b7fa0cf6"),
slot1StorageValue,
})
trieStateNodes = [][]byte{
stateBranchRootNode,
bankAccountLeafNode,
minerAccountLeafNode,
contractAccountLeafNode,
account1LeafNode,
account2LeafNode,
}
trieStorageNodes = [][]byte{
storageBranchRootNode,
slot0StorageLeafNode,
slot1StorageLeafNode,
}
missingRootStateNodes = [][]byte{
bankAccountLeafNode,
minerAccountLeafNode,
contractAccountLeafNode,
account1LeafNode,
account2LeafNode,
}
missingRootStorageNodes = [][]byte{
slot0StorageLeafNode,
slot1StorageLeafNode,
}
missingNodeStateNodes = [][]byte{
stateBranchRootNode,
bankAccountLeafNode,
minerAccountLeafNode,
contractAccountLeafNode,
account2LeafNode,
}
missingNodeStorageNodes = [][]byte{
storageBranchRootNode,
slot1StorageLeafNode,
}
)
var (
v *validator.Validator
db *sqlx.DB
err error
)
var _ = Describe("PG-IPFS Validator", func() {
BeforeEach(func() {
db, err = pgipfsethdb.TestDB()
Expect(err).ToNot(HaveOccurred())
v = validator.NewPGIPFSValidator(db)
})
Describe("ValidateTrie", func() {
AfterEach(func() {
err = validator.ResetTestDB(db)
Expect(err).ToNot(HaveOccurred())
})
It("Returns an error the state root node is missing", func() {
loadTrie(missingRootStateNodes, trieStorageNodes)
err = v.ValidateTrie(stateRoot)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("does not exist in database"))
})
It("Fails to return an error if the storage root node is missing", func() {
// NOTE this failure was not expected and renders this approach unreliable, this is an issue with the go-ethereum core/state/iterator.NodeIterator
loadTrie(trieStateNodes, missingRootStorageNodes)
err = v.ValidateTrie(stateRoot)
Expect(err).ToNot(HaveOccurred())
})
It("Fails to return an error if the entire state (state trie and storage tries) cannot be validated", func() {
// NOTE this failure was not expected and renders this approach unreliable, this is an issue with the go-ethereum core/state/iterator.NodeIterator
loadTrie(missingNodeStateNodes, trieStorageNodes)
err = v.ValidateTrie(stateRoot)
Expect(err).ToNot(HaveOccurred())
})
It("Fails to return an error if the entire state (state trie and storage tries) cannot be validated", func() {
// NOTE this failure was not expected and renders this approach unreliable, this is an issue with the go-ethereum core/state/iterator.NodeIterator
loadTrie(trieStateNodes, missingNodeStorageNodes)
err = v.ValidateTrie(stateRoot)
Expect(err).ToNot(HaveOccurred())
})
It("Returns no error if the entire state (state trie and storage tries) can be validated", func() {
loadTrie(trieStateNodes, trieStorageNodes)
err = v.ValidateTrie(stateRoot)
Expect(err).ToNot(HaveOccurred())
})
})
Describe("ValidateStateTrie", func() {
AfterEach(func() {
err = validator.ResetTestDB(db)
Expect(err).ToNot(HaveOccurred())
})
It("Returns an error the state root node is missing", func() {
loadTrie(missingRootStateNodes, nil)
err = v.ValidateStateTrie(stateRoot)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("missing trie node"))
})
It("Returns an error if the entire state trie cannot be validated", func() {
loadTrie(missingNodeStateNodes, nil)
err = v.ValidateStateTrie(stateRoot)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("missing trie node"))
})
It("Returns no error if the entire state trie can be validated", func() {
loadTrie(trieStateNodes, nil)
err = v.ValidateStateTrie(stateRoot)
Expect(err).ToNot(HaveOccurred())
})
})
Describe("ValidateStorageTrie", func() {
AfterEach(func() {
err = validator.ResetTestDB(db)
Expect(err).ToNot(HaveOccurred())
})
It("Returns an error the storage root node is missing", func() {
loadTrie(nil, missingRootStorageNodes)
err = v.ValidateStorageTrie(contractAddr, storageRoot)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("missing trie node"))
})
It("Returns an error if the entire storage trie cannot be validated", func() {
loadTrie(nil, missingNodeStorageNodes)
err = v.ValidateStorageTrie(contractAddr, storageRoot)
Expect(err).To(HaveOccurred())
Expect(err.Error()).To(ContainSubstring("missing trie node"))
})
It("Returns no error if the entire storage trie can be validated", func() {
loadTrie(nil, trieStorageNodes)
err = v.ValidateStorageTrie(contractAddr, storageRoot)
Expect(err).ToNot(HaveOccurred())
})
})
})
func loadTrie(stateNodes, storageNodes [][]byte) {
tx, err := db.Beginx()
Expect(err).ToNot(HaveOccurred())
for _, node := range stateNodes {
_, err := validator.PublishRaw(tx, cid.EthStateTrie, multihash.KECCAK_256, node)
Expect(err).ToNot(HaveOccurred())
}
for _, node := range storageNodes {
_, err := validator.PublishRaw(tx, cid.EthStorageTrie, multihash.KECCAK_256, node)
Expect(err).ToNot(HaveOccurred())
}
err = tx.Commit()
Expect(err).ToNot(HaveOccurred())
}