cerc-io/eth-ipfs-state-validator
16
0
Fork 1
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #2 from vulcanize/validateTrie

Browse Source 
Different modes; fix for geth 1.9.15 compatibility
This commit is contained in:
Ian Norden 2020-06-29 19:40:36 -05:00 committed by GitHub
commit d5495e9648
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 257 additions and 788 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

45
README.md
View File

@ -1,15 +1,35 @@
# eth-ipfs-state-validator
Uses [pg-ipfs-ethdb](https://github.com/vulcanize/pg-ipfs-ethdb) to validate completeness of Ethereum state data on PG-IPFS
Uses [pg-ipfs-ethdb](https://github.com/vulcanize/pg-ipfs-ethdb) to validate completeness of Ethereum state data on IPFS
## Background
State data on Ethereum takes the form of [Modified Merkle Patricia Tries](https://eth.wiki/en/fundamentals/patricia-tree).
On disk each unique node of a trie is stored as a key-value pair between the Keccak256 hash of the RLP-encoded node and the RLP-encoded node.
To prove the existence of a specific node in an MMPT with a known root hash, one provides a list of all of the nodes along the path descending
from the root node to the node in question. To validate the completeness of a state database- to confirm every node for a state and/or storage trie(s) is present
in a database- requires traversing the entire trie (or linked set of tries) and confirming the presence of every node in the database.
## Usage
Run
`./eth-ipfs-state-validator validateTrie --root={state root string} --config={path to .toml config file} `
`full` validates completeness of the entire state corresponding to a provided state root, including both state and storage tries
With `root` as the state root hash we want to validate the corresponding trie for.
The config file holds the parameters for connecting to the IPFS-backing Postgres database.
`./eth-ipfs-state-validator validateTrie --config={path to db config} --type=full --state-root={state root hex string}`
`state` validates completeness of the state trie corresponding to a provided state root, excluding the storage tries
`./eth-ipfs-state-validator validateTrie --config={path to db config} --type=state --state-root={state root hex string}`
`storage` validates completeness of only the storage trie corresponding to a provided storage root and contract address
`./eth-ipfs-state-validator validateTrie --config={path to db config} --type=storage --storage-root={state root hex string} --address={contract address hex string}`
The config file holds the parameters for connecting to an [IPFS-backing Postgres database](https://github.com/ipfs/go-ds-sql).
```toml
[database]
@ -18,4 +38,17 @@ The config file holds the parameters for connecting to the IPFS-backing Postgres
user = "postgres"
password = ""
port = 5432
```
```
## Maintainers
@vulcanize
@AFDudley
@i-norden
## Contributing
Contributions are welcome!
VulcanizeDB follows the [Contributor Covenant Code of Conduct](https://www.contributor-covenant.org/version/1/4/code-of-conduct).
## License
[AGPL-3.0](LICENSE) © Vulcanize Inc

13
cmd/root.go
View File

@ -24,10 +24,13 @@ import (
)
var (
subCommand string
logWithCommand logrus.Entry
rootStr string
cfgFile string
subCommand string
logWithCommand logrus.Entry
stateRootStr string
storageRootStr string
validationType string
contractAddrStr string
cfgFile string
)
// rootCmd represents the base command when called without any subcommands
@ -87,6 +90,7 @@ func init() {
rootCmd.PersistentFlags().String("database-hostname", "localhost", "database hostname")
rootCmd.PersistentFlags().String("database-user", "", "database user")
rootCmd.PersistentFlags().String("database-password", "", "database password")
rootCmd.PersistentFlags().String("log-level", logrus.InfoLevel.String(), "Log level (trace, debug, info, warn, error, fatal, panic")
viper.BindPFlag("logfile", rootCmd.PersistentFlags().Lookup("logfile"))
viper.BindPFlag("database.name", rootCmd.PersistentFlags().Lookup("database-name"))
@ -94,4 +98,5 @@ func init() {
viper.BindPFlag("database.hostname", rootCmd.PersistentFlags().Lookup("database-hostname"))
viper.BindPFlag("database.user", rootCmd.PersistentFlags().Lookup("database-user"))
viper.BindPFlag("database.password", rootCmd.PersistentFlags().Lookup("database-password"))
viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log-level"))
}

63
cmd/validateTrie.go
View File

@ -16,7 +16,7 @@
package cmd
import (
"fmt"
"strings"
"github.com/ethereum/go-ethereum/common"
_ "github.com/lib/pq" //postgres driver
@ -30,7 +30,24 @@ import (
var validateTrieCmd = &cobra.Command{
Use: "validateTrie",
Short: "Validate completeness of state data on IPFS",
Long: `This command is used to validate the completeness of the state trie corresponding to a specific state root`,
Long: `This command is used to validate the completeness of state data corresponding specific to a specific root
It can operate at three levels:
"full" validates completeness of the entire state corresponding to a provided state root, including both state and storage tries
./eth-ipfs-state-validator validateTrie --config={path to db config} --type=full --state-root={state root hex string}
"state" validates completeness of the state trie corresponding to a provided state root, excluding the storage tries
./eth-ipfs-state-validator validateTrie --config={path to db config} --type=state --state-root={state root hex string}
"storage" validates completeness of only the storage trie corresponding to a provided storage root and contract address
./eth-ipfs-state-validator validateTrie --config={path to db config} --type=storage --storage-root={state root hex string} --address={contract address hex string}
"`,
Run: func(cmd *cobra.Command, args []string) {
subCommand = cmd.CalledAs()
logWithCommand = *logrus.WithField("SubCommand", subCommand)
@ -44,15 +61,45 @@ func validateTrie() {
logWithCommand.Fatal(err)
}
v := validator.NewValidator(db)
rootHash := common.HexToHash(rootStr)
if _, err = v.ValidateTrie(rootHash); err != nil {
fmt.Printf("State trie is not complete\r\nerr: %v", err)
logWithCommand.Fatal(err)
switch strings.ToLower(validationType) {
case "f", "full":
if stateRootStr == "" {
logWithCommand.Fatal("must provide a state root for full state validation")
}
stateRoot := common.HexToHash(stateRootStr)
if err = v.ValidateTrie(stateRoot); err != nil {
logWithCommand.Fatalf("State for root %s is not complete\r\nerr: %v", stateRoot.String(), err)
}
logWithCommand.Infof("State for root %s is complete", stateRoot.String())
case "state":
if stateRootStr == "" {
logWithCommand.Fatal("must provide a state root for state trie validation")
}
stateRoot := common.HexToHash(stateRootStr)
if err = v.ValidateStateTrie(stateRoot); err != nil {
logWithCommand.Fatalf("State trie for root %s is not complete\r\nerr: %v", stateRoot.String(), err)
}
logWithCommand.Infof("State trie for root %s is complete", stateRoot.String())
case "storage":
if storageRootStr == "" {
logWithCommand.Fatal("must provide a storage root for storage trie validation")
}
if contractAddrStr == "" {
logWithCommand.Fatal("must provide a contract address for storage trie validation")
}
storageRoot := common.HexToHash(storageRootStr)
addr := common.HexToAddress(contractAddrStr)
if err = v.ValidateStorageTrie(addr, storageRoot); err != nil {
logWithCommand.Fatalf("Storage trie for contract %s and root %s not complete\r\nerr: %v", addr.String(), storageRoot.String(), err)
}
logWithCommand.Infof("Storage trie for contract %s and root %s is complete", addr.String(), storageRoot.String())
}
fmt.Printf("State trie for root %s is complete", rootStr)
}
func init() {
rootCmd.AddCommand(validateTrieCmd)
validateTrieCmd.Flags().StringVarP(&rootStr, "root", "r", "", "Root of the state trie we wish to validate")
validateTrieCmd.Flags().StringVarP(&stateRootStr, "state-root", "s", "", "Root of the state trie we wish to validate; for full or state validation")
validateTrieCmd.Flags().StringVarP(&validationType, "type", "t", "full", "Type of validations: full, state, storage")
validateTrieCmd.Flags().StringVarP(&storageRootStr, "storage-root", "o", "", "Root of the storage trie we wish to validate; for storage validation")
validateTrieCmd.Flags().StringVarP(&contractAddrStr, "address", "a", "", "Contract address for the storage trie we wish to validate; for storage validation")
}

6
environments/example.toml Normal file
View File

@ -0,0 +1,6 @@
[database]
name = "vulcanize_public" # $DATABASE_NAME
hostname = "localhost" # $DATABASE_HOSTNAME
port = 5432 # $DATABASE_PORT
user = "postgres" # $DATABASE_USER
password = "" # $DATABASE_PASSWORD

4
environments/testing.toml Normal file
View File

@ -0,0 +1,4 @@
[database]
name = "vulcanize_testing"
hostname = "localhost"
port = 5432

1
go.mod
View File

@ -11,6 +11,5 @@ require (
github.com/sirupsen/logrus v1.6.0
github.com/spf13/cobra v1.0.0
github.com/spf13/viper v1.7.0
github.com/vulcanize/ipfs-blockchain-watcher v0.0.11-alpha
github.com/vulcanize/pg-ipfs-ethdb v0.0.2-alpha
)

737
go.sum
View File

File diff suppressed because it is too large Load Diff

74
pkg/database.go Normal file
View File

@ -0,0 +1,74 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package validator
import (
"fmt"
"github.com/jmoiron/sqlx"
"github.com/spf13/viper"
)
// Env variables
const (
DATABASE_NAME = "DATABASE_NAME"
DATABASE_HOSTNAME = "DATABASE_HOSTNAME"
DATABASE_PORT = "DATABASE_PORT"
DATABASE_USER = "DATABASE_USER"
DATABASE_PASSWORD = "DATABASE_PASSWORD"
)
// NewDB returns a new sqlx.DB from config/cli/env variables
func NewDB() (*sqlx.DB, error) {
c := Config{}
c.Init()
return sqlx.Connect("postgres", c.ConnString())
}
type Config struct {
Hostname string
Name string
User string
Password string
Port int
}
func (c *Config) ConnString() string {
if len(c.User) > 0 && len(c.Password) > 0 {
return fmt.Sprintf("postgresql://%s:%s@%s:%d/%s?sslmode=disable",
c.User, c.Password, c.Hostname, c.Port, c.Name)
}
if len(c.User) > 0 && len(c.Password) == 0 {
return fmt.Sprintf("postgresql://%s@%s:%d/%s?sslmode=disable",
c.User, c.Hostname, c.Port, c.Name)
}
return fmt.Sprintf("postgresql://%s:%d/%s?sslmode=disable", c.Hostname, c.Port, c.Name)
}
func (c *Config) Init() {
viper.BindEnv("database.name", DATABASE_NAME)
viper.BindEnv("database.hostname", DATABASE_HOSTNAME)
viper.BindEnv("database.port", DATABASE_PORT)
viper.BindEnv("database.user", DATABASE_USER)
viper.BindEnv("database.password", DATABASE_PASSWORD)
c.Name = viper.GetString("database.name")
c.Hostname = viper.GetString("database.hostname")
c.Port = viper.GetInt("database.port")
c.User = viper.GetString("database.user")
c.Password = viper.GetString("database.password")
}

32
pkg/postgres.go
View File

@ -1,32 +0,0 @@
// VulcanizeDB
// Copyright © 2020 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package validator
import (
"github.com/jmoiron/sqlx"
_ "github.com/lib/pq" //postgres driver
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/config"
)
// NewDB returns a new sqlx.DB from env variables
func NewDB() (*sqlx.DB, error) {
c := config.Database{}
c.Init()
connectStr := config.DbConnectionString(c)
return sqlx.Connect("postgres", connectStr)
}

70
pkg/validator.go
View File

@ -17,9 +17,11 @@
package validator
import (
"fmt"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/core/state/snapshot"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/trie"
"github.com/jmoiron/sqlx"
@ -35,6 +37,8 @@ type Validator struct {
}
// NewValidator returns a new trie validator
// Validating the completeness of a modified merkle patricia tries requires traversing the entire trie and verifying that
// every node is present, this is an expensive operation
func NewValidator(db *sqlx.DB) *Validator {
kvs := ipfsethdb.NewKeyValueStore(db)
database := ipfsethdb.NewDatabase(db)
@ -45,24 +49,62 @@ func NewValidator(db *sqlx.DB) *Validator {
}
}
// ValidateTrie returns whether or not the trie for the provided root hash is valid and complete
// Validating the completeness of a modified merkle patricia trie requires traversing the entire trie and verifying that
// every node is present, this is an expensive operation
func (v *Validator) ValidateTrie(root common.Hash) (bool, error) {
// ValidateTrie returns an error if the state and storage tries for the provided state root cannot be confirmed as complete
// This does consider child storage tries
func (v *Validator) ValidateTrie(stateRoot common.Hash) error {
// Generate the state.NodeIterator for this root
snapshotTree := snapshot.New(v.kvs, v.trieDB, 0, root, false)
stateDB, err := state.New(common.Hash{}, v.stateDatabase, snapshotTree)
stateDB, err := state.New(common.Hash{}, v.stateDatabase, nil)
if err != nil {
return false, err
return err
}
it := state.NewNodeIterator(stateDB)
// state.NodeIterator won't throw an error if we can't find the root node
// check if it exists first
exists, err := v.kvs.Has(stateRoot.Bytes())
if err != nil {
return err
}
if !exists {
return fmt.Errorf("root node for hash %s does not exist in database", stateRoot.Hex())
}
for it.Next() {
// iterate through entire trie
// it.Next() will return false when we have either completed iteration of the entire trie or have ran into an error
// iterate through entire state trie and descendent storage tries
// it.Next() will return false when we have either completed iteration of the entire trie or have ran into an error (e.g. a missing node)
// if we are able to iterate through the entire trie without error then the trie is complete
}
if it.Error != nil {
return false, it.Error
}
return true, nil
return it.Error
}
// ValidateStateTrie returns an error if the state trie for the provided state root cannot be confirmed as complete
// This does not consider child storage tries
func (v *Validator) ValidateStateTrie(stateRoot common.Hash) error {
// Generate the trie.NodeIterator for this root
t, err := v.stateDatabase.OpenTrie(stateRoot)
if err != nil {
return err
}
it := t.NodeIterator(nil)
for it.Next(true) {
// iterate through entire state trie
// it.Next() will return false when we have either completed iteration of the entire trie or have ran into an error (e.g. a missing node)
// if we are able to iterate through the entire trie without error then the trie is complete
}
return it.Error()
}
// ValidateStorageTrie returns an error if the storage trie for the provided storage root and contract address cannot be confirmed as complete
func (v *Validator) ValidateStorageTrie(address common.Address, storageRoot common.Hash) error {
// Generate the state.NodeIterator for this root
addrHash := crypto.Keccak256Hash(address.Bytes())
t, err := v.stateDatabase.OpenStorageTrie(addrHash, storageRoot)
if err != nil {
return err
}
it := t.NodeIterator(nil)
for it.Next(true) {
// iterate through entire storage trie
// it.Next() will return false when we have either completed iteration of the entire trie or have ran into an error (e.g. a missing node)
// if we are able to iterate through the entire trie without error then the trie is complete
}
return it.Error()
}