Merge pull request #2 from vulcanize/validateTrie
Different modes; fix for geth 1.9.15 compatibility
This commit is contained in:
Ian Norden
GitHub
commit
d5495e9648
43
README.md
43
README.md
@ -1,15 +1,35 @@
|
||||
# eth-ipfs-state-validator
|
||||
|
||||
Uses [pg-ipfs-ethdb](https://github.com/vulcanize/pg-ipfs-ethdb) to validate completeness of Ethereum state data on PG-IPFS
|
||||
Uses [pg-ipfs-ethdb](https://github.com/vulcanize/pg-ipfs-ethdb) to validate completeness of Ethereum state data on IPFS
|
||||
|
||||
## Background
|
||||
|
||||
State data on Ethereum takes the form of [Modified Merkle Patricia Tries](https://eth.wiki/en/fundamentals/patricia-tree).
|
||||
On disk each unique node of a trie is stored as a key-value pair between the Keccak256 hash of the RLP-encoded node and the RLP-encoded node.
|
||||
To prove the existence of a specific node in an MMPT with a known root hash, one provides a list of all of the nodes along the path descending
|
||||
from the root node to the node in question. To validate the completeness of a state database- to confirm every node for a state and/or storage trie(s) is present
|
||||
in a database- requires traversing the entire trie (or linked set of tries) and confirming the presence of every node in the database.
|
||||
|
||||
|
||||
## Usage
|
||||
|
||||
Run
|
||||
|
||||
`./eth-ipfs-state-validator validateTrie --root={state root string} --config={path to .toml config file} `
|
||||
`full` validates completeness of the entire state corresponding to a provided state root, including both state and storage tries
|
||||
|
||||
With `root` as the state root hash we want to validate the corresponding trie for.
|
||||
The config file holds the parameters for connecting to the IPFS-backing Postgres database.
|
||||
`./eth-ipfs-state-validator validateTrie --config={path to db config} --type=full --state-root={state root hex string}`
|
||||
|
||||
|
||||
`state` validates completeness of the state trie corresponding to a provided state root, excluding the storage tries
|
||||
|
||||
`./eth-ipfs-state-validator validateTrie --config={path to db config} --type=state --state-root={state root hex string}`
|
||||
|
||||
|
||||
`storage` validates completeness of only the storage trie corresponding to a provided storage root and contract address
|
||||
|
||||
`./eth-ipfs-state-validator validateTrie --config={path to db config} --type=storage --storage-root={state root hex string} --address={contract address hex string}`
|
||||
|
||||
|
||||
The config file holds the parameters for connecting to an [IPFS-backing Postgres database](https://github.com/ipfs/go-ds-sql).
|
||||
|
||||
```toml
|
||||
[database]
|
||||
@ -19,3 +39,16 @@ The config file holds the parameters for connecting to the IPFS-backing Postgres
|
||||
password = ""
|
||||
port = 5432
|
||||
```
|
||||
|
||||
## Maintainers
|
||||
@vulcanize
|
||||
@AFDudley
|
||||
@i-norden
|
||||
|
||||
## Contributing
|
||||
Contributions are welcome!
|
||||
|
||||
VulcanizeDB follows the [Contributor Covenant Code of Conduct](https://www.contributor-covenant.org/version/1/4/code-of-conduct).
|
||||
|
||||
## License
|
||||
[AGPL-3.0](LICENSE) © Vulcanize Inc
|
||||
13
cmd/root.go
13
cmd/root.go
@ -24,10 +24,13 @@ import (
|
||||
)
|
||||
|
||||
var (
|
||||
subCommand string
|
||||
logWithCommand logrus.Entry
|
||||
rootStr string
|
||||
cfgFile string
|
||||
subCommand string
|
||||
logWithCommand logrus.Entry
|
||||
stateRootStr string
|
||||
storageRootStr string
|
||||
validationType string
|
||||
contractAddrStr string
|
||||
cfgFile string
|
||||
)
|
||||
|
||||
// rootCmd represents the base command when called without any subcommands
|
||||
@ -87,6 +90,7 @@ func init() {
|
||||
rootCmd.PersistentFlags().String("database-hostname", "localhost", "database hostname")
|
||||
rootCmd.PersistentFlags().String("database-user", "", "database user")
|
||||
rootCmd.PersistentFlags().String("database-password", "", "database password")
|
||||
rootCmd.PersistentFlags().String("log-level", logrus.InfoLevel.String(), "Log level (trace, debug, info, warn, error, fatal, panic")
|
||||
|
||||
viper.BindPFlag("logfile", rootCmd.PersistentFlags().Lookup("logfile"))
|
||||
viper.BindPFlag("database.name", rootCmd.PersistentFlags().Lookup("database-name"))
|
||||
@ -94,4 +98,5 @@ func init() {
|
||||
viper.BindPFlag("database.hostname", rootCmd.PersistentFlags().Lookup("database-hostname"))
|
||||
viper.BindPFlag("database.user", rootCmd.PersistentFlags().Lookup("database-user"))
|
||||
viper.BindPFlag("database.password", rootCmd.PersistentFlags().Lookup("database-password"))
|
||||
viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log-level"))
|
||||
}
|
||||
|
||||
@ -16,7 +16,7 @@
|
||||
package cmd
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
_ "github.com/lib/pq" //postgres driver
|
||||
@ -30,7 +30,24 @@ import (
|
||||
var validateTrieCmd = &cobra.Command{
|
||||
Use: "validateTrie",
|
||||
Short: "Validate completeness of state data on IPFS",
|
||||
Long: `This command is used to validate the completeness of the state trie corresponding to a specific state root`,
|
||||
Long: `This command is used to validate the completeness of state data corresponding specific to a specific root
|
||||
|
||||
It can operate at three levels:
|
||||
|
||||
"full" validates completeness of the entire state corresponding to a provided state root, including both state and storage tries
|
||||
|
||||
./eth-ipfs-state-validator validateTrie --config={path to db config} --type=full --state-root={state root hex string}
|
||||
|
||||
|
||||
"state" validates completeness of the state trie corresponding to a provided state root, excluding the storage tries
|
||||
|
||||
./eth-ipfs-state-validator validateTrie --config={path to db config} --type=state --state-root={state root hex string}
|
||||
|
||||
|
||||
"storage" validates completeness of only the storage trie corresponding to a provided storage root and contract address
|
||||
|
||||
./eth-ipfs-state-validator validateTrie --config={path to db config} --type=storage --storage-root={state root hex string} --address={contract address hex string}
|
||||
"`,
|
||||
Run: func(cmd *cobra.Command, args []string) {
|
||||
subCommand = cmd.CalledAs()
|
||||
logWithCommand = *logrus.WithField("SubCommand", subCommand)
|
||||
@ -44,15 +61,45 @@ func validateTrie() {
|
||||
logWithCommand.Fatal(err)
|
||||
}
|
||||
v := validator.NewValidator(db)
|
||||
rootHash := common.HexToHash(rootStr)
|
||||
if _, err = v.ValidateTrie(rootHash); err != nil {
|
||||
fmt.Printf("State trie is not complete\r\nerr: %v", err)
|
||||
logWithCommand.Fatal(err)
|
||||
switch strings.ToLower(validationType) {
|
||||
case "f", "full":
|
||||
if stateRootStr == "" {
|
||||
logWithCommand.Fatal("must provide a state root for full state validation")
|
||||
}
|
||||
stateRoot := common.HexToHash(stateRootStr)
|
||||
if err = v.ValidateTrie(stateRoot); err != nil {
|
||||
logWithCommand.Fatalf("State for root %s is not complete\r\nerr: %v", stateRoot.String(), err)
|
||||
}
|
||||
logWithCommand.Infof("State for root %s is complete", stateRoot.String())
|
||||
case "state":
|
||||
if stateRootStr == "" {
|
||||
logWithCommand.Fatal("must provide a state root for state trie validation")
|
||||
}
|
||||
stateRoot := common.HexToHash(stateRootStr)
|
||||
if err = v.ValidateStateTrie(stateRoot); err != nil {
|
||||
logWithCommand.Fatalf("State trie for root %s is not complete\r\nerr: %v", stateRoot.String(), err)
|
||||
}
|
||||
logWithCommand.Infof("State trie for root %s is complete", stateRoot.String())
|
||||
case "storage":
|
||||
if storageRootStr == "" {
|
||||
logWithCommand.Fatal("must provide a storage root for storage trie validation")
|
||||
}
|
||||
if contractAddrStr == "" {
|
||||
logWithCommand.Fatal("must provide a contract address for storage trie validation")
|
||||
}
|
||||
storageRoot := common.HexToHash(storageRootStr)
|
||||
addr := common.HexToAddress(contractAddrStr)
|
||||
if err = v.ValidateStorageTrie(addr, storageRoot); err != nil {
|
||||
logWithCommand.Fatalf("Storage trie for contract %s and root %s not complete\r\nerr: %v", addr.String(), storageRoot.String(), err)
|
||||
}
|
||||
logWithCommand.Infof("Storage trie for contract %s and root %s is complete", addr.String(), storageRoot.String())
|
||||
}
|
||||
fmt.Printf("State trie for root %s is complete", rootStr)
|
||||
}
|
||||
|
||||
func init() {
|
||||
rootCmd.AddCommand(validateTrieCmd)
|
||||
validateTrieCmd.Flags().StringVarP(&rootStr, "root", "r", "", "Root of the state trie we wish to validate")
|
||||
validateTrieCmd.Flags().StringVarP(&stateRootStr, "state-root", "s", "", "Root of the state trie we wish to validate; for full or state validation")
|
||||
validateTrieCmd.Flags().StringVarP(&validationType, "type", "t", "full", "Type of validations: full, state, storage")
|
||||
validateTrieCmd.Flags().StringVarP(&storageRootStr, "storage-root", "o", "", "Root of the storage trie we wish to validate; for storage validation")
|
||||
validateTrieCmd.Flags().StringVarP(&contractAddrStr, "address", "a", "", "Contract address for the storage trie we wish to validate; for storage validation")
|
||||
}
|
||||
|
||||
6
environments/example.toml
Normal file
6
environments/example.toml
Normal file
@ -0,0 +1,6 @@
|
||||
[database]
|
||||
name = "vulcanize_public" # $DATABASE_NAME
|
||||
hostname = "localhost" # $DATABASE_HOSTNAME
|
||||
port = 5432 # $DATABASE_PORT
|
||||
user = "postgres" # $DATABASE_USER
|
||||
password = "" # $DATABASE_PASSWORD
|
||||
4
environments/testing.toml
Normal file
4
environments/testing.toml
Normal file
@ -0,0 +1,4 @@
|
||||
[database]
|
||||
name = "vulcanize_testing"
|
||||
hostname = "localhost"
|
||||
port = 5432
|
||||
1
go.mod
1
go.mod
@ -11,6 +11,5 @@ require (
|
||||
github.com/sirupsen/logrus v1.6.0
|
||||
github.com/spf13/cobra v1.0.0
|
||||
github.com/spf13/viper v1.7.0
|
||||
github.com/vulcanize/ipfs-blockchain-watcher v0.0.11-alpha
|
||||
github.com/vulcanize/pg-ipfs-ethdb v0.0.2-alpha
|
||||
)
|
||||
|
||||
74
pkg/database.go
Normal file
74
pkg/database.go
Normal file
@ -0,0 +1,74 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2019 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package validator
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/jmoiron/sqlx"
|
||||
"github.com/spf13/viper"
|
||||
)
|
||||
|
||||
// Env variables
|
||||
const (
|
||||
DATABASE_NAME = "DATABASE_NAME"
|
||||
DATABASE_HOSTNAME = "DATABASE_HOSTNAME"
|
||||
DATABASE_PORT = "DATABASE_PORT"
|
||||
DATABASE_USER = "DATABASE_USER"
|
||||
DATABASE_PASSWORD = "DATABASE_PASSWORD"
|
||||
)
|
||||
|
||||
// NewDB returns a new sqlx.DB from config/cli/env variables
|
||||
func NewDB() (*sqlx.DB, error) {
|
||||
c := Config{}
|
||||
c.Init()
|
||||
return sqlx.Connect("postgres", c.ConnString())
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
Hostname string
|
||||
Name string
|
||||
User string
|
||||
Password string
|
||||
Port int
|
||||
}
|
||||
|
||||
func (c *Config) ConnString() string {
|
||||
if len(c.User) > 0 && len(c.Password) > 0 {
|
||||
return fmt.Sprintf("postgresql://%s:%s@%s:%d/%s?sslmode=disable",
|
||||
c.User, c.Password, c.Hostname, c.Port, c.Name)
|
||||
}
|
||||
if len(c.User) > 0 && len(c.Password) == 0 {
|
||||
return fmt.Sprintf("postgresql://%s@%s:%d/%s?sslmode=disable",
|
||||
c.User, c.Hostname, c.Port, c.Name)
|
||||
}
|
||||
return fmt.Sprintf("postgresql://%s:%d/%s?sslmode=disable", c.Hostname, c.Port, c.Name)
|
||||
}
|
||||
|
||||
func (c *Config) Init() {
|
||||
viper.BindEnv("database.name", DATABASE_NAME)
|
||||
viper.BindEnv("database.hostname", DATABASE_HOSTNAME)
|
||||
viper.BindEnv("database.port", DATABASE_PORT)
|
||||
viper.BindEnv("database.user", DATABASE_USER)
|
||||
viper.BindEnv("database.password", DATABASE_PASSWORD)
|
||||
|
||||
c.Name = viper.GetString("database.name")
|
||||
c.Hostname = viper.GetString("database.hostname")
|
||||
c.Port = viper.GetInt("database.port")
|
||||
c.User = viper.GetString("database.user")
|
||||
c.Password = viper.GetString("database.password")
|
||||
}
|
||||
@ -1,32 +0,0 @@
|
||||
// VulcanizeDB
|
||||
// Copyright © 2020 Vulcanize
|
||||
|
||||
// This program is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Affero General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
|
||||
// This program is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Affero General Public License for more details.
|
||||
|
||||
// You should have received a copy of the GNU Affero General Public License
|
||||
// along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package validator
|
||||
|
||||
import (
|
||||
"github.com/jmoiron/sqlx"
|
||||
_ "github.com/lib/pq" //postgres driver
|
||||
|
||||
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/config"
|
||||
)
|
||||
|
||||
// NewDB returns a new sqlx.DB from env variables
|
||||
func NewDB() (*sqlx.DB, error) {
|
||||
c := config.Database{}
|
||||
c.Init()
|
||||
connectStr := config.DbConnectionString(c)
|
||||
return sqlx.Connect("postgres", connectStr)
|
||||
}
|
||||
@ -17,9 +17,11 @@
|
||||
package validator
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/core/state"
|
||||
"github.com/ethereum/go-ethereum/core/state/snapshot"
|
||||
"github.com/ethereum/go-ethereum/crypto"
|
||||
"github.com/ethereum/go-ethereum/ethdb"
|
||||
"github.com/ethereum/go-ethereum/trie"
|
||||
"github.com/jmoiron/sqlx"
|
||||
@ -35,6 +37,8 @@ type Validator struct {
|
||||
}
|
||||
|
||||
// NewValidator returns a new trie validator
|
||||
// Validating the completeness of a modified merkle patricia tries requires traversing the entire trie and verifying that
|
||||
// every node is present, this is an expensive operation
|
||||
func NewValidator(db *sqlx.DB) *Validator {
|
||||
kvs := ipfsethdb.NewKeyValueStore(db)
|
||||
database := ipfsethdb.NewDatabase(db)
|
||||
@ -45,24 +49,62 @@ func NewValidator(db *sqlx.DB) *Validator {
|
||||
}
|
||||
}
|
||||
|
||||
// ValidateTrie returns whether or not the trie for the provided root hash is valid and complete
|
||||
// Validating the completeness of a modified merkle patricia trie requires traversing the entire trie and verifying that
|
||||
// every node is present, this is an expensive operation
|
||||
func (v *Validator) ValidateTrie(root common.Hash) (bool, error) {
|
||||
// ValidateTrie returns an error if the state and storage tries for the provided state root cannot be confirmed as complete
|
||||
// This does consider child storage tries
|
||||
func (v *Validator) ValidateTrie(stateRoot common.Hash) error {
|
||||
// Generate the state.NodeIterator for this root
|
||||
snapshotTree := snapshot.New(v.kvs, v.trieDB, 0, root, false)
|
||||
stateDB, err := state.New(common.Hash{}, v.stateDatabase, snapshotTree)
|
||||
stateDB, err := state.New(common.Hash{}, v.stateDatabase, nil)
|
||||
if err != nil {
|
||||
return false, err
|
||||
return err
|
||||
}
|
||||
it := state.NewNodeIterator(stateDB)
|
||||
// state.NodeIterator won't throw an error if we can't find the root node
|
||||
// check if it exists first
|
||||
exists, err := v.kvs.Has(stateRoot.Bytes())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if !exists {
|
||||
return fmt.Errorf("root node for hash %s does not exist in database", stateRoot.Hex())
|
||||
}
|
||||
for it.Next() {
|
||||
// iterate through entire trie
|
||||
// it.Next() will return false when we have either completed iteration of the entire trie or have ran into an error
|
||||
// iterate through entire state trie and descendent storage tries
|
||||
// it.Next() will return false when we have either completed iteration of the entire trie or have ran into an error (e.g. a missing node)
|
||||
// if we are able to iterate through the entire trie without error then the trie is complete
|
||||
}
|
||||
if it.Error != nil {
|
||||
return false, it.Error
|
||||
}
|
||||
return true, nil
|
||||
return it.Error
|
||||
}
|
||||
|
||||
// ValidateStateTrie returns an error if the state trie for the provided state root cannot be confirmed as complete
|
||||
// This does not consider child storage tries
|
||||
func (v *Validator) ValidateStateTrie(stateRoot common.Hash) error {
|
||||
// Generate the trie.NodeIterator for this root
|
||||
t, err := v.stateDatabase.OpenTrie(stateRoot)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
it := t.NodeIterator(nil)
|
||||
for it.Next(true) {
|
||||
// iterate through entire state trie
|
||||
// it.Next() will return false when we have either completed iteration of the entire trie or have ran into an error (e.g. a missing node)
|
||||
// if we are able to iterate through the entire trie without error then the trie is complete
|
||||
}
|
||||
return it.Error()
|
||||
}
|
||||
|
||||
// ValidateStorageTrie returns an error if the storage trie for the provided storage root and contract address cannot be confirmed as complete
|
||||
func (v *Validator) ValidateStorageTrie(address common.Address, storageRoot common.Hash) error {
|
||||
// Generate the state.NodeIterator for this root
|
||||
addrHash := crypto.Keccak256Hash(address.Bytes())
|
||||
t, err := v.stateDatabase.OpenStorageTrie(addrHash, storageRoot)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
it := t.NodeIterator(nil)
|
||||
for it.Next(true) {
|
||||
// iterate through entire storage trie
|
||||
// it.Next() will return false when we have either completed iteration of the entire trie or have ran into an error (e.g. a missing node)
|
||||
// if we are able to iterate through the entire trie without error then the trie is complete
|
||||
}
|
||||
return it.Error()
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user