Merge pull request #2 from vulcanize/validateTrie

Different modes; fix for geth 1.9.15 compatibility

README.md

@@ -1,15 +1,35 @@
 # eth-ipfs-state-validator
 
-Uses [pg-ipfs-ethdb](https://github.com/vulcanize/pg-ipfs-ethdb) to validate completeness of Ethereum state data on PG-IPFS
+Uses [pg-ipfs-ethdb](https://github.com/vulcanize/pg-ipfs-ethdb) to validate completeness of Ethereum state data on IPFS
+
+## Background
+
+State data on Ethereum takes the form of [Modified Merkle Patricia Tries](https://eth.wiki/en/fundamentals/patricia-tree).
+On disk each unique node of a trie is stored as a key-value pair between the Keccak256 hash of the RLP-encoded node and the RLP-encoded node.
+To prove the existence of a specific node in an MMPT with a known root hash, one provides a list of all of the nodes along the path descending
+from the root node to the node in question. To validate the completeness of a state database- to confirm every node for a state and/or storage trie(s) is present
+in a database- requires traversing the entire trie (or linked set of tries) and confirming the presence of every node in the database.
+
 
 ## Usage
 
-Run
 
-`./eth-ipfs-state-validator validateTrie --root={state root string} --config={path to .toml config file} `
+`full` validates completeness of the entire state corresponding to a provided state root, including both state and storage tries
 
-With `root` as the state root hash we want to validate the corresponding trie for.
+`./eth-ipfs-state-validator validateTrie --config={path to db config} --type=full --state-root={state root hex string}`
-The config file holds the parameters for connecting to the IPFS-backing Postgres database.
+
+
+`state` validates completeness of the state trie corresponding to a provided state root, excluding the storage tries
+
+`./eth-ipfs-state-validator validateTrie --config={path to db config} --type=state --state-root={state root hex string}`
+
+
+`storage` validates completeness of only the storage trie corresponding to a provided storage root and contract address
+
+`./eth-ipfs-state-validator validateTrie --config={path to db config} --type=storage --storage-root={state root hex string} --address={contract address hex string}`
+
+
+The config file holds the parameters for connecting to an [IPFS-backing Postgres database](https://github.com/ipfs/go-ds-sql).
 
 ```toml
 [database]
@@ -18,4 +38,17 @@ The config file holds the parameters for connecting to the IPFS-backing Postgres
     user     = "postgres"
     password = ""
     port     = 5432
-```
+```
+
+## Maintainers
+@vulcanize
+@AFDudley
+@i-norden
+
+## Contributing
+Contributions are welcome!
+
+VulcanizeDB follows the [Contributor Covenant Code of Conduct](https://www.contributor-covenant.org/version/1/4/code-of-conduct).
+
+## License
+[AGPL-3.0](LICENSE) © Vulcanize Inc

cmd/root.go

@@ -24,10 +24,13 @@ import (
 )
 
 var (
-	subCommand     string
+	subCommand      string
-	logWithCommand logrus.Entry
+	logWithCommand  logrus.Entry
-	rootStr        string
+	stateRootStr    string
-	cfgFile        string
+	storageRootStr  string
+	validationType  string
+	contractAddrStr string
+	cfgFile         string
 )
 
 // rootCmd represents the base command when called without any subcommands
@@ -87,6 +90,7 @@ func init() {
 	rootCmd.PersistentFlags().String("database-hostname", "localhost", "database hostname")
 	rootCmd.PersistentFlags().String("database-user", "", "database user")
 	rootCmd.PersistentFlags().String("database-password", "", "database password")
+	rootCmd.PersistentFlags().String("log-level", logrus.InfoLevel.String(), "Log level (trace, debug, info, warn, error, fatal, panic")
 
 	viper.BindPFlag("logfile", rootCmd.PersistentFlags().Lookup("logfile"))
 	viper.BindPFlag("database.name", rootCmd.PersistentFlags().Lookup("database-name"))
@@ -94,4 +98,5 @@ func init() {
 	viper.BindPFlag("database.hostname", rootCmd.PersistentFlags().Lookup("database-hostname"))
 	viper.BindPFlag("database.user", rootCmd.PersistentFlags().Lookup("database-user"))
 	viper.BindPFlag("database.password", rootCmd.PersistentFlags().Lookup("database-password"))
+	viper.BindPFlag("log.level", rootCmd.PersistentFlags().Lookup("log-level"))
 }

cmd/validateTrie.go

@@ -16,7 +16,7 @@
 package cmd
 
 import (
-	"fmt"
+	"strings"
 
 	"github.com/ethereum/go-ethereum/common"
 	_ "github.com/lib/pq" //postgres driver
@@ -30,7 +30,24 @@ import (
 var validateTrieCmd = &cobra.Command{
 	Use:   "validateTrie",
 	Short: "Validate completeness of state data on IPFS",
-	Long:  `This command is used to validate the completeness of the state trie corresponding to a specific state root`,
+	Long: `This command is used to validate the completeness of state data corresponding specific to a specific root
+
+It can operate at three levels: 
+
+"full" validates completeness of the entire state corresponding to a provided state root, including both state and storage tries
+
+./eth-ipfs-state-validator validateTrie --config={path to db config} --type=full --state-root={state root hex string}
+
+
+"state" validates completeness of the state trie corresponding to a provided state root, excluding the storage tries
+
+./eth-ipfs-state-validator validateTrie --config={path to db config} --type=state --state-root={state root hex string}
+
+
+"storage" validates completeness of only the storage trie corresponding to a provided storage root and contract address
+
+./eth-ipfs-state-validator validateTrie --config={path to db config} --type=storage --storage-root={state root hex string} --address={contract address hex string}
+"`,
 	Run: func(cmd *cobra.Command, args []string) {
 		subCommand = cmd.CalledAs()
 		logWithCommand = *logrus.WithField("SubCommand", subCommand)
@@ -44,15 +61,45 @@ func validateTrie() {
 		logWithCommand.Fatal(err)
 	}
 	v := validator.NewValidator(db)
-	rootHash := common.HexToHash(rootStr)
+	switch strings.ToLower(validationType) {
-	if _, err = v.ValidateTrie(rootHash); err != nil {
+	case "f", "full":
-		fmt.Printf("State trie is not complete\r\nerr: %v", err)
+		if stateRootStr == "" {
-		logWithCommand.Fatal(err)
+			logWithCommand.Fatal("must provide a state root for full state validation")
+		}
+		stateRoot := common.HexToHash(stateRootStr)
+		if err = v.ValidateTrie(stateRoot); err != nil {
+			logWithCommand.Fatalf("State for root %s is not complete\r\nerr: %v", stateRoot.String(), err)
+		}
+		logWithCommand.Infof("State for root %s is complete", stateRoot.String())
+	case "state":
+		if stateRootStr == "" {
+			logWithCommand.Fatal("must provide a state root for state trie validation")
+		}
+		stateRoot := common.HexToHash(stateRootStr)
+		if err = v.ValidateStateTrie(stateRoot); err != nil {
+			logWithCommand.Fatalf("State trie for root %s is not complete\r\nerr: %v", stateRoot.String(), err)
+		}
+		logWithCommand.Infof("State trie for root %s is complete", stateRoot.String())
+	case "storage":
+		if storageRootStr == "" {
+			logWithCommand.Fatal("must provide a storage root for storage trie validation")
+		}
+		if contractAddrStr == "" {
+			logWithCommand.Fatal("must provide a contract address for storage trie validation")
+		}
+		storageRoot := common.HexToHash(storageRootStr)
+		addr := common.HexToAddress(contractAddrStr)
+		if err = v.ValidateStorageTrie(addr, storageRoot); err != nil {
+			logWithCommand.Fatalf("Storage trie for contract %s and root %s not complete\r\nerr: %v", addr.String(), storageRoot.String(), err)
+		}
+		logWithCommand.Infof("Storage trie for contract %s and root %s is complete", addr.String(), storageRoot.String())
 	}
-	fmt.Printf("State trie for root %s is complete", rootStr)
 }
 
 func init() {
 	rootCmd.AddCommand(validateTrieCmd)
-	validateTrieCmd.Flags().StringVarP(&rootStr, "root", "r", "", "Root of the state trie we wish to validate")
+	validateTrieCmd.Flags().StringVarP(&stateRootStr, "state-root", "s", "", "Root of the state trie we wish to validate; for full or state validation")
+	validateTrieCmd.Flags().StringVarP(&validationType, "type", "t", "full", "Type of validations: full, state, storage")
+	validateTrieCmd.Flags().StringVarP(&storageRootStr, "storage-root", "o", "", "Root of the storage trie we wish to validate; for storage validation")
+	validateTrieCmd.Flags().StringVarP(&contractAddrStr, "address", "a", "", "Contract address for the storage trie we wish to validate; for storage validation")
 }

environments/example.toml

@@ -0,0 +1,6 @@
+[database]
+    name     = "vulcanize_public" # $DATABASE_NAME
+    hostname = "localhost" # $DATABASE_HOSTNAME
+    port     = 5432 # $DATABASE_PORT
+    user     = "postgres" # $DATABASE_USER
+    password = "" # $DATABASE_PASSWORD

environments/testing.toml

@@ -0,0 +1,4 @@
+[database]
+  name     = "vulcanize_testing"
+  hostname = "localhost"
+  port     = 5432

go.mod

@@ -11,6 +11,5 @@ require (
 	github.com/sirupsen/logrus v1.6.0
 	github.com/spf13/cobra v1.0.0
 	github.com/spf13/viper v1.7.0
-	github.com/vulcanize/ipfs-blockchain-watcher v0.0.11-alpha
 	github.com/vulcanize/pg-ipfs-ethdb v0.0.2-alpha
 )

pkg/database.go

@@ -0,0 +1,74 @@
+// VulcanizeDB
+// Copyright © 2019 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package validator
+
+import (
+	"fmt"
+
+	"github.com/jmoiron/sqlx"
+	"github.com/spf13/viper"
+)
+
+// Env variables
+const (
+	DATABASE_NAME     = "DATABASE_NAME"
+	DATABASE_HOSTNAME = "DATABASE_HOSTNAME"
+	DATABASE_PORT     = "DATABASE_PORT"
+	DATABASE_USER     = "DATABASE_USER"
+	DATABASE_PASSWORD = "DATABASE_PASSWORD"
+)
+
+// NewDB returns a new sqlx.DB from config/cli/env variables
+func NewDB() (*sqlx.DB, error) {
+	c := Config{}
+	c.Init()
+	return sqlx.Connect("postgres", c.ConnString())
+}
+
+type Config struct {
+	Hostname string
+	Name     string
+	User     string
+	Password string
+	Port     int
+}
+
+func (c *Config) ConnString() string {
+	if len(c.User) > 0 && len(c.Password) > 0 {
+		return fmt.Sprintf("postgresql://%s:%s@%s:%d/%s?sslmode=disable",
+			c.User, c.Password, c.Hostname, c.Port, c.Name)
+	}
+	if len(c.User) > 0 && len(c.Password) == 0 {
+		return fmt.Sprintf("postgresql://%s@%s:%d/%s?sslmode=disable",
+			c.User, c.Hostname, c.Port, c.Name)
+	}
+	return fmt.Sprintf("postgresql://%s:%d/%s?sslmode=disable", c.Hostname, c.Port, c.Name)
+}
+
+func (c *Config) Init() {
+	viper.BindEnv("database.name", DATABASE_NAME)
+	viper.BindEnv("database.hostname", DATABASE_HOSTNAME)
+	viper.BindEnv("database.port", DATABASE_PORT)
+	viper.BindEnv("database.user", DATABASE_USER)
+	viper.BindEnv("database.password", DATABASE_PASSWORD)
+
+	c.Name = viper.GetString("database.name")
+	c.Hostname = viper.GetString("database.hostname")
+	c.Port = viper.GetInt("database.port")
+	c.User = viper.GetString("database.user")
+	c.Password = viper.GetString("database.password")
+}

pkg/postgres.go

@@ -1,32 +0,0 @@
-// VulcanizeDB
-// Copyright © 2020 Vulcanize
-
-// This program is free software: you can redistribute it and/or modify
-// it under the terms of the GNU Affero General Public License as published by
-// the Free Software Foundation, either version 3 of the License, or
-// (at your option) any later version.
-
-// This program is distributed in the hope that it will be useful,
-// but WITHOUT ANY WARRANTY; without even the implied warranty of
-// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-// GNU Affero General Public License for more details.
-
-// You should have received a copy of the GNU Affero General Public License
-// along with this program.  If not, see <http://www.gnu.org/licenses/>.
-
-package validator
-
-import (
-	"github.com/jmoiron/sqlx"
-	_ "github.com/lib/pq" //postgres driver
-
-	"github.com/vulcanize/ipfs-blockchain-watcher/pkg/config"
-)
-
-// NewDB returns a new sqlx.DB from env variables
-func NewDB() (*sqlx.DB, error) {
-	c := config.Database{}
-	c.Init()
-	connectStr := config.DbConnectionString(c)
-	return sqlx.Connect("postgres", connectStr)
-}

pkg/validator.go

@@ -17,9 +17,11 @@
 package validator
 
 import (
+	"fmt"
+
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/state"
-	"github.com/ethereum/go-ethereum/core/state/snapshot"
+	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/trie"
 	"github.com/jmoiron/sqlx"
@@ -35,6 +37,8 @@ type Validator struct {
 }
 
 // NewValidator returns a new trie validator
+// Validating the completeness of a modified merkle patricia tries requires traversing the entire trie and verifying that
+// every node is present, this is an expensive operation
 func NewValidator(db *sqlx.DB) *Validator {
 	kvs := ipfsethdb.NewKeyValueStore(db)
 	database := ipfsethdb.NewDatabase(db)
@@ -45,24 +49,62 @@ func NewValidator(db *sqlx.DB) *Validator {
 	}
 }
 
-// ValidateTrie returns whether or not the trie for the provided root hash is valid and complete
+// ValidateTrie returns an error if the state and storage tries for the provided state root cannot be confirmed as complete
-// Validating the completeness of a modified merkle patricia trie requires traversing the entire trie and verifying that
+// This does consider child storage tries
-// every node is present, this is an expensive operation
+func (v *Validator) ValidateTrie(stateRoot common.Hash) error {
-func (v *Validator) ValidateTrie(root common.Hash) (bool, error) {
 	// Generate the state.NodeIterator for this root
-	snapshotTree := snapshot.New(v.kvs, v.trieDB, 0, root, false)
+	stateDB, err := state.New(common.Hash{}, v.stateDatabase, nil)
-	stateDB, err := state.New(common.Hash{}, v.stateDatabase, snapshotTree)
 	if err != nil {
-		return false, err
+		return err
 	}
 	it := state.NewNodeIterator(stateDB)
+	// state.NodeIterator won't throw an error if we can't find the root node
+	// check if it exists first
+	exists, err := v.kvs.Has(stateRoot.Bytes())
+	if err != nil {
+		return err
+	}
+	if !exists {
+		return fmt.Errorf("root node for hash %s does not exist in database", stateRoot.Hex())
+	}
 	for it.Next() {
-		// iterate through entire trie
+		// iterate through entire state trie and descendent storage tries
-		// it.Next() will return false when we have either completed iteration of the entire trie or have ran into an error
+		// it.Next() will return false when we have either completed iteration of the entire trie or have ran into an error (e.g. a missing node)
 		// if we are able to iterate through the entire trie without error then the trie is complete
 	}
-	if it.Error != nil {
+	return it.Error
-		return false, it.Error
+}
-	}
+
-	return true, nil
+// ValidateStateTrie returns an error if the state trie for the provided state root cannot be confirmed as complete
+// This does not consider child storage tries
+func (v *Validator) ValidateStateTrie(stateRoot common.Hash) error {
+	// Generate the trie.NodeIterator for this root
+	t, err := v.stateDatabase.OpenTrie(stateRoot)
+	if err != nil {
+		return err
+	}
+	it := t.NodeIterator(nil)
+	for it.Next(true) {
+		// iterate through entire state trie
+		// it.Next() will return false when we have either completed iteration of the entire trie or have ran into an error (e.g. a missing node)
+		// if we are able to iterate through the entire trie without error then the trie is complete
+	}
+	return it.Error()
+}
+
+// ValidateStorageTrie returns an error if the storage trie for the provided storage root and contract address cannot be confirmed as complete
+func (v *Validator) ValidateStorageTrie(address common.Address, storageRoot common.Hash) error {
+	// Generate the state.NodeIterator for this root
+	addrHash := crypto.Keccak256Hash(address.Bytes())
+	t, err := v.stateDatabase.OpenStorageTrie(addrHash, storageRoot)
+	if err != nil {
+		return err
+	}
+	it := t.NodeIterator(nil)
+	for it.Next(true) {
+		// iterate through entire storage trie
+		// it.Next() will return false when we have either completed iteration of the entire trie or have ran into an error (e.g. a missing node)
+		// if we are able to iterate through the entire trie without error then the trie is complete
+	}
+	return it.Error()
 }