Merge pull request #1 from vulcanize/validateTrie

Validate trie command

cmd/root.go

@@ -0,0 +1,97 @@
+// Copyright © 2020 Vulcanize, Inc
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"os"
+
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+var (
+	subCommand     string
+	logWithCommand logrus.Entry
+	rootStr        string
+	cfgFile        string
+)
+
+// rootCmd represents the base command when called without any subcommands
+var rootCmd = &cobra.Command{
+	Use:              "eth-ipfs-state-validator",
+	PersistentPreRun: initFuncs,
+}
+
+// Execute begins execution of the command
+func Execute() {
+	logrus.Info("----- Starting eth-ipfs-state-validator -----")
+	if err := rootCmd.Execute(); err != nil {
+		logrus.Fatal(err)
+	}
+}
+
+func initFuncs(cmd *cobra.Command, args []string) {
+	logfile := viper.GetString("logfile")
+	if logfile != "" {
+		file, err := os.OpenFile(logfile,
+			os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
+		if err == nil {
+			logrus.Infof("Directing output to %s", logfile)
+			logrus.SetOutput(file)
+		} else {
+			logrus.SetOutput(os.Stdout)
+			logrus.Info("Failed to log to file, using default stdout")
+		}
+	} else {
+		logrus.SetOutput(os.Stdout)
+	}
+	if err := logLevel(); err != nil {
+		logrus.Fatal("Could not set log level: ", err)
+	}
+}
+
+func logLevel() error {
+	lvl, err := logrus.ParseLevel(viper.GetString("log.level"))
+	if err != nil {
+		return err
+	}
+	logrus.SetLevel(lvl)
+	if lvl > logrus.InfoLevel {
+		logrus.SetReportCaller(true)
+	}
+	logrus.Info("Log level set to ", lvl.String())
+	return nil
+}
+
+func init() {
+	viper.AutomaticEnv()
+
+	rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file location")
+	rootCmd.PersistentFlags().String("logfile", "", "file path for logging")
+	rootCmd.PersistentFlags().String("database-name", "vulcanize_public", "database name")
+	rootCmd.PersistentFlags().Int("database-port", 5432, "database port")
+	rootCmd.PersistentFlags().String("database-hostname", "localhost", "database hostname")
+	rootCmd.PersistentFlags().String("database-user", "", "database user")
+	rootCmd.PersistentFlags().String("database-password", "", "database password")
+
+	viper.BindPFlag("logfile", rootCmd.PersistentFlags().Lookup("logfile"))
+	viper.BindPFlag("database.name", rootCmd.PersistentFlags().Lookup("database-name"))
+	viper.BindPFlag("database.port", rootCmd.PersistentFlags().Lookup("database-port"))
+	viper.BindPFlag("database.hostname", rootCmd.PersistentFlags().Lookup("database-hostname"))
+	viper.BindPFlag("database.user", rootCmd.PersistentFlags().Lookup("database-user"))
+	viper.BindPFlag("database.password", rootCmd.PersistentFlags().Lookup("database-password"))
+}

cmd/validateTrie.go

@@ -0,0 +1,58 @@
+// Copyright © 2020 Vulcanize, Inc
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/ethereum/go-ethereum/common"
+	_ "github.com/lib/pq" //postgres driver
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+
+	"github.com/vulcanize/eth-ipfs-state-validator/pkg"
+)
+
+// validateTrieCmd represents the validateTrie command
+var validateTrieCmd = &cobra.Command{
+	Use:   "validateTrie",
+	Short: "Validate completeness of state data on IPFS",
+	Long:  `This command is used to validate the completeness of the state trie corresponding to a specific state root`,
+	Run: func(cmd *cobra.Command, args []string) {
+		subCommand = cmd.CalledAs()
+		logWithCommand = *logrus.WithField("SubCommand", subCommand)
+		validateTrie()
+	},
+}
+
+func validateTrie() {
+	db, err := validator.NewDB()
+	if err != nil {
+		logWithCommand.Fatal(err)
+	}
+	v := validator.NewValidator(db)
+	rootHash := common.HexToHash(rootStr)
+	if _, err = v.ValidateTrie(rootHash); err != nil {
+		fmt.Printf("State trie is not complete\r\nerr: %v", err)
+		logWithCommand.Fatal(err)
+	}
+	fmt.Printf("State trie for root %s is complete", rootStr)
+}
+
+func init() {
+	rootCmd.AddCommand(validateTrieCmd)
+	validateTrieCmd.Flags().StringVarP(&rootStr, "root", "r", "", "Root of the state trie we wish to validate")
+}

go.mod

@@ -0,0 +1,16 @@
+module github.com/vulcanize/eth-ipfs-state-validator
+
+go 1.13
+
+require (
+	github.com/ethereum/go-ethereum v1.9.15
+	github.com/jmoiron/sqlx v1.2.0
+	github.com/lib/pq v1.5.2
+	github.com/onsi/ginkgo v1.12.1
+	github.com/onsi/gomega v1.10.1
+	github.com/sirupsen/logrus v1.6.0
+	github.com/spf13/cobra v1.0.0
+	github.com/spf13/viper v1.7.0
+	github.com/vulcanize/ipfs-blockchain-watcher v0.0.11-alpha
+	github.com/vulcanize/pg-ipfs-ethdb v0.0.2-alpha
+)

main.go

@@ -0,0 +1,29 @@
+// Copyright © 2020 Vulcanize, Inc
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+package main
+
+import (
+	"github.com/sirupsen/logrus"
+
+	"github.com/vulcanize/eth-ipfs-state-validator/cmd"
+)
+
+func main() {
+	logrus.SetFormatter(&logrus.TextFormatter{
+		FullTimestamp: true,
+	})
+	cmd.Execute()
+}

pkg/postgres.go

@@ -0,0 +1,32 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package validator
+
+import (
+	"github.com/jmoiron/sqlx"
+	_ "github.com/lib/pq" //postgres driver
+
+	"github.com/vulcanize/ipfs-blockchain-watcher/pkg/config"
+)
+
+// NewDB returns a new sqlx.DB from env variables
+func NewDB() (*sqlx.DB, error) {
+	c := config.Database{}
+	c.Init()
+	connectStr := config.DbConnectionString(c)
+	return sqlx.Connect("postgres", connectStr)
+}

pkg/validator.go

@@ -0,0 +1,68 @@
+// VulcanizeDB
+// Copyright © 2020 Vulcanize
+
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+package validator
+
+import (
+	"github.com/ethereum/go-ethereum/common"
+	"github.com/ethereum/go-ethereum/core/state"
+	"github.com/ethereum/go-ethereum/core/state/snapshot"
+	"github.com/ethereum/go-ethereum/ethdb"
+	"github.com/ethereum/go-ethereum/trie"
+	"github.com/jmoiron/sqlx"
+
+	"github.com/vulcanize/pg-ipfs-ethdb"
+)
+
+// Validator is used for validating Ethereum state and storage tries on PG-IPFS
+type Validator struct {
+	kvs           ethdb.KeyValueStore
+	trieDB        *trie.Database
+	stateDatabase state.Database
+}
+
+// NewValidator returns a new trie validator
+func NewValidator(db *sqlx.DB) *Validator {
+	kvs := ipfsethdb.NewKeyValueStore(db)
+	database := ipfsethdb.NewDatabase(db)
+	return &Validator{
+		kvs:           kvs,
+		trieDB:        trie.NewDatabase(kvs),
+		stateDatabase: state.NewDatabase(database),
+	}
+}
+
+// ValidateTrie returns whether or not the trie for the provided root hash is valid and complete
+// Validating the completeness of a modified merkle patricia trie requires traversing the entire trie and verifying that
+// every node is present, this is an expensive operation
+func (v *Validator) ValidateTrie(root common.Hash) (bool, error) {
+	// Generate the state.NodeIterator for this root
+	snapshotTree := snapshot.New(v.kvs, v.trieDB, 0, root, false)
+	stateDB, err := state.New(common.Hash{}, v.stateDatabase, snapshotTree)
+	if err != nil {
+		return false, err
+	}
+	it := state.NewNodeIterator(stateDB)
+	for it.Next() {
+		// iterate through entire trie
+		// it.Next() will return false when we have either completed iteration of the entire trie or have ran into an error
+		// if we are able to iterate through the entire trie without error then the trie is complete
+	}
+	if it.Error != nil {
+		return false, it.Error
+	}
+	return true, nil
+}