Merge pull request #1 from vulcanize/validateTrie

Validate trie command
This commit is contained in:
Ian Norden 2020-06-25 15:29:53 -05:00 committed by GitHub
commit 97d7b6e590
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 1519 additions and 0 deletions

97
cmd/root.go Normal file
View File

@ -0,0 +1,97 @@
// Copyright © 2020 Vulcanize, Inc
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"os"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/spf13/viper"
)
var (
subCommand string
logWithCommand logrus.Entry
rootStr string
cfgFile string
)
// rootCmd represents the base command when called without any subcommands
var rootCmd = &cobra.Command{
Use: "eth-ipfs-state-validator",
PersistentPreRun: initFuncs,
}
// Execute begins execution of the command
func Execute() {
logrus.Info("----- Starting eth-ipfs-state-validator -----")
if err := rootCmd.Execute(); err != nil {
logrus.Fatal(err)
}
}
func initFuncs(cmd *cobra.Command, args []string) {
logfile := viper.GetString("logfile")
if logfile != "" {
file, err := os.OpenFile(logfile,
os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)
if err == nil {
logrus.Infof("Directing output to %s", logfile)
logrus.SetOutput(file)
} else {
logrus.SetOutput(os.Stdout)
logrus.Info("Failed to log to file, using default stdout")
}
} else {
logrus.SetOutput(os.Stdout)
}
if err := logLevel(); err != nil {
logrus.Fatal("Could not set log level: ", err)
}
}
func logLevel() error {
lvl, err := logrus.ParseLevel(viper.GetString("log.level"))
if err != nil {
return err
}
logrus.SetLevel(lvl)
if lvl > logrus.InfoLevel {
logrus.SetReportCaller(true)
}
logrus.Info("Log level set to ", lvl.String())
return nil
}
func init() {
viper.AutomaticEnv()
rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file location")
rootCmd.PersistentFlags().String("logfile", "", "file path for logging")
rootCmd.PersistentFlags().String("database-name", "vulcanize_public", "database name")
rootCmd.PersistentFlags().Int("database-port", 5432, "database port")
rootCmd.PersistentFlags().String("database-hostname", "localhost", "database hostname")
rootCmd.PersistentFlags().String("database-user", "", "database user")
rootCmd.PersistentFlags().String("database-password", "", "database password")
viper.BindPFlag("logfile", rootCmd.PersistentFlags().Lookup("logfile"))
viper.BindPFlag("database.name", rootCmd.PersistentFlags().Lookup("database-name"))
viper.BindPFlag("database.port", rootCmd.PersistentFlags().Lookup("database-port"))
viper.BindPFlag("database.hostname", rootCmd.PersistentFlags().Lookup("database-hostname"))
viper.BindPFlag("database.user", rootCmd.PersistentFlags().Lookup("database-user"))
viper.BindPFlag("database.password", rootCmd.PersistentFlags().Lookup("database-password"))
}

58
cmd/validateTrie.go Normal file
View File

@ -0,0 +1,58 @@
// Copyright © 2020 Vulcanize, Inc
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package cmd
import (
"fmt"
"github.com/ethereum/go-ethereum/common"
_ "github.com/lib/pq" //postgres driver
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"
"github.com/vulcanize/eth-ipfs-state-validator/pkg"
)
// validateTrieCmd represents the validateTrie command
var validateTrieCmd = &cobra.Command{
Use: "validateTrie",
Short: "Validate completeness of state data on IPFS",
Long: `This command is used to validate the completeness of the state trie corresponding to a specific state root`,
Run: func(cmd *cobra.Command, args []string) {
subCommand = cmd.CalledAs()
logWithCommand = *logrus.WithField("SubCommand", subCommand)
validateTrie()
},
}
func validateTrie() {
db, err := validator.NewDB()
if err != nil {
logWithCommand.Fatal(err)
}
v := validator.NewValidator(db)
rootHash := common.HexToHash(rootStr)
if _, err = v.ValidateTrie(rootHash); err != nil {
fmt.Printf("State trie is not complete\r\nerr: %v", err)
logWithCommand.Fatal(err)
}
fmt.Printf("State trie for root %s is complete", rootStr)
}
func init() {
rootCmd.AddCommand(validateTrieCmd)
validateTrieCmd.Flags().StringVarP(&rootStr, "root", "r", "", "Root of the state trie we wish to validate")
}

16
go.mod Normal file
View File

@ -0,0 +1,16 @@
module github.com/vulcanize/eth-ipfs-state-validator
go 1.13
require (
github.com/ethereum/go-ethereum v1.9.15
github.com/jmoiron/sqlx v1.2.0
github.com/lib/pq v1.5.2
github.com/onsi/ginkgo v1.12.1
github.com/onsi/gomega v1.10.1
github.com/sirupsen/logrus v1.6.0
github.com/spf13/cobra v1.0.0
github.com/spf13/viper v1.7.0
github.com/vulcanize/ipfs-blockchain-watcher v0.0.11-alpha
github.com/vulcanize/pg-ipfs-ethdb v0.0.2-alpha
)

1219
go.sum Normal file

File diff suppressed because it is too large Load Diff

29
main.go Normal file
View File

@ -0,0 +1,29 @@
// Copyright © 2020 Vulcanize, Inc
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package main
import (
"github.com/sirupsen/logrus"
"github.com/vulcanize/eth-ipfs-state-validator/cmd"
)
func main() {
logrus.SetFormatter(&logrus.TextFormatter{
FullTimestamp: true,
})
cmd.Execute()
}

32
pkg/postgres.go Normal file
View File

@ -0,0 +1,32 @@
// VulcanizeDB
// Copyright © 2020 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package validator
import (
"github.com/jmoiron/sqlx"
_ "github.com/lib/pq" //postgres driver
"github.com/vulcanize/ipfs-blockchain-watcher/pkg/config"
)
// NewDB returns a new sqlx.DB from env variables
func NewDB() (*sqlx.DB, error) {
c := config.Database{}
c.Init()
connectStr := config.DbConnectionString(c)
return sqlx.Connect("postgres", connectStr)
}

68
pkg/validator.go Normal file
View File

@ -0,0 +1,68 @@
// VulcanizeDB
// Copyright © 2020 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package validator
import (
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/state"
"github.com/ethereum/go-ethereum/core/state/snapshot"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/trie"
"github.com/jmoiron/sqlx"
"github.com/vulcanize/pg-ipfs-ethdb"
)
// Validator is used for validating Ethereum state and storage tries on PG-IPFS
type Validator struct {
kvs ethdb.KeyValueStore
trieDB *trie.Database
stateDatabase state.Database
}
// NewValidator returns a new trie validator
func NewValidator(db *sqlx.DB) *Validator {
kvs := ipfsethdb.NewKeyValueStore(db)
database := ipfsethdb.NewDatabase(db)
return &Validator{
kvs: kvs,
trieDB: trie.NewDatabase(kvs),
stateDatabase: state.NewDatabase(database),
}
}
// ValidateTrie returns whether or not the trie for the provided root hash is valid and complete
// Validating the completeness of a modified merkle patricia trie requires traversing the entire trie and verifying that
// every node is present, this is an expensive operation
func (v *Validator) ValidateTrie(root common.Hash) (bool, error) {
// Generate the state.NodeIterator for this root
snapshotTree := snapshot.New(v.kvs, v.trieDB, 0, root, false)
stateDB, err := state.New(common.Hash{}, v.stateDatabase, snapshotTree)
if err != nil {
return false, err
}
it := state.NewNodeIterator(stateDB)
for it.Next() {
// iterate through entire trie
// it.Next() will return false when we have either completed iteration of the entire trie or have ran into an error
// if we are able to iterate through the entire trie without error then the trie is complete
}
if it.Error != nil {
return false, it.Error
}
return true, nil
}