ipfs blockservice based validator

This commit is contained in:
Ian Norden 2020-07-12 19:57:47 -05:00
parent cad3bb3fb6
commit 4f089a7a1a
9 changed files with 797 additions and 40 deletions

View File

@ -18,20 +18,27 @@ in a database- requires traversing the entire trie (or linked set of tries) and
`full` validates completeness of the entire state corresponding to a provided state root, including both state and storage tries `full` validates completeness of the entire state corresponding to a provided state root, including both state and storage tries
`./eth-ipfs-state-validator validateTrie --config={path to db config} --type=full --state-root={state root hex string}` `./eth-ipfs-state-validator validateTrie --ipfs-path={path to ipfs repo} --type=full --state-root={state root hex string}`
`state` validates completeness of the state trie corresponding to a provided state root, excluding the storage tries `state` validates completeness of the state trie corresponding to a provided state root, excluding the storage tries
`./eth-ipfs-state-validator validateTrie --config={path to db config} --type=state --state-root={state root hex string}` `./eth-ipfs-state-validator validateTrie --ipfs-path={path to ipfs repo} --type=state --state-root={state root hex string}`
`storage` validates completeness of only the storage trie corresponding to a provided storage root and contract address `storage` validates completeness of only the storage trie corresponding to a provided storage root and contract address
`./eth-ipfs-state-validator validateTrie --config={path to db config} --type=storage --storage-root={state root hex string} --address={contract address hex string}` `./eth-ipfs-state-validator validateTrie --ipfs-path={path to ipfs repo} --type=storage --storage-root={state root hex string} --address={contract address hex string}`
The config file holds the parameters for connecting to an [IPFS-backing Postgres database](https://github.com/ipfs/go-ds-sql). If an IPFS path is provided with the `--ipfs-path` flag, the validator operates through an IPFS block-service and expects a configured IPFS repository at
the provided path. In this case, the validator will vie for contention on the lockfile located at the ipfs path.
Alternatively, if no IPFS path is provided, the `--config` flag can be used to provide a path to a .toml config file with
Postgres database connection parameters. In this case, the validator interfaces directly with the Postgres database and the
database is assumed to be [IPFS-backing](https://github.com/ipfs/go-ds-sql).
Postgres DB config:
```toml ```toml
[database] [database]

View File

@ -31,6 +31,7 @@ var (
validationType string validationType string
contractAddrStr string contractAddrStr string
cfgFile string cfgFile string
ipfsPath string
) )
// rootCmd represents the base command when called without any subcommands // rootCmd represents the base command when called without any subcommands

View File

@ -32,6 +32,8 @@ var validateTrieCmd = &cobra.Command{
Short: "Validate completeness of state data on IPFS", Short: "Validate completeness of state data on IPFS",
Long: `This command is used to validate the completeness of state data corresponding specific to a specific root Long: `This command is used to validate the completeness of state data corresponding specific to a specific root
If an ipfs-path is provided it will use a blockservice, otherwise it expects Postgres db configuration in a linked config file.
It can operate at three levels: It can operate at three levels:
"full" validates completeness of the entire state corresponding to a provided state root, including both state and storage tries "full" validates completeness of the entire state corresponding to a provided state root, including both state and storage tries
@ -56,11 +58,10 @@ It can operate at three levels:
} }
func validateTrie() { func validateTrie() {
db, err := validator.NewDB() v, err := newValidator()
if err != nil { if err != nil {
logWithCommand.Fatal(err) logWithCommand.Fatal(err)
} }
v := validator.NewValidator(db)
switch strings.ToLower(validationType) { switch strings.ToLower(validationType) {
case "f", "full": case "f", "full":
if stateRootStr == "" { if stateRootStr == "" {
@ -96,10 +97,26 @@ func validateTrie() {
} }
} }
func newValidator() (*validator.Validator, error) {
if ipfsPath == "" {
db, err := validator.NewDB()
if err != nil {
logWithCommand.Fatal(err)
}
return validator.NewPGIPFSValidator(db), nil
}
bs, err := validator.InitIPFSBlockService(ipfsPath)
if err != nil {
return nil, err
}
return validator.NewIPFSValidator(bs), nil
}
func init() { func init() {
rootCmd.AddCommand(validateTrieCmd) rootCmd.AddCommand(validateTrieCmd)
validateTrieCmd.Flags().StringVarP(&stateRootStr, "state-root", "s", "", "Root of the state trie we wish to validate; for full or state validation") validateTrieCmd.Flags().StringVarP(&stateRootStr, "state-root", "s", "", "Root of the state trie we wish to validate; for full or state validation")
validateTrieCmd.Flags().StringVarP(&validationType, "type", "t", "full", "Type of validations: full, state, storage") validateTrieCmd.Flags().StringVarP(&validationType, "type", "t", "full", "Type of validations: full, state, storage")
validateTrieCmd.Flags().StringVarP(&storageRootStr, "storage-root", "o", "", "Root of the storage trie we wish to validate; for storage validation") validateTrieCmd.Flags().StringVarP(&storageRootStr, "storage-root", "o", "", "Root of the storage trie we wish to validate; for storage validation")
validateTrieCmd.Flags().StringVarP(&contractAddrStr, "address", "a", "", "Contract address for the storage trie we wish to validate; for storage validation") validateTrieCmd.Flags().StringVarP(&contractAddrStr, "address", "a", "", "Contract address for the storage trie we wish to validate; for storage validation")
validateTrieCmd.Flags().StringVarP(&ipfsPath, "ipfs-path", "i", "", "Path to IPFS repository")
} }

8
go.mod
View File

@ -4,9 +4,15 @@ go 1.13
require ( require (
github.com/ethereum/go-ethereum v1.9.15 github.com/ethereum/go-ethereum v1.9.15
github.com/hashicorp/golang-lru v0.5.4
github.com/ipfs/go-block-format v0.0.2
github.com/ipfs/go-blockservice v0.1.3
github.com/ipfs/go-cid v0.0.5 github.com/ipfs/go-cid v0.0.5
github.com/ipfs/go-filestore v1.0.0 //indirect
github.com/ipfs/go-ipfs v0.5.1
github.com/ipfs/go-ipfs-blockstore v1.0.0 github.com/ipfs/go-ipfs-blockstore v1.0.0
github.com/ipfs/go-ipfs-ds-help v1.0.0 github.com/ipfs/go-ipfs-ds-help v1.0.0
github.com/ipfs/go-ipfs-exchange-interface v0.0.1
github.com/jmoiron/sqlx v1.2.0 github.com/jmoiron/sqlx v1.2.0
github.com/lib/pq v1.5.2 github.com/lib/pq v1.5.2
github.com/multiformats/go-multihash v0.0.13 github.com/multiformats/go-multihash v0.0.13
@ -15,5 +21,5 @@ require (
github.com/sirupsen/logrus v1.6.0 github.com/sirupsen/logrus v1.6.0
github.com/spf13/cobra v1.0.0 github.com/spf13/cobra v1.0.0
github.com/spf13/viper v1.7.0 github.com/spf13/viper v1.7.0
github.com/vulcanize/pg-ipfs-ethdb v0.0.2-alpha github.com/vulcanize/ipfs-ethdb v0.0.2
) )

708
go.sum

File diff suppressed because it is too large Load Diff

View File

@ -1,5 +1,5 @@
// VulcanizeDB // VulcanizeDB
// Copyright © 2019 Vulcanize // Copyright © 2020 Vulcanize
// This program is free software: you can redistribute it and/or modify // This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by // it under the terms of the GNU Affero General Public License as published by

View File

@ -17,9 +17,14 @@
package validator package validator
import ( import (
"context"
"github.com/ipfs/go-blockservice"
"github.com/ipfs/go-cid" "github.com/ipfs/go-cid"
"github.com/ipfs/go-ipfs-blockstore" "github.com/ipfs/go-ipfs-blockstore"
"github.com/ipfs/go-ipfs-ds-help" "github.com/ipfs/go-ipfs-ds-help"
"github.com/ipfs/go-ipfs/core"
"github.com/ipfs/go-ipfs/repo/fsrepo"
"github.com/jmoiron/sqlx" "github.com/jmoiron/sqlx"
) )
@ -49,3 +54,27 @@ func RawdataToCid(codec uint64, rawdata []byte, multiHash uint64) (cid.Cid, erro
} }
return c, nil return c, nil
} }
// InitIPFSBlockService is used to configure and return a BlockService using an ipfs repo path (e.g. ~/.ipfs)
func InitIPFSBlockService(ipfsPath string) (blockservice.BlockService, error) {
r, openErr := fsrepo.Open(ipfsPath)
if openErr != nil {
return nil, openErr
}
ctx := context.Background()
cfg := &core.BuildCfg{
Online: false,
Repo: r,
}
ipfsNode, newNodeErr := core.NewNode(ctx, cfg)
if newNodeErr != nil {
return nil, newNodeErr
}
return ipfsNode.Blocks, nil
}
// ResetTestDB drops all rows in the test db public.blocks table
func ResetTestDB(db *sqlx.DB) error {
_, err := db.Exec("DELETE FROM public.blocks")
return err
}

View File

@ -24,9 +24,11 @@ import (
"github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie"
"github.com/ipfs/go-blockservice"
"github.com/jmoiron/sqlx" "github.com/jmoiron/sqlx"
pgipfsethdb "github.com/vulcanize/pg-ipfs-ethdb/postgres" "github.com/vulcanize/ipfs-ethdb"
"github.com/vulcanize/ipfs-ethdb/postgres"
) )
// Validator is used for validating Ethereum state and storage tries on PG-IPFS // Validator is used for validating Ethereum state and storage tries on PG-IPFS
@ -36,12 +38,32 @@ type Validator struct {
stateDatabase state.Database stateDatabase state.Database
} }
// NewPGIPFSValidator returns a new trie validator ontop of a connection pool for an IPFS backing Postgres database
func NewPGIPFSValidator(db *sqlx.DB) *Validator {
kvs := pgipfsethdb.NewKeyValueStore(db)
database := pgipfsethdb.NewDatabase(db)
return &Validator{
kvs: kvs,
trieDB: trie.NewDatabase(kvs),
stateDatabase: state.NewDatabase(database),
}
}
// NewIPFSValidator returns a new trie validator ontop of an IPFS blockservice
func NewIPFSValidator(bs blockservice.BlockService) *Validator {
kvs := ipfsethdb.NewKeyValueStore(bs)
database := ipfsethdb.NewDatabase(bs)
return &Validator{
kvs: kvs,
trieDB: trie.NewDatabase(kvs),
stateDatabase: state.NewDatabase(database),
}
}
// NewValidator returns a new trie validator // NewValidator returns a new trie validator
// Validating the completeness of a modified merkle patricia tries requires traversing the entire trie and verifying that // Validating the completeness of a modified merkle patricia tries requires traversing the entire trie and verifying that
// every node is present, this is an expensive operation // every node is present, this is an expensive operation
func NewValidator(db *sqlx.DB) *Validator { func NewValidator(kvs ethdb.KeyValueStore, database ethdb.Database) *Validator {
kvs := pgipfsethdb.NewKeyValueStore(db)
database := pgipfsethdb.NewDatabase(db)
return &Validator{ return &Validator{
kvs: kvs, kvs: kvs,
trieDB: trie.NewDatabase(kvs), trieDB: trie.NewDatabase(kvs),

View File

@ -30,7 +30,7 @@ import (
. "github.com/onsi/gomega" . "github.com/onsi/gomega"
"github.com/vulcanize/eth-ipfs-state-validator/pkg" "github.com/vulcanize/eth-ipfs-state-validator/pkg"
"github.com/vulcanize/pg-ipfs-ethdb/postgres" "github.com/vulcanize/ipfs-ethdb/postgres"
) )
var ( var (
@ -192,18 +192,17 @@ var (
err error err error
) )
var _ = Describe("Validator", func() { var _ = Describe("PG-IPFS Validator", func() {
BeforeEach(func() { BeforeEach(func() {
db, err = pgipfsethdb.TestDB() db, err = pgipfsethdb.TestDB()
Expect(err).ToNot(HaveOccurred()) Expect(err).ToNot(HaveOccurred())
v = validator.NewValidator(db) v = validator.NewPGIPFSValidator(db)
}) })
AfterEach(func() {
err = pgipfsethdb.ResetTestDB(db)
Expect(err).ToNot(HaveOccurred())
})
Describe("ValidateTrie", func() { Describe("ValidateTrie", func() {
AfterEach(func() {
err = validator.ResetTestDB(db)
Expect(err).ToNot(HaveOccurred())
})
It("Returns an error the state root node is missing", func() { It("Returns an error the state root node is missing", func() {
loadTrie(missingRootStateNodes, trieStorageNodes) loadTrie(missingRootStateNodes, trieStorageNodes)
err = v.ValidateTrie(stateRoot) err = v.ValidateTrie(stateRoot)
@ -236,6 +235,10 @@ var _ = Describe("Validator", func() {
}) })
Describe("ValidateStateTrie", func() { Describe("ValidateStateTrie", func() {
AfterEach(func() {
err = validator.ResetTestDB(db)
Expect(err).ToNot(HaveOccurred())
})
It("Returns an error the state root node is missing", func() { It("Returns an error the state root node is missing", func() {
loadTrie(missingRootStateNodes, nil) loadTrie(missingRootStateNodes, nil)
err = v.ValidateStateTrie(stateRoot) err = v.ValidateStateTrie(stateRoot)
@ -256,6 +259,10 @@ var _ = Describe("Validator", func() {
}) })
Describe("ValidateStorageTrie", func() { Describe("ValidateStorageTrie", func() {
AfterEach(func() {
err = validator.ResetTestDB(db)
Expect(err).ToNot(HaveOccurred())
})
It("Returns an error the storage root node is missing", func() { It("Returns an error the storage root node is missing", func() {
loadTrie(nil, missingRootStorageNodes) loadTrie(nil, missingRootStorageNodes)
err = v.ValidateStorageTrie(contractAddr, storageRoot) err = v.ValidateStorageTrie(contractAddr, storageRoot)