ipfs blockservice based validator
This commit is contained in:
parent
cad3bb3fb6
commit
4f089a7a1a
15
README.md
15
README.md
@ -18,20 +18,27 @@ in a database- requires traversing the entire trie (or linked set of tries) and
|
|||||||
|
|
||||||
`full` validates completeness of the entire state corresponding to a provided state root, including both state and storage tries
|
`full` validates completeness of the entire state corresponding to a provided state root, including both state and storage tries
|
||||||
|
|
||||||
`./eth-ipfs-state-validator validateTrie --config={path to db config} --type=full --state-root={state root hex string}`
|
`./eth-ipfs-state-validator validateTrie --ipfs-path={path to ipfs repo} --type=full --state-root={state root hex string}`
|
||||||
|
|
||||||
|
|
||||||
`state` validates completeness of the state trie corresponding to a provided state root, excluding the storage tries
|
`state` validates completeness of the state trie corresponding to a provided state root, excluding the storage tries
|
||||||
|
|
||||||
`./eth-ipfs-state-validator validateTrie --config={path to db config} --type=state --state-root={state root hex string}`
|
`./eth-ipfs-state-validator validateTrie --ipfs-path={path to ipfs repo} --type=state --state-root={state root hex string}`
|
||||||
|
|
||||||
|
|
||||||
`storage` validates completeness of only the storage trie corresponding to a provided storage root and contract address
|
`storage` validates completeness of only the storage trie corresponding to a provided storage root and contract address
|
||||||
|
|
||||||
`./eth-ipfs-state-validator validateTrie --config={path to db config} --type=storage --storage-root={state root hex string} --address={contract address hex string}`
|
`./eth-ipfs-state-validator validateTrie --ipfs-path={path to ipfs repo} --type=storage --storage-root={state root hex string} --address={contract address hex string}`
|
||||||
|
|
||||||
|
|
||||||
The config file holds the parameters for connecting to an [IPFS-backing Postgres database](https://github.com/ipfs/go-ds-sql).
|
If an IPFS path is provided with the `--ipfs-path` flag, the validator operates through an IPFS block-service and expects a configured IPFS repository at
|
||||||
|
the provided path. In this case, the validator will vie for contention on the lockfile located at the ipfs path.
|
||||||
|
|
||||||
|
Alternatively, if no IPFS path is provided, the `--config` flag can be used to provide a path to a .toml config file with
|
||||||
|
Postgres database connection parameters. In this case, the validator interfaces directly with the Postgres database and the
|
||||||
|
database is assumed to be [IPFS-backing](https://github.com/ipfs/go-ds-sql).
|
||||||
|
|
||||||
|
Postgres DB config:
|
||||||
|
|
||||||
```toml
|
```toml
|
||||||
[database]
|
[database]
|
||||||
|
@ -31,6 +31,7 @@ var (
|
|||||||
validationType string
|
validationType string
|
||||||
contractAddrStr string
|
contractAddrStr string
|
||||||
cfgFile string
|
cfgFile string
|
||||||
|
ipfsPath string
|
||||||
)
|
)
|
||||||
|
|
||||||
// rootCmd represents the base command when called without any subcommands
|
// rootCmd represents the base command when called without any subcommands
|
||||||
|
@ -32,6 +32,8 @@ var validateTrieCmd = &cobra.Command{
|
|||||||
Short: "Validate completeness of state data on IPFS",
|
Short: "Validate completeness of state data on IPFS",
|
||||||
Long: `This command is used to validate the completeness of state data corresponding specific to a specific root
|
Long: `This command is used to validate the completeness of state data corresponding specific to a specific root
|
||||||
|
|
||||||
|
If an ipfs-path is provided it will use a blockservice, otherwise it expects Postgres db configuration in a linked config file.
|
||||||
|
|
||||||
It can operate at three levels:
|
It can operate at three levels:
|
||||||
|
|
||||||
"full" validates completeness of the entire state corresponding to a provided state root, including both state and storage tries
|
"full" validates completeness of the entire state corresponding to a provided state root, including both state and storage tries
|
||||||
@ -56,11 +58,10 @@ It can operate at three levels:
|
|||||||
}
|
}
|
||||||
|
|
||||||
func validateTrie() {
|
func validateTrie() {
|
||||||
db, err := validator.NewDB()
|
v, err := newValidator()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logWithCommand.Fatal(err)
|
logWithCommand.Fatal(err)
|
||||||
}
|
}
|
||||||
v := validator.NewValidator(db)
|
|
||||||
switch strings.ToLower(validationType) {
|
switch strings.ToLower(validationType) {
|
||||||
case "f", "full":
|
case "f", "full":
|
||||||
if stateRootStr == "" {
|
if stateRootStr == "" {
|
||||||
@ -96,10 +97,26 @@ func validateTrie() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func newValidator() (*validator.Validator, error) {
|
||||||
|
if ipfsPath == "" {
|
||||||
|
db, err := validator.NewDB()
|
||||||
|
if err != nil {
|
||||||
|
logWithCommand.Fatal(err)
|
||||||
|
}
|
||||||
|
return validator.NewPGIPFSValidator(db), nil
|
||||||
|
}
|
||||||
|
bs, err := validator.InitIPFSBlockService(ipfsPath)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return validator.NewIPFSValidator(bs), nil
|
||||||
|
}
|
||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
rootCmd.AddCommand(validateTrieCmd)
|
rootCmd.AddCommand(validateTrieCmd)
|
||||||
validateTrieCmd.Flags().StringVarP(&stateRootStr, "state-root", "s", "", "Root of the state trie we wish to validate; for full or state validation")
|
validateTrieCmd.Flags().StringVarP(&stateRootStr, "state-root", "s", "", "Root of the state trie we wish to validate; for full or state validation")
|
||||||
validateTrieCmd.Flags().StringVarP(&validationType, "type", "t", "full", "Type of validations: full, state, storage")
|
validateTrieCmd.Flags().StringVarP(&validationType, "type", "t", "full", "Type of validations: full, state, storage")
|
||||||
validateTrieCmd.Flags().StringVarP(&storageRootStr, "storage-root", "o", "", "Root of the storage trie we wish to validate; for storage validation")
|
validateTrieCmd.Flags().StringVarP(&storageRootStr, "storage-root", "o", "", "Root of the storage trie we wish to validate; for storage validation")
|
||||||
validateTrieCmd.Flags().StringVarP(&contractAddrStr, "address", "a", "", "Contract address for the storage trie we wish to validate; for storage validation")
|
validateTrieCmd.Flags().StringVarP(&contractAddrStr, "address", "a", "", "Contract address for the storage trie we wish to validate; for storage validation")
|
||||||
|
validateTrieCmd.Flags().StringVarP(&ipfsPath, "ipfs-path", "i", "", "Path to IPFS repository")
|
||||||
}
|
}
|
||||||
|
8
go.mod
8
go.mod
@ -4,9 +4,15 @@ go 1.13
|
|||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/ethereum/go-ethereum v1.9.15
|
github.com/ethereum/go-ethereum v1.9.15
|
||||||
|
github.com/hashicorp/golang-lru v0.5.4
|
||||||
|
github.com/ipfs/go-block-format v0.0.2
|
||||||
|
github.com/ipfs/go-blockservice v0.1.3
|
||||||
github.com/ipfs/go-cid v0.0.5
|
github.com/ipfs/go-cid v0.0.5
|
||||||
|
github.com/ipfs/go-filestore v1.0.0 //indirect
|
||||||
|
github.com/ipfs/go-ipfs v0.5.1
|
||||||
github.com/ipfs/go-ipfs-blockstore v1.0.0
|
github.com/ipfs/go-ipfs-blockstore v1.0.0
|
||||||
github.com/ipfs/go-ipfs-ds-help v1.0.0
|
github.com/ipfs/go-ipfs-ds-help v1.0.0
|
||||||
|
github.com/ipfs/go-ipfs-exchange-interface v0.0.1
|
||||||
github.com/jmoiron/sqlx v1.2.0
|
github.com/jmoiron/sqlx v1.2.0
|
||||||
github.com/lib/pq v1.5.2
|
github.com/lib/pq v1.5.2
|
||||||
github.com/multiformats/go-multihash v0.0.13
|
github.com/multiformats/go-multihash v0.0.13
|
||||||
@ -15,5 +21,5 @@ require (
|
|||||||
github.com/sirupsen/logrus v1.6.0
|
github.com/sirupsen/logrus v1.6.0
|
||||||
github.com/spf13/cobra v1.0.0
|
github.com/spf13/cobra v1.0.0
|
||||||
github.com/spf13/viper v1.7.0
|
github.com/spf13/viper v1.7.0
|
||||||
github.com/vulcanize/pg-ipfs-ethdb v0.0.2-alpha
|
github.com/vulcanize/ipfs-ethdb v0.0.2
|
||||||
)
|
)
|
||||||
|
@ -1,5 +1,5 @@
|
|||||||
// VulcanizeDB
|
// VulcanizeDB
|
||||||
// Copyright © 2019 Vulcanize
|
// Copyright © 2020 Vulcanize
|
||||||
|
|
||||||
// This program is free software: you can redistribute it and/or modify
|
// This program is free software: you can redistribute it and/or modify
|
||||||
// it under the terms of the GNU Affero General Public License as published by
|
// it under the terms of the GNU Affero General Public License as published by
|
||||||
|
29
pkg/util.go
29
pkg/util.go
@ -17,9 +17,14 @@
|
|||||||
package validator
|
package validator
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"context"
|
||||||
|
|
||||||
|
"github.com/ipfs/go-blockservice"
|
||||||
"github.com/ipfs/go-cid"
|
"github.com/ipfs/go-cid"
|
||||||
"github.com/ipfs/go-ipfs-blockstore"
|
"github.com/ipfs/go-ipfs-blockstore"
|
||||||
"github.com/ipfs/go-ipfs-ds-help"
|
"github.com/ipfs/go-ipfs-ds-help"
|
||||||
|
"github.com/ipfs/go-ipfs/core"
|
||||||
|
"github.com/ipfs/go-ipfs/repo/fsrepo"
|
||||||
"github.com/jmoiron/sqlx"
|
"github.com/jmoiron/sqlx"
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -49,3 +54,27 @@ func RawdataToCid(codec uint64, rawdata []byte, multiHash uint64) (cid.Cid, erro
|
|||||||
}
|
}
|
||||||
return c, nil
|
return c, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// InitIPFSBlockService is used to configure and return a BlockService using an ipfs repo path (e.g. ~/.ipfs)
|
||||||
|
func InitIPFSBlockService(ipfsPath string) (blockservice.BlockService, error) {
|
||||||
|
r, openErr := fsrepo.Open(ipfsPath)
|
||||||
|
if openErr != nil {
|
||||||
|
return nil, openErr
|
||||||
|
}
|
||||||
|
ctx := context.Background()
|
||||||
|
cfg := &core.BuildCfg{
|
||||||
|
Online: false,
|
||||||
|
Repo: r,
|
||||||
|
}
|
||||||
|
ipfsNode, newNodeErr := core.NewNode(ctx, cfg)
|
||||||
|
if newNodeErr != nil {
|
||||||
|
return nil, newNodeErr
|
||||||
|
}
|
||||||
|
return ipfsNode.Blocks, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ResetTestDB drops all rows in the test db public.blocks table
|
||||||
|
func ResetTestDB(db *sqlx.DB) error {
|
||||||
|
_, err := db.Exec("DELETE FROM public.blocks")
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
@ -24,9 +24,11 @@ import (
|
|||||||
"github.com/ethereum/go-ethereum/crypto"
|
"github.com/ethereum/go-ethereum/crypto"
|
||||||
"github.com/ethereum/go-ethereum/ethdb"
|
"github.com/ethereum/go-ethereum/ethdb"
|
||||||
"github.com/ethereum/go-ethereum/trie"
|
"github.com/ethereum/go-ethereum/trie"
|
||||||
|
"github.com/ipfs/go-blockservice"
|
||||||
"github.com/jmoiron/sqlx"
|
"github.com/jmoiron/sqlx"
|
||||||
|
|
||||||
pgipfsethdb "github.com/vulcanize/pg-ipfs-ethdb/postgres"
|
"github.com/vulcanize/ipfs-ethdb"
|
||||||
|
"github.com/vulcanize/ipfs-ethdb/postgres"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Validator is used for validating Ethereum state and storage tries on PG-IPFS
|
// Validator is used for validating Ethereum state and storage tries on PG-IPFS
|
||||||
@ -36,12 +38,32 @@ type Validator struct {
|
|||||||
stateDatabase state.Database
|
stateDatabase state.Database
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// NewPGIPFSValidator returns a new trie validator ontop of a connection pool for an IPFS backing Postgres database
|
||||||
|
func NewPGIPFSValidator(db *sqlx.DB) *Validator {
|
||||||
|
kvs := pgipfsethdb.NewKeyValueStore(db)
|
||||||
|
database := pgipfsethdb.NewDatabase(db)
|
||||||
|
return &Validator{
|
||||||
|
kvs: kvs,
|
||||||
|
trieDB: trie.NewDatabase(kvs),
|
||||||
|
stateDatabase: state.NewDatabase(database),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewIPFSValidator returns a new trie validator ontop of an IPFS blockservice
|
||||||
|
func NewIPFSValidator(bs blockservice.BlockService) *Validator {
|
||||||
|
kvs := ipfsethdb.NewKeyValueStore(bs)
|
||||||
|
database := ipfsethdb.NewDatabase(bs)
|
||||||
|
return &Validator{
|
||||||
|
kvs: kvs,
|
||||||
|
trieDB: trie.NewDatabase(kvs),
|
||||||
|
stateDatabase: state.NewDatabase(database),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// NewValidator returns a new trie validator
|
// NewValidator returns a new trie validator
|
||||||
// Validating the completeness of a modified merkle patricia tries requires traversing the entire trie and verifying that
|
// Validating the completeness of a modified merkle patricia tries requires traversing the entire trie and verifying that
|
||||||
// every node is present, this is an expensive operation
|
// every node is present, this is an expensive operation
|
||||||
func NewValidator(db *sqlx.DB) *Validator {
|
func NewValidator(kvs ethdb.KeyValueStore, database ethdb.Database) *Validator {
|
||||||
kvs := pgipfsethdb.NewKeyValueStore(db)
|
|
||||||
database := pgipfsethdb.NewDatabase(db)
|
|
||||||
return &Validator{
|
return &Validator{
|
||||||
kvs: kvs,
|
kvs: kvs,
|
||||||
trieDB: trie.NewDatabase(kvs),
|
trieDB: trie.NewDatabase(kvs),
|
||||||
|
@ -30,7 +30,7 @@ import (
|
|||||||
. "github.com/onsi/gomega"
|
. "github.com/onsi/gomega"
|
||||||
|
|
||||||
"github.com/vulcanize/eth-ipfs-state-validator/pkg"
|
"github.com/vulcanize/eth-ipfs-state-validator/pkg"
|
||||||
"github.com/vulcanize/pg-ipfs-ethdb/postgres"
|
"github.com/vulcanize/ipfs-ethdb/postgres"
|
||||||
)
|
)
|
||||||
|
|
||||||
var (
|
var (
|
||||||
@ -192,18 +192,17 @@ var (
|
|||||||
err error
|
err error
|
||||||
)
|
)
|
||||||
|
|
||||||
var _ = Describe("Validator", func() {
|
var _ = Describe("PG-IPFS Validator", func() {
|
||||||
BeforeEach(func() {
|
BeforeEach(func() {
|
||||||
db, err = pgipfsethdb.TestDB()
|
db, err = pgipfsethdb.TestDB()
|
||||||
Expect(err).ToNot(HaveOccurred())
|
Expect(err).ToNot(HaveOccurred())
|
||||||
v = validator.NewValidator(db)
|
v = validator.NewPGIPFSValidator(db)
|
||||||
})
|
})
|
||||||
AfterEach(func() {
|
|
||||||
err = pgipfsethdb.ResetTestDB(db)
|
|
||||||
Expect(err).ToNot(HaveOccurred())
|
|
||||||
})
|
|
||||||
|
|
||||||
Describe("ValidateTrie", func() {
|
Describe("ValidateTrie", func() {
|
||||||
|
AfterEach(func() {
|
||||||
|
err = validator.ResetTestDB(db)
|
||||||
|
Expect(err).ToNot(HaveOccurred())
|
||||||
|
})
|
||||||
It("Returns an error the state root node is missing", func() {
|
It("Returns an error the state root node is missing", func() {
|
||||||
loadTrie(missingRootStateNodes, trieStorageNodes)
|
loadTrie(missingRootStateNodes, trieStorageNodes)
|
||||||
err = v.ValidateTrie(stateRoot)
|
err = v.ValidateTrie(stateRoot)
|
||||||
@ -236,6 +235,10 @@ var _ = Describe("Validator", func() {
|
|||||||
})
|
})
|
||||||
|
|
||||||
Describe("ValidateStateTrie", func() {
|
Describe("ValidateStateTrie", func() {
|
||||||
|
AfterEach(func() {
|
||||||
|
err = validator.ResetTestDB(db)
|
||||||
|
Expect(err).ToNot(HaveOccurred())
|
||||||
|
})
|
||||||
It("Returns an error the state root node is missing", func() {
|
It("Returns an error the state root node is missing", func() {
|
||||||
loadTrie(missingRootStateNodes, nil)
|
loadTrie(missingRootStateNodes, nil)
|
||||||
err = v.ValidateStateTrie(stateRoot)
|
err = v.ValidateStateTrie(stateRoot)
|
||||||
@ -256,6 +259,10 @@ var _ = Describe("Validator", func() {
|
|||||||
})
|
})
|
||||||
|
|
||||||
Describe("ValidateStorageTrie", func() {
|
Describe("ValidateStorageTrie", func() {
|
||||||
|
AfterEach(func() {
|
||||||
|
err = validator.ResetTestDB(db)
|
||||||
|
Expect(err).ToNot(HaveOccurred())
|
||||||
|
})
|
||||||
It("Returns an error the storage root node is missing", func() {
|
It("Returns an error the storage root node is missing", func() {
|
||||||
loadTrie(nil, missingRootStorageNodes)
|
loadTrie(nil, missingRootStorageNodes)
|
||||||
err = v.ValidateStorageTrie(contractAddr, storageRoot)
|
err = v.ValidateStorageTrie(contractAddr, storageRoot)
|
||||||
|
Loading…
Reference in New Issue
Block a user