ipfs blockservice based validator

README.md

@@ -18,20 +18,27 @@ in a database- requires traversing the entire trie (or linked set of tries) and
 
 `full` validates completeness of the entire state corresponding to a provided state root, including both state and storage tries
 
-`./eth-ipfs-state-validator validateTrie --config={path to db config} --type=full --state-root={state root hex string}`
+`./eth-ipfs-state-validator validateTrie --ipfs-path={path to ipfs repo} --type=full --state-root={state root hex string}`
 
 
 `state` validates completeness of the state trie corresponding to a provided state root, excluding the storage tries
 
-`./eth-ipfs-state-validator validateTrie --config={path to db config} --type=state --state-root={state root hex string}`
+`./eth-ipfs-state-validator validateTrie --ipfs-path={path to ipfs repo} --type=state --state-root={state root hex string}`
 
 
 `storage` validates completeness of only the storage trie corresponding to a provided storage root and contract address
 
-`./eth-ipfs-state-validator validateTrie --config={path to db config} --type=storage --storage-root={state root hex string} --address={contract address hex string}`
+`./eth-ipfs-state-validator validateTrie --ipfs-path={path to ipfs repo} --type=storage --storage-root={state root hex string} --address={contract address hex string}`
 
 
-The config file holds the parameters for connecting to an [IPFS-backing Postgres database](https://github.com/ipfs/go-ds-sql).
+If an IPFS path is provided with the `--ipfs-path` flag, the validator operates through an IPFS block-service and expects a configured IPFS repository at
+the provided path. In this case, the validator will vie for contention on the lockfile located at the ipfs path.
+
+Alternatively, if no IPFS path is provided, the `--config` flag can be used to provide a path to a .toml config file with
+Postgres database connection parameters. In this case, the validator interfaces directly with the Postgres database and the
+database is assumed to be [IPFS-backing](https://github.com/ipfs/go-ds-sql).
+
+Postgres DB config: 
 
 ```toml
 [database]

cmd/root.go

@@ -31,6 +31,7 @@ var (
 	validationType  string
 	contractAddrStr string
 	cfgFile         string
+	ipfsPath        string
 )
 
 // rootCmd represents the base command when called without any subcommands

cmd/validateTrie.go

@@ -32,6 +32,8 @@ var validateTrieCmd = &cobra.Command{
 	Short: "Validate completeness of state data on IPFS",
 	Long: `This command is used to validate the completeness of state data corresponding specific to a specific root
 
+If an ipfs-path is provided it will use a blockservice, otherwise it expects Postgres db configuration in a linked config file.
+
 It can operate at three levels: 
 
 "full" validates completeness of the entire state corresponding to a provided state root, including both state and storage tries
@@ -56,11 +58,10 @@ It can operate at three levels:
 }
 
 func validateTrie() {
-	db, err := validator.NewDB()
+	v, err := newValidator()
 	if err != nil {
 		logWithCommand.Fatal(err)
 	}
-	v := validator.NewValidator(db)
 	switch strings.ToLower(validationType) {
 	case "f", "full":
 		if stateRootStr == "" {
@@ -96,10 +97,26 @@ func validateTrie() {
 	}
 }
 
+func newValidator() (*validator.Validator, error) {
+	if ipfsPath == "" {
+		db, err := validator.NewDB()
+		if err != nil {
+			logWithCommand.Fatal(err)
+		}
+		return validator.NewPGIPFSValidator(db), nil
+	}
+	bs, err := validator.InitIPFSBlockService(ipfsPath)
+	if err != nil {
+		return nil, err
+	}
+	return validator.NewIPFSValidator(bs), nil
+}
+
 func init() {
 	rootCmd.AddCommand(validateTrieCmd)
 	validateTrieCmd.Flags().StringVarP(&stateRootStr, "state-root", "s", "", "Root of the state trie we wish to validate; for full or state validation")
 	validateTrieCmd.Flags().StringVarP(&validationType, "type", "t", "full", "Type of validations: full, state, storage")
 	validateTrieCmd.Flags().StringVarP(&storageRootStr, "storage-root", "o", "", "Root of the storage trie we wish to validate; for storage validation")
 	validateTrieCmd.Flags().StringVarP(&contractAddrStr, "address", "a", "", "Contract address for the storage trie we wish to validate; for storage validation")
+	validateTrieCmd.Flags().StringVarP(&ipfsPath, "ipfs-path", "i", "", "Path to IPFS repository")
 }

go.mod

@@ -4,9 +4,15 @@ go 1.13
 
 require (
 	github.com/ethereum/go-ethereum v1.9.15
+	github.com/hashicorp/golang-lru v0.5.4
+	github.com/ipfs/go-block-format v0.0.2
+	github.com/ipfs/go-blockservice v0.1.3
 	github.com/ipfs/go-cid v0.0.5
+	github.com/ipfs/go-filestore v1.0.0 //indirect
+	github.com/ipfs/go-ipfs v0.5.1
 	github.com/ipfs/go-ipfs-blockstore v1.0.0
 	github.com/ipfs/go-ipfs-ds-help v1.0.0
+	github.com/ipfs/go-ipfs-exchange-interface v0.0.1
 	github.com/jmoiron/sqlx v1.2.0
 	github.com/lib/pq v1.5.2
 	github.com/multiformats/go-multihash v0.0.13
@@ -15,5 +21,5 @@ require (
 	github.com/sirupsen/logrus v1.6.0
 	github.com/spf13/cobra v1.0.0
 	github.com/spf13/viper v1.7.0
-	github.com/vulcanize/pg-ipfs-ethdb v0.0.2-alpha
+	github.com/vulcanize/ipfs-ethdb v0.0.2
 )

pkg/database.go

@@ -1,5 +1,5 @@
 // VulcanizeDB
-// Copyright © 2019 Vulcanize
+// Copyright © 2020 Vulcanize
 
 // This program is free software: you can redistribute it and/or modify
 // it under the terms of the GNU Affero General Public License as published by

pkg/util.go

@@ -17,9 +17,14 @@
 package validator
 
 import (
+	"context"
+
+	"github.com/ipfs/go-blockservice"
 	"github.com/ipfs/go-cid"
 	"github.com/ipfs/go-ipfs-blockstore"
 	"github.com/ipfs/go-ipfs-ds-help"
+	"github.com/ipfs/go-ipfs/core"
+	"github.com/ipfs/go-ipfs/repo/fsrepo"
 	"github.com/jmoiron/sqlx"
 )
 
@@ -49,3 +54,27 @@ func RawdataToCid(codec uint64, rawdata []byte, multiHash uint64) (cid.Cid, erro
 	}
 	return c, nil
 }
+
+// InitIPFSBlockService is used to configure and return a BlockService using an ipfs repo path (e.g. ~/.ipfs)
+func InitIPFSBlockService(ipfsPath string) (blockservice.BlockService, error) {
+	r, openErr := fsrepo.Open(ipfsPath)
+	if openErr != nil {
+		return nil, openErr
+	}
+	ctx := context.Background()
+	cfg := &core.BuildCfg{
+		Online: false,
+		Repo:   r,
+	}
+	ipfsNode, newNodeErr := core.NewNode(ctx, cfg)
+	if newNodeErr != nil {
+		return nil, newNodeErr
+	}
+	return ipfsNode.Blocks, nil
+}
+
+// ResetTestDB drops all rows in the test db public.blocks table
+func ResetTestDB(db *sqlx.DB) error {
+	_, err := db.Exec("DELETE FROM public.blocks")
+	return err
+}

pkg/validator.go

@@ -24,9 +24,11 @@ import (
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/ethdb"
 	"github.com/ethereum/go-ethereum/trie"
+	"github.com/ipfs/go-blockservice"
 	"github.com/jmoiron/sqlx"
 
-	pgipfsethdb "github.com/vulcanize/pg-ipfs-ethdb/postgres"
+	"github.com/vulcanize/ipfs-ethdb"
+	"github.com/vulcanize/ipfs-ethdb/postgres"
 )
 
 // Validator is used for validating Ethereum state and storage tries on PG-IPFS
@@ -36,12 +38,32 @@ type Validator struct {
 	stateDatabase state.Database
 }
 
+// NewPGIPFSValidator returns a new trie validator ontop of a connection pool for an IPFS backing Postgres database
+func NewPGIPFSValidator(db *sqlx.DB) *Validator {
+	kvs := pgipfsethdb.NewKeyValueStore(db)
+	database := pgipfsethdb.NewDatabase(db)
+	return &Validator{
+		kvs:           kvs,
+		trieDB:        trie.NewDatabase(kvs),
+		stateDatabase: state.NewDatabase(database),
+	}
+}
+
+// NewIPFSValidator returns a new trie validator ontop of an IPFS blockservice
+func NewIPFSValidator(bs blockservice.BlockService) *Validator {
+	kvs := ipfsethdb.NewKeyValueStore(bs)
+	database := ipfsethdb.NewDatabase(bs)
+	return &Validator{
+		kvs:           kvs,
+		trieDB:        trie.NewDatabase(kvs),
+		stateDatabase: state.NewDatabase(database),
+	}
+}
+
 // NewValidator returns a new trie validator
 // Validating the completeness of a modified merkle patricia tries requires traversing the entire trie and verifying that
 // every node is present, this is an expensive operation
-func NewValidator(db *sqlx.DB) *Validator {
+func NewValidator(kvs ethdb.KeyValueStore, database ethdb.Database) *Validator {
-	kvs := pgipfsethdb.NewKeyValueStore(db)
-	database := pgipfsethdb.NewDatabase(db)
 	return &Validator{
 		kvs:           kvs,
 		trieDB:        trie.NewDatabase(kvs),

pkg/validator_test.go

@@ -30,7 +30,7 @@ import (
 	. "github.com/onsi/gomega"
 
 	"github.com/vulcanize/eth-ipfs-state-validator/pkg"
-	"github.com/vulcanize/pg-ipfs-ethdb/postgres"
+	"github.com/vulcanize/ipfs-ethdb/postgres"
 )
 
 var (
@@ -192,18 +192,17 @@ var (
 	err error
 )
 
-var _ = Describe("Validator", func() {
+var _ = Describe("PG-IPFS Validator", func() {
 	BeforeEach(func() {
 		db, err = pgipfsethdb.TestDB()
 		Expect(err).ToNot(HaveOccurred())
-		v = validator.NewValidator(db)
+		v = validator.NewPGIPFSValidator(db)
 	})
+	Describe("ValidateTrie", func() {
 		AfterEach(func() {
-		err = pgipfsethdb.ResetTestDB(db)
+			err = validator.ResetTestDB(db)
 			Expect(err).ToNot(HaveOccurred())
 		})
-
-	Describe("ValidateTrie", func() {
 		It("Returns an error the state root node is missing", func() {
 			loadTrie(missingRootStateNodes, trieStorageNodes)
 			err = v.ValidateTrie(stateRoot)
@@ -236,6 +235,10 @@ var _ = Describe("Validator", func() {
 	})
 
 	Describe("ValidateStateTrie", func() {
+		AfterEach(func() {
+			err = validator.ResetTestDB(db)
+			Expect(err).ToNot(HaveOccurred())
+		})
 		It("Returns an error the state root node is missing", func() {
 			loadTrie(missingRootStateNodes, nil)
 			err = v.ValidateStateTrie(stateRoot)
@@ -256,6 +259,10 @@ var _ = Describe("Validator", func() {
 	})
 
 	Describe("ValidateStorageTrie", func() {
+		AfterEach(func() {
+			err = validator.ResetTestDB(db)
+			Expect(err).ToNot(HaveOccurred())
+		})
 		It("Returns an error the storage root node is missing", func() {
 			loadTrie(nil, missingRootStorageNodes)
 			err = v.ValidateStorageTrie(contractAddr, storageRoot)