Merge pull request #22 from deep-stack/pm-v4-schema-updates

Update to use v4 schema

cmd/validateTrie.go

@@ -24,7 +24,7 @@ import (
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
 
-	validator "github.com/vulcanize/eth-ipfs-state-validator/pkg"
+	validator "github.com/vulcanize/eth-ipfs-state-validator/v4/pkg"
 )
 
 // validateTrieCmd represents the validateTrie command

go.mod

@@ -1,23 +1,23 @@
-module github.com/vulcanize/eth-ipfs-state-validator
+module github.com/vulcanize/eth-ipfs-state-validator/v4
 
-go 1.13
+go 1.15
 
 require (
-	github.com/ethereum/go-ethereum v1.9.15
+	github.com/ethereum/go-ethereum v1.10.17
-	github.com/ipfs/go-blockservice v0.1.3
+	github.com/ipfs/go-blockservice v0.1.7
 	github.com/ipfs/go-cid v0.0.7
 	github.com/ipfs/go-filestore v1.0.0 //indirect
-	github.com/ipfs/go-ipfs v0.7.0
+	github.com/ipfs/go-ipfs v0.10.0
-	github.com/ipfs/go-ipfs-blockstore v1.0.0
+	github.com/ipfs/go-ipfs-blockstore v1.0.1
 	github.com/ipfs/go-ipfs-ds-help v1.0.0
-	github.com/jmoiron/sqlx v1.2.0
+	github.com/jmoiron/sqlx v1.3.5
-	github.com/lib/pq v1.5.2
+	github.com/lib/pq v1.10.5
-	github.com/mailgun/groupcache/v2 v2.2.1
+	github.com/mailgun/groupcache/v2 v2.3.0
-	github.com/multiformats/go-multihash v0.0.14
+	github.com/multiformats/go-multihash v0.1.0
-	github.com/onsi/ginkgo v1.14.0
+	github.com/onsi/ginkgo v1.16.5
-	github.com/onsi/gomega v1.10.1
+	github.com/onsi/gomega v1.19.0
-	github.com/sirupsen/logrus v1.6.0
+	github.com/sirupsen/logrus v1.8.1
-	github.com/spf13/cobra v1.0.0
+	github.com/spf13/cobra v1.4.0
-	github.com/spf13/viper v1.7.0
+	github.com/spf13/viper v1.11.0
-	github.com/vulcanize/ipfs-ethdb v0.0.3
+	github.com/vulcanize/ipfs-ethdb/v4 v4.0.0-alpha
 )

main.go

@@ -18,7 +18,7 @@ package main
 import (
 	"github.com/sirupsen/logrus"
 
-	"github.com/vulcanize/eth-ipfs-state-validator/cmd"
+	"github.com/vulcanize/eth-ipfs-state-validator/v4/cmd"
 )
 
 func main() {

pkg/util.go

@@ -21,22 +21,22 @@ import (
 
 	"github.com/ipfs/go-blockservice"
 	"github.com/ipfs/go-cid"
-	"github.com/ipfs/go-ipfs-blockstore"
+	blockstore "github.com/ipfs/go-ipfs-blockstore"
-	"github.com/ipfs/go-ipfs-ds-help"
+	dshelp "github.com/ipfs/go-ipfs-ds-help"
 	"github.com/ipfs/go-ipfs/core"
 	"github.com/ipfs/go-ipfs/repo/fsrepo"
 	"github.com/jmoiron/sqlx"
 )
 
 // PublishRaw derives a cid from raw bytes and provided codec and multihash type, and writes it to the db tx
-func PublishRaw(tx *sqlx.Tx, codec, mh uint64, raw []byte) (string, error) {
+func PublishRaw(tx *sqlx.Tx, codec, mh uint64, raw []byte, blockNumber uint64) (string, error) {
 	c, err := RawdataToCid(codec, raw, mh)
 	if err != nil {
 		return "", err
 	}
 	dbKey := dshelp.MultihashToDsKey(c.Hash())
 	prefixedKey := blockstore.BlockPrefix.String() + dbKey.String()
-	_, err = tx.Exec(`INSERT INTO public.blocks (key, data) VALUES ($1, $2) ON CONFLICT (key) DO NOTHING`, prefixedKey, raw)
+	_, err = tx.Exec(`INSERT INTO public.blocks (key, data, block_number) VALUES ($1, $2, $3) ON CONFLICT DO NOTHING`, prefixedKey, raw, blockNumber)
 	return c.String(), err
 }
 

pkg/validator.go

@@ -28,8 +28,8 @@ import (
 	"github.com/jmoiron/sqlx"
 	"github.com/mailgun/groupcache/v2"
 
-	ipfsethdb "github.com/vulcanize/ipfs-ethdb"
+	ipfsethdb "github.com/vulcanize/ipfs-ethdb/v4"
-	pgipfsethdb "github.com/vulcanize/ipfs-ethdb/postgres"
+	pgipfsethdb "github.com/vulcanize/ipfs-ethdb/v4/postgres"
 )
 
 // Validator is used for validating Ethereum state and storage tries on PG-IPFS
@@ -58,7 +58,7 @@ func NewPGIPFSValidator(db *sqlx.DB) *Validator {
 		kvs:           kvs,
 		trieDB:        trie.NewDatabase(kvs),
 		stateDatabase: state.NewDatabase(database),
-		db:            database,
+		db:            database.(*pgipfsethdb.Database),
 	}
 }
 
@@ -138,3 +138,11 @@ func (v *Validator) ValidateStorageTrie(address common.Address, storageRoot comm
 	}
 	return it.Error()
 }
+
+// Close implements io.Closer
+// it deregisters the groupcache name
+func (v *Validator) Close() error {
+	groupcache.DeregisterGroup("kv")
+	groupcache.DeregisterGroup("db")
+	return nil
+}

pkg/validator_test.go

@@ -21,28 +21,29 @@ import (
 	"math/big"
 
 	"github.com/ethereum/go-ethereum/common"
-	"github.com/ethereum/go-ethereum/core/state"
+	"github.com/ethereum/go-ethereum/core/types"
 	"github.com/ethereum/go-ethereum/crypto"
 	"github.com/ethereum/go-ethereum/rlp"
-	"github.com/ipfs/go-cid/_rsrch/cidiface"
+	cid "github.com/ipfs/go-cid/_rsrch/cidiface"
 	"github.com/jmoiron/sqlx"
 	"github.com/multiformats/go-multihash"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 
-	"github.com/vulcanize/eth-ipfs-state-validator/pkg"
+	validator "github.com/vulcanize/eth-ipfs-state-validator/v4/pkg"
-	"github.com/vulcanize/ipfs-ethdb/postgres"
+	pgipfsethdb "github.com/vulcanize/ipfs-ethdb/v4/postgres"
 )
 
 var (
+	blockNumber       = uint64(1)
 	contractAddr      = common.HexToAddress("0xaE9BEa628c4Ce503DcFD7E305CaB4e29E7476592")
 	slot0StorageValue = common.Hex2Bytes("94703c4b2bd70c169f5717101caee543299fc946c7")
 	slot1StorageValue = common.Hex2Bytes("01")
 	nullCodeHash      = crypto.Keccak256Hash([]byte{})
-	emptyRootNode, _  = rlp.EncodeToBytes([]byte{})
+	emptyRootNode, _  = rlp.EncodeToBytes(&[]byte{})
 	emptyContractRoot = crypto.Keccak256Hash(emptyRootNode)
 
-	stateBranchRootNode, _ = rlp.EncodeToBytes([]interface{}{
+	stateBranchRootNode, _ = rlp.EncodeToBytes(&[]interface{}{
 		crypto.Keccak256(bankAccountLeafNode),
 		[]byte{},
 		[]byte{},
@@ -65,62 +66,62 @@ var (
 
 	mockCode           = []byte{1, 2, 3, 4, 5}
 	codeHash           = crypto.Keccak256Hash(mockCode)
-	contractAccount, _ = rlp.EncodeToBytes(state.Account{
+	contractAccount, _ = rlp.EncodeToBytes(&types.StateAccount{
 		Nonce:    1,
 		Balance:  big.NewInt(0),
 		CodeHash: codeHash.Bytes(),
 		Root:     crypto.Keccak256Hash(storageBranchRootNode),
 	})
-	contractAccountLeafNode, _ = rlp.EncodeToBytes([]interface{}{
+	contractAccountLeafNode, _ = rlp.EncodeToBytes(&[]interface{}{
 		common.Hex2Bytes("3114658a74d9cc9f7acf2c5cd696c3494d7c344d78bfec3add0d91ec4e8d1c45"),
 		contractAccount,
 	})
 
-	minerAccount, _ = rlp.EncodeToBytes(state.Account{
+	minerAccount, _ = rlp.EncodeToBytes(&types.StateAccount{
 		Nonce:    0,
 		Balance:  big.NewInt(1000),
 		CodeHash: nullCodeHash.Bytes(),
 		Root:     emptyContractRoot,
 	})
-	minerAccountLeafNode, _ = rlp.EncodeToBytes([]interface{}{
+	minerAccountLeafNode, _ = rlp.EncodeToBytes(&[]interface{}{
 		common.Hex2Bytes("3380c7b7ae81a58eb98d9c78de4a1fd7fd9535fc953ed2be602daaa41767312a"),
 		minerAccount,
 	})
 
-	account1, _ = rlp.EncodeToBytes(state.Account{
+	account1, _ = rlp.EncodeToBytes(&types.StateAccount{
 		Nonce:    2,
 		Balance:  big.NewInt(1000),
 		CodeHash: nullCodeHash.Bytes(),
 		Root:     emptyContractRoot,
 	})
-	account1LeafNode, _ = rlp.EncodeToBytes([]interface{}{
+	account1LeafNode, _ = rlp.EncodeToBytes(&[]interface{}{
 		common.Hex2Bytes("3926db69aaced518e9b9f0f434a473e7174109c943548bb8f23be41ca76d9ad2"),
 		account1,
 	})
 
-	account2, _ = rlp.EncodeToBytes(state.Account{
+	account2, _ = rlp.EncodeToBytes(&types.StateAccount{
 		Nonce:    0,
 		Balance:  big.NewInt(1000),
 		CodeHash: nullCodeHash.Bytes(),
 		Root:     emptyContractRoot,
 	})
-	account2LeafNode, _ = rlp.EncodeToBytes([]interface{}{
+	account2LeafNode, _ = rlp.EncodeToBytes(&[]interface{}{
 		common.Hex2Bytes("3957f3e2f04a0764c3a0491b175f69926da61efbcc8f61fa1455fd2d2b4cdd45"),
 		account2,
 	})
 
-	bankAccount, _ = rlp.EncodeToBytes(state.Account{
+	bankAccount, _ = rlp.EncodeToBytes(&types.StateAccount{
 		Nonce:    2,
 		Balance:  big.NewInt(1000),
 		CodeHash: nullCodeHash.Bytes(),
 		Root:     emptyContractRoot,
 	})
-	bankAccountLeafNode, _ = rlp.EncodeToBytes([]interface{}{
+	bankAccountLeafNode, _ = rlp.EncodeToBytes(&[]interface{}{
 		common.Hex2Bytes("30bf49f440a1cd0527e4d06e2765654c0f56452257516d793a9b8d604dcfdf2a"),
 		bankAccount,
 	})
 
-	storageBranchRootNode, _ = rlp.EncodeToBytes([]interface{}{
+	storageBranchRootNode, _ = rlp.EncodeToBytes(&[]interface{}{
 		[]byte{},
 		[]byte{},
 		crypto.Keccak256(slot0StorageLeafNode),
@@ -141,11 +142,11 @@ var (
 	})
 	storageRoot = crypto.Keccak256Hash(storageBranchRootNode)
 
-	slot0StorageLeafNode, _ = rlp.EncodeToBytes([]interface{}{
+	slot0StorageLeafNode, _ = rlp.EncodeToBytes(&[]interface{}{
 		common.Hex2Bytes("390decd9548b62a8d60345a988386fc84ba6bc95484008f6362f93160ef3e563"),
 		slot0StorageValue,
 	})
-	slot1StorageLeafNode, _ = rlp.EncodeToBytes([]interface{}{
+	slot1StorageLeafNode, _ = rlp.EncodeToBytes(&[]interface{}{
 		common.Hex2Bytes("310e2d527612073b26eecdfd717e6a320cf44b4afac2b0732d9fcbe2b7fa0cf6"),
 		slot1StorageValue,
 	})
@@ -201,6 +202,9 @@ var _ = Describe("PG-IPFS Validator", func() {
 		Expect(err).ToNot(HaveOccurred())
 		v = validator.NewPGIPFSValidator(db)
 	})
+	AfterEach(func() {
+		v.Close()
+	})
 	Describe("ValidateTrie", func() {
 		AfterEach(func() {
 			err = validator.ResetTestDB(db)
@@ -236,7 +240,7 @@ var _ = Describe("PG-IPFS Validator", func() {
 			loadTrie(trieStateNodes, trieStorageNodes)
 			err = v.ValidateTrie(stateRoot)
 			Expect(err).To(HaveOccurred())
-			subStr := fmt.Sprintf("code %s: sql: no rows in result set", codeHash.Hex()[2:])
+			subStr := fmt.Sprintf("code %s: not found", codeHash.Hex()[2:])
 			Expect(err.Error()).To(ContainSubstring(subStr))
 		})
 		It("Returns no error if the entire state (state trie and storage tries) can be validated", func() {
@@ -299,11 +303,11 @@ func loadTrie(stateNodes, storageNodes [][]byte) {
 	tx, err := db.Beginx()
 	Expect(err).ToNot(HaveOccurred())
 	for _, node := range stateNodes {
-		_, err := validator.PublishRaw(tx, cid.EthStateTrie, multihash.KECCAK_256, node)
+		_, err := validator.PublishRaw(tx, cid.EthStateTrie, multihash.KECCAK_256, node, blockNumber)
 		Expect(err).ToNot(HaveOccurred())
 	}
 	for _, node := range storageNodes {
-		_, err := validator.PublishRaw(tx, cid.EthStorageTrie, multihash.KECCAK_256, node)
+		_, err := validator.PublishRaw(tx, cid.EthStorageTrie, multihash.KECCAK_256, node, blockNumber)
 		Expect(err).ToNot(HaveOccurred())
 	}
 	err = tx.Commit()