From 1bc449d0360ede0df4ab48210d967d6b77c5e7c7 Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Mon, 15 Apr 2024 19:42:58 +0800 Subject: [PATCH] updates, fixes --- pkg/util.go | 2 +- pkg/validator.go | 42 +++++++++++++++++++++++++----------------- pkg/validator_test.go | 24 +++++++++++++----------- 3 files changed, 39 insertions(+), 29 deletions(-) diff --git a/pkg/util.go b/pkg/util.go index 5a2ca8e..116450a 100644 --- a/pkg/util.go +++ b/pkg/util.go @@ -19,7 +19,7 @@ package validator import ( "context" - "github.com/ipfs/go-blockservice" + "github.com/ipfs/boxo/blockservice" "github.com/ipfs/kubo/core" "github.com/ipfs/kubo/repo/fsrepo" ) diff --git a/pkg/validator.go b/pkg/validator.go index 4fcfaa0..34bbaaa 100644 --- a/pkg/validator.go +++ b/pkg/validator.go @@ -25,14 +25,12 @@ import ( "syscall" "time" - "github.com/spf13/viper" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/rlp" - "github.com/ipfs/go-blockservice" + "github.com/ipfs/boxo/blockservice" "github.com/jmoiron/sqlx" "github.com/mailgun/groupcache/v2" log "github.com/sirupsen/logrus" @@ -44,12 +42,13 @@ import ( pgipfsethdb "github.com/cerc-io/ipfs-ethdb/v5/postgres/v0" "github.com/cerc-io/ipld-eth-statedb/trie_by_cid/state" "github.com/cerc-io/ipld-eth-statedb/trie_by_cid/trie" + "github.com/cerc-io/ipld-eth-statedb/trie_by_cid/triedb" ) // Validator is used for validating Ethereum state and storage tries on PG-IPFS type Validator struct { kvs ethdb.KeyValueStore - trieDB *trie.Database + trieDB *triedb.Database stateDatabase state.Database db *pgipfsethdb.Database @@ -94,7 +93,7 @@ func NewPGIPFSValidator(db *sqlx.DB, par Params) *Validator { normalizeParams(&par) return &Validator{ kvs: kvs, - trieDB: trie.NewDatabase(NewKVSDatabaseWithAncient(kvs)), + trieDB: triedb.NewDatabase(NewKVSDatabaseWithAncient(kvs), nil), stateDatabase: state.NewDatabase(database), db: database.(*pgipfsethdb.Database), params: par, @@ -112,7 +111,7 @@ func NewIPFSValidator(bs blockservice.BlockService, par Params) *Validator { normalizeParams(&par) return &Validator{ kvs: kvs, - trieDB: trie.NewDatabase(NewKVSDatabaseWithAncient(kvs)), + trieDB: triedb.NewDatabase(NewKVSDatabaseWithAncient(kvs), nil), stateDatabase: state.NewDatabase(database), params: par, } @@ -124,7 +123,7 @@ func NewIPFSValidator(bs blockservice.BlockService, par Params) *Validator { func NewValidator(kvs ethdb.KeyValueStore, database ethdb.Database) *Validator { return &Validator{ kvs: kvs, - trieDB: trie.NewDatabase(NewKVSDatabaseWithAncient(kvs)), + trieDB: triedb.NewDatabase(NewKVSDatabaseWithAncient(kvs), nil), stateDatabase: state.NewDatabase(database), } } @@ -146,7 +145,7 @@ func (v *Validator) ValidateTrie(stateRoot common.Hash) error { if err != nil { return err } - iterate := func(ctx context.Context, it trie.NodeIterator) error { return v.iterate(ctx, it, true) } + iterate := func(ctx context.Context, it trie.NodeIterator) error { return v.iterate(ctx, stateRoot, it, true) } return iterateTracked(t, fmt.Sprintf(v.params.RecoveryFormat, fullTraversal), v.params.Workers, iterate) } @@ -158,7 +157,7 @@ func (v *Validator) ValidateStateTrie(stateRoot common.Hash) error { if err != nil { return err } - iterate := func(ctx context.Context, it trie.NodeIterator) error { return v.iterate(ctx, it, false) } + iterate := func(ctx context.Context, it trie.NodeIterator) error { return v.iterate(ctx, stateRoot, it, false) } return iterateTracked(t, fmt.Sprintf(v.params.RecoveryFormat, stateTraversal), v.params.Workers, iterate) } @@ -166,12 +165,13 @@ func (v *Validator) ValidateStateTrie(stateRoot common.Hash) error { func (v *Validator) ValidateStorageTrie(stateRoot common.Hash, address common.Address, storageRoot common.Hash) error { // Generate the state.NodeIterator for this root addrHash := crypto.Keccak256Hash(address.Bytes()) - t, err := v.stateDatabase.OpenStorageTrie(stateRoot, addrHash, storageRoot) + // Note: the last argument is the redundant state trie, but will be needed for Verkle tries + storage, err := v.stateDatabase.OpenStorageTrie(stateRoot, addrHash, storageRoot, nil) if err != nil { return err } - iterate := func(ctx context.Context, it trie.NodeIterator) error { return v.iterate(ctx, it, false) } - return iterateTracked(t, fmt.Sprintf(v.params.RecoveryFormat, storageTraversal), v.params.Workers, iterate) + iterate := func(ctx context.Context, it trie.NodeIterator) error { return v.iterate(ctx, stateRoot, it, false) } + return iterateTracked(storage, fmt.Sprintf(v.params.RecoveryFormat, storageTraversal), v.params.Workers, iterate) } // Close implements io.Closer @@ -184,7 +184,7 @@ func (v *Validator) Close() error { // Traverses one iterator fully // If storage = true, also traverse storage tries for each leaf. -func (v *Validator) iterate(ctx context.Context, it trie.NodeIterator, storage bool) error { +func (v *Validator) iterate(ctx context.Context, stateRoot common.Hash, it trie.NodeIterator, storage bool) error { // Iterate through entire state trie. it.Next() will return false when we have // either completed iteration of the entire trie or run into an error (e.g. a // missing node). If we are able to iterate through the entire trie without error @@ -206,13 +206,18 @@ func (v *Validator) iterate(ctx context.Context, it trie.NodeIterator, storage b if err := rlp.Decode(bytes.NewReader(it.LeafBlob()), &account); err != nil { return err } - dataTrie, err := v.stateDatabase.OpenStorageTrie(common.HexToHash(viper.GetString("validator.stateRoot")), common.BytesToHash(it.LeafKey()), account.Root) + // Note: the last argument is the redundant state trie, but will be needed for Verkle tries + dataTrie, err := v.stateDatabase.OpenStorageTrie(stateRoot, common.BytesToHash(it.LeafKey()), account.Root, nil) if err != nil { return err } - dataIt := dataTrie.NodeIterator(nil) + dataIt, err := dataTrie.NodeIterator(nil) + if err != nil { + return err + } + if !bytes.Equal(account.CodeHash, emptyCodeHash) { - _, err := v.stateDatabase.ContractCode(common.BytesToHash(account.CodeHash)) + _, err := v.stateDatabase.ContractCode(common.Address{}, common.BytesToHash(account.CodeHash)) if err != nil { return fmt.Errorf("code hash %x: %w (path %x)", account.CodeHash, err, iterutils.HexToKeyBytes(it.Path())) } @@ -252,7 +257,10 @@ func iterateTracked( } if iters == nil { // nothing restored - iters = iterutils.SubtrieIterators(tree.NodeIterator, iterCount) + iters, err = iterutils.SubtrieIterators(tree.NodeIterator, iterCount) + if err != nil { + return err + } for i, it := range iters { iters[i] = tracker.Tracked(it) } diff --git a/pkg/validator_test.go b/pkg/validator_test.go index 229d469..a57c620 100644 --- a/pkg/validator_test.go +++ b/pkg/validator_test.go @@ -17,7 +17,6 @@ package validator_test import ( - "math/big" "os" "path/filepath" @@ -25,6 +24,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/rlp" + "github.com/holiman/uint256" cid "github.com/ipfs/go-cid/_rsrch/cidiface" "github.com/jmoiron/sqlx" "github.com/multiformats/go-multihash" @@ -69,7 +69,7 @@ var ( codePath = common.Hex2Bytes("6114658a74d9cc9f7acf2c5cd696c3494d7c344d78bfec3add0d91ec4e8d1c45") contractAccount, _ = rlp.EncodeToBytes(&types.StateAccount{ Nonce: 1, - Balance: big.NewInt(0), + Balance: uint256.NewInt(0), CodeHash: codeHash.Bytes(), Root: crypto.Keccak256Hash(storageBranchRootNode), }) @@ -80,7 +80,7 @@ var ( minerAccount, _ = rlp.EncodeToBytes(&types.StateAccount{ Nonce: 0, - Balance: big.NewInt(1000), + Balance: uint256.NewInt(1000), CodeHash: nullCodeHash.Bytes(), Root: emptyContractRoot, }) @@ -91,7 +91,7 @@ var ( account1, _ = rlp.EncodeToBytes(&types.StateAccount{ Nonce: 2, - Balance: big.NewInt(1000), + Balance: uint256.NewInt(1000), CodeHash: nullCodeHash.Bytes(), Root: emptyContractRoot, }) @@ -102,7 +102,7 @@ var ( account2, _ = rlp.EncodeToBytes(&types.StateAccount{ Nonce: 0, - Balance: big.NewInt(1000), + Balance: uint256.NewInt(1000), CodeHash: nullCodeHash.Bytes(), Root: emptyContractRoot, }) @@ -113,7 +113,7 @@ var ( bankAccount, _ = rlp.EncodeToBytes(&types.StateAccount{ Nonce: 2, - Balance: big.NewInt(1000), + Balance: uint256.NewInt(1000), CodeHash: nullCodeHash.Bytes(), Root: emptyContractRoot, }) @@ -249,14 +249,14 @@ var _ = Describe("PG-IPFS Validator", func() { err = v.ValidateTrie(stateRoot) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("missing trie node")) - Expect(err.Error()).To(ContainSubstring("%x", missingStateNodePath)) + Expect(err.Error()).To(ContainSubstring("path %x", missingStateNodePath)) }) It("Returns an error if the storage trie is missing node(s)", func() { loadTrie(trieStateNodes, missingNodeStorageNodes, mockCode) err = v.ValidateTrie(stateRoot) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("missing trie node")) - Expect(err.Error()).To(ContainSubstring("%x", missingStorageNodePath)) + Expect(err.Error()).To(ContainSubstring("path %x", missingStorageNodePath)) }) It("Returns an error if contract code is missing", func() { loadTrie(trieStateNodes, trieStorageNodes) @@ -288,6 +288,7 @@ var _ = Describe("PG-IPFS Validator", func() { err = v.ValidateStateTrie(stateRoot) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("missing trie node")) + Expect(err.Error()).To(ContainSubstring("path %x", missingStateNodePath)) }) It("Returns no error if the entire state trie can be validated", func() { loadTrie(trieStateNodes, nil) @@ -302,19 +303,20 @@ var _ = Describe("PG-IPFS Validator", func() { Expect(err).ToNot(HaveOccurred()) }) It("Returns an error the storage root node is missing", func() { - loadTrie(nil, missingRootStorageNodes) + loadTrie(trieStateNodes, missingRootStorageNodes) err = v.ValidateStorageTrie(stateRoot, contractAddr, storageRoot) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("missing trie node")) }) It("Returns an error if the entire storage trie cannot be validated", func() { - loadTrie(nil, missingNodeStorageNodes) + loadTrie(trieStateNodes, missingNodeStorageNodes) err = v.ValidateStorageTrie(stateRoot, contractAddr, storageRoot) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("missing trie node")) + Expect(err.Error()).To(ContainSubstring("path %x", missingStorageNodePath)) }) It("Returns no error if the entire storage trie can be validated", func() { - loadTrie(nil, trieStorageNodes) + loadTrie(trieStateNodes, trieStorageNodes) err = v.ValidateStorageTrie(stateRoot, contractAddr, storageRoot) Expect(err).ToNot(HaveOccurred()) })