updates, fixes
This commit is contained in:
parent
a58101e1cd
commit
1bc449d036
@ -19,7 +19,7 @@ package validator
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
|
||||||
"github.com/ipfs/go-blockservice"
|
"github.com/ipfs/boxo/blockservice"
|
||||||
"github.com/ipfs/kubo/core"
|
"github.com/ipfs/kubo/core"
|
||||||
"github.com/ipfs/kubo/repo/fsrepo"
|
"github.com/ipfs/kubo/repo/fsrepo"
|
||||||
)
|
)
|
||||||
|
@ -25,14 +25,12 @@ import (
|
|||||||
"syscall"
|
"syscall"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/spf13/viper"
|
|
||||||
|
|
||||||
"github.com/ethereum/go-ethereum/common"
|
"github.com/ethereum/go-ethereum/common"
|
||||||
"github.com/ethereum/go-ethereum/core/types"
|
"github.com/ethereum/go-ethereum/core/types"
|
||||||
"github.com/ethereum/go-ethereum/crypto"
|
"github.com/ethereum/go-ethereum/crypto"
|
||||||
"github.com/ethereum/go-ethereum/ethdb"
|
"github.com/ethereum/go-ethereum/ethdb"
|
||||||
"github.com/ethereum/go-ethereum/rlp"
|
"github.com/ethereum/go-ethereum/rlp"
|
||||||
"github.com/ipfs/go-blockservice"
|
"github.com/ipfs/boxo/blockservice"
|
||||||
"github.com/jmoiron/sqlx"
|
"github.com/jmoiron/sqlx"
|
||||||
"github.com/mailgun/groupcache/v2"
|
"github.com/mailgun/groupcache/v2"
|
||||||
log "github.com/sirupsen/logrus"
|
log "github.com/sirupsen/logrus"
|
||||||
@ -44,12 +42,13 @@ import (
|
|||||||
pgipfsethdb "github.com/cerc-io/ipfs-ethdb/v5/postgres/v0"
|
pgipfsethdb "github.com/cerc-io/ipfs-ethdb/v5/postgres/v0"
|
||||||
"github.com/cerc-io/ipld-eth-statedb/trie_by_cid/state"
|
"github.com/cerc-io/ipld-eth-statedb/trie_by_cid/state"
|
||||||
"github.com/cerc-io/ipld-eth-statedb/trie_by_cid/trie"
|
"github.com/cerc-io/ipld-eth-statedb/trie_by_cid/trie"
|
||||||
|
"github.com/cerc-io/ipld-eth-statedb/trie_by_cid/triedb"
|
||||||
)
|
)
|
||||||
|
|
||||||
// Validator is used for validating Ethereum state and storage tries on PG-IPFS
|
// Validator is used for validating Ethereum state and storage tries on PG-IPFS
|
||||||
type Validator struct {
|
type Validator struct {
|
||||||
kvs ethdb.KeyValueStore
|
kvs ethdb.KeyValueStore
|
||||||
trieDB *trie.Database
|
trieDB *triedb.Database
|
||||||
stateDatabase state.Database
|
stateDatabase state.Database
|
||||||
db *pgipfsethdb.Database
|
db *pgipfsethdb.Database
|
||||||
|
|
||||||
@ -94,7 +93,7 @@ func NewPGIPFSValidator(db *sqlx.DB, par Params) *Validator {
|
|||||||
normalizeParams(&par)
|
normalizeParams(&par)
|
||||||
return &Validator{
|
return &Validator{
|
||||||
kvs: kvs,
|
kvs: kvs,
|
||||||
trieDB: trie.NewDatabase(NewKVSDatabaseWithAncient(kvs)),
|
trieDB: triedb.NewDatabase(NewKVSDatabaseWithAncient(kvs), nil),
|
||||||
stateDatabase: state.NewDatabase(database),
|
stateDatabase: state.NewDatabase(database),
|
||||||
db: database.(*pgipfsethdb.Database),
|
db: database.(*pgipfsethdb.Database),
|
||||||
params: par,
|
params: par,
|
||||||
@ -112,7 +111,7 @@ func NewIPFSValidator(bs blockservice.BlockService, par Params) *Validator {
|
|||||||
normalizeParams(&par)
|
normalizeParams(&par)
|
||||||
return &Validator{
|
return &Validator{
|
||||||
kvs: kvs,
|
kvs: kvs,
|
||||||
trieDB: trie.NewDatabase(NewKVSDatabaseWithAncient(kvs)),
|
trieDB: triedb.NewDatabase(NewKVSDatabaseWithAncient(kvs), nil),
|
||||||
stateDatabase: state.NewDatabase(database),
|
stateDatabase: state.NewDatabase(database),
|
||||||
params: par,
|
params: par,
|
||||||
}
|
}
|
||||||
@ -124,7 +123,7 @@ func NewIPFSValidator(bs blockservice.BlockService, par Params) *Validator {
|
|||||||
func NewValidator(kvs ethdb.KeyValueStore, database ethdb.Database) *Validator {
|
func NewValidator(kvs ethdb.KeyValueStore, database ethdb.Database) *Validator {
|
||||||
return &Validator{
|
return &Validator{
|
||||||
kvs: kvs,
|
kvs: kvs,
|
||||||
trieDB: trie.NewDatabase(NewKVSDatabaseWithAncient(kvs)),
|
trieDB: triedb.NewDatabase(NewKVSDatabaseWithAncient(kvs), nil),
|
||||||
stateDatabase: state.NewDatabase(database),
|
stateDatabase: state.NewDatabase(database),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -146,7 +145,7 @@ func (v *Validator) ValidateTrie(stateRoot common.Hash) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
iterate := func(ctx context.Context, it trie.NodeIterator) error { return v.iterate(ctx, it, true) }
|
iterate := func(ctx context.Context, it trie.NodeIterator) error { return v.iterate(ctx, stateRoot, it, true) }
|
||||||
return iterateTracked(t, fmt.Sprintf(v.params.RecoveryFormat, fullTraversal), v.params.Workers, iterate)
|
return iterateTracked(t, fmt.Sprintf(v.params.RecoveryFormat, fullTraversal), v.params.Workers, iterate)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -158,7 +157,7 @@ func (v *Validator) ValidateStateTrie(stateRoot common.Hash) error {
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
iterate := func(ctx context.Context, it trie.NodeIterator) error { return v.iterate(ctx, it, false) }
|
iterate := func(ctx context.Context, it trie.NodeIterator) error { return v.iterate(ctx, stateRoot, it, false) }
|
||||||
return iterateTracked(t, fmt.Sprintf(v.params.RecoveryFormat, stateTraversal), v.params.Workers, iterate)
|
return iterateTracked(t, fmt.Sprintf(v.params.RecoveryFormat, stateTraversal), v.params.Workers, iterate)
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -166,12 +165,13 @@ func (v *Validator) ValidateStateTrie(stateRoot common.Hash) error {
|
|||||||
func (v *Validator) ValidateStorageTrie(stateRoot common.Hash, address common.Address, storageRoot common.Hash) error {
|
func (v *Validator) ValidateStorageTrie(stateRoot common.Hash, address common.Address, storageRoot common.Hash) error {
|
||||||
// Generate the state.NodeIterator for this root
|
// Generate the state.NodeIterator for this root
|
||||||
addrHash := crypto.Keccak256Hash(address.Bytes())
|
addrHash := crypto.Keccak256Hash(address.Bytes())
|
||||||
t, err := v.stateDatabase.OpenStorageTrie(stateRoot, addrHash, storageRoot)
|
// Note: the last argument is the redundant state trie, but will be needed for Verkle tries
|
||||||
|
storage, err := v.stateDatabase.OpenStorageTrie(stateRoot, addrHash, storageRoot, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
iterate := func(ctx context.Context, it trie.NodeIterator) error { return v.iterate(ctx, it, false) }
|
iterate := func(ctx context.Context, it trie.NodeIterator) error { return v.iterate(ctx, stateRoot, it, false) }
|
||||||
return iterateTracked(t, fmt.Sprintf(v.params.RecoveryFormat, storageTraversal), v.params.Workers, iterate)
|
return iterateTracked(storage, fmt.Sprintf(v.params.RecoveryFormat, storageTraversal), v.params.Workers, iterate)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Close implements io.Closer
|
// Close implements io.Closer
|
||||||
@ -184,7 +184,7 @@ func (v *Validator) Close() error {
|
|||||||
|
|
||||||
// Traverses one iterator fully
|
// Traverses one iterator fully
|
||||||
// If storage = true, also traverse storage tries for each leaf.
|
// If storage = true, also traverse storage tries for each leaf.
|
||||||
func (v *Validator) iterate(ctx context.Context, it trie.NodeIterator, storage bool) error {
|
func (v *Validator) iterate(ctx context.Context, stateRoot common.Hash, it trie.NodeIterator, storage bool) error {
|
||||||
// Iterate through entire state trie. it.Next() will return false when we have
|
// Iterate through entire state trie. it.Next() will return false when we have
|
||||||
// either completed iteration of the entire trie or run into an error (e.g. a
|
// either completed iteration of the entire trie or run into an error (e.g. a
|
||||||
// missing node). If we are able to iterate through the entire trie without error
|
// missing node). If we are able to iterate through the entire trie without error
|
||||||
@ -206,13 +206,18 @@ func (v *Validator) iterate(ctx context.Context, it trie.NodeIterator, storage b
|
|||||||
if err := rlp.Decode(bytes.NewReader(it.LeafBlob()), &account); err != nil {
|
if err := rlp.Decode(bytes.NewReader(it.LeafBlob()), &account); err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
dataTrie, err := v.stateDatabase.OpenStorageTrie(common.HexToHash(viper.GetString("validator.stateRoot")), common.BytesToHash(it.LeafKey()), account.Root)
|
// Note: the last argument is the redundant state trie, but will be needed for Verkle tries
|
||||||
|
dataTrie, err := v.stateDatabase.OpenStorageTrie(stateRoot, common.BytesToHash(it.LeafKey()), account.Root, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
||||||
}
|
}
|
||||||
dataIt := dataTrie.NodeIterator(nil)
|
dataIt, err := dataTrie.NodeIterator(nil)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
if !bytes.Equal(account.CodeHash, emptyCodeHash) {
|
if !bytes.Equal(account.CodeHash, emptyCodeHash) {
|
||||||
_, err := v.stateDatabase.ContractCode(common.BytesToHash(account.CodeHash))
|
_, err := v.stateDatabase.ContractCode(common.Address{}, common.BytesToHash(account.CodeHash))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return fmt.Errorf("code hash %x: %w (path %x)", account.CodeHash, err, iterutils.HexToKeyBytes(it.Path()))
|
return fmt.Errorf("code hash %x: %w (path %x)", account.CodeHash, err, iterutils.HexToKeyBytes(it.Path()))
|
||||||
}
|
}
|
||||||
@ -252,7 +257,10 @@ func iterateTracked(
|
|||||||
}
|
}
|
||||||
|
|
||||||
if iters == nil { // nothing restored
|
if iters == nil { // nothing restored
|
||||||
iters = iterutils.SubtrieIterators(tree.NodeIterator, iterCount)
|
iters, err = iterutils.SubtrieIterators(tree.NodeIterator, iterCount)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
for i, it := range iters {
|
for i, it := range iters {
|
||||||
iters[i] = tracker.Tracked(it)
|
iters[i] = tracker.Tracked(it)
|
||||||
}
|
}
|
||||||
|
@ -17,7 +17,6 @@
|
|||||||
package validator_test
|
package validator_test
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"math/big"
|
|
||||||
"os"
|
"os"
|
||||||
"path/filepath"
|
"path/filepath"
|
||||||
|
|
||||||
@ -25,6 +24,7 @@ import (
|
|||||||
"github.com/ethereum/go-ethereum/core/types"
|
"github.com/ethereum/go-ethereum/core/types"
|
||||||
"github.com/ethereum/go-ethereum/crypto"
|
"github.com/ethereum/go-ethereum/crypto"
|
||||||
"github.com/ethereum/go-ethereum/rlp"
|
"github.com/ethereum/go-ethereum/rlp"
|
||||||
|
"github.com/holiman/uint256"
|
||||||
cid "github.com/ipfs/go-cid/_rsrch/cidiface"
|
cid "github.com/ipfs/go-cid/_rsrch/cidiface"
|
||||||
"github.com/jmoiron/sqlx"
|
"github.com/jmoiron/sqlx"
|
||||||
"github.com/multiformats/go-multihash"
|
"github.com/multiformats/go-multihash"
|
||||||
@ -69,7 +69,7 @@ var (
|
|||||||
codePath = common.Hex2Bytes("6114658a74d9cc9f7acf2c5cd696c3494d7c344d78bfec3add0d91ec4e8d1c45")
|
codePath = common.Hex2Bytes("6114658a74d9cc9f7acf2c5cd696c3494d7c344d78bfec3add0d91ec4e8d1c45")
|
||||||
contractAccount, _ = rlp.EncodeToBytes(&types.StateAccount{
|
contractAccount, _ = rlp.EncodeToBytes(&types.StateAccount{
|
||||||
Nonce: 1,
|
Nonce: 1,
|
||||||
Balance: big.NewInt(0),
|
Balance: uint256.NewInt(0),
|
||||||
CodeHash: codeHash.Bytes(),
|
CodeHash: codeHash.Bytes(),
|
||||||
Root: crypto.Keccak256Hash(storageBranchRootNode),
|
Root: crypto.Keccak256Hash(storageBranchRootNode),
|
||||||
})
|
})
|
||||||
@ -80,7 +80,7 @@ var (
|
|||||||
|
|
||||||
minerAccount, _ = rlp.EncodeToBytes(&types.StateAccount{
|
minerAccount, _ = rlp.EncodeToBytes(&types.StateAccount{
|
||||||
Nonce: 0,
|
Nonce: 0,
|
||||||
Balance: big.NewInt(1000),
|
Balance: uint256.NewInt(1000),
|
||||||
CodeHash: nullCodeHash.Bytes(),
|
CodeHash: nullCodeHash.Bytes(),
|
||||||
Root: emptyContractRoot,
|
Root: emptyContractRoot,
|
||||||
})
|
})
|
||||||
@ -91,7 +91,7 @@ var (
|
|||||||
|
|
||||||
account1, _ = rlp.EncodeToBytes(&types.StateAccount{
|
account1, _ = rlp.EncodeToBytes(&types.StateAccount{
|
||||||
Nonce: 2,
|
Nonce: 2,
|
||||||
Balance: big.NewInt(1000),
|
Balance: uint256.NewInt(1000),
|
||||||
CodeHash: nullCodeHash.Bytes(),
|
CodeHash: nullCodeHash.Bytes(),
|
||||||
Root: emptyContractRoot,
|
Root: emptyContractRoot,
|
||||||
})
|
})
|
||||||
@ -102,7 +102,7 @@ var (
|
|||||||
|
|
||||||
account2, _ = rlp.EncodeToBytes(&types.StateAccount{
|
account2, _ = rlp.EncodeToBytes(&types.StateAccount{
|
||||||
Nonce: 0,
|
Nonce: 0,
|
||||||
Balance: big.NewInt(1000),
|
Balance: uint256.NewInt(1000),
|
||||||
CodeHash: nullCodeHash.Bytes(),
|
CodeHash: nullCodeHash.Bytes(),
|
||||||
Root: emptyContractRoot,
|
Root: emptyContractRoot,
|
||||||
})
|
})
|
||||||
@ -113,7 +113,7 @@ var (
|
|||||||
|
|
||||||
bankAccount, _ = rlp.EncodeToBytes(&types.StateAccount{
|
bankAccount, _ = rlp.EncodeToBytes(&types.StateAccount{
|
||||||
Nonce: 2,
|
Nonce: 2,
|
||||||
Balance: big.NewInt(1000),
|
Balance: uint256.NewInt(1000),
|
||||||
CodeHash: nullCodeHash.Bytes(),
|
CodeHash: nullCodeHash.Bytes(),
|
||||||
Root: emptyContractRoot,
|
Root: emptyContractRoot,
|
||||||
})
|
})
|
||||||
@ -249,14 +249,14 @@ var _ = Describe("PG-IPFS Validator", func() {
|
|||||||
err = v.ValidateTrie(stateRoot)
|
err = v.ValidateTrie(stateRoot)
|
||||||
Expect(err).To(HaveOccurred())
|
Expect(err).To(HaveOccurred())
|
||||||
Expect(err.Error()).To(ContainSubstring("missing trie node"))
|
Expect(err.Error()).To(ContainSubstring("missing trie node"))
|
||||||
Expect(err.Error()).To(ContainSubstring("%x", missingStateNodePath))
|
Expect(err.Error()).To(ContainSubstring("path %x", missingStateNodePath))
|
||||||
})
|
})
|
||||||
It("Returns an error if the storage trie is missing node(s)", func() {
|
It("Returns an error if the storage trie is missing node(s)", func() {
|
||||||
loadTrie(trieStateNodes, missingNodeStorageNodes, mockCode)
|
loadTrie(trieStateNodes, missingNodeStorageNodes, mockCode)
|
||||||
err = v.ValidateTrie(stateRoot)
|
err = v.ValidateTrie(stateRoot)
|
||||||
Expect(err).To(HaveOccurred())
|
Expect(err).To(HaveOccurred())
|
||||||
Expect(err.Error()).To(ContainSubstring("missing trie node"))
|
Expect(err.Error()).To(ContainSubstring("missing trie node"))
|
||||||
Expect(err.Error()).To(ContainSubstring("%x", missingStorageNodePath))
|
Expect(err.Error()).To(ContainSubstring("path %x", missingStorageNodePath))
|
||||||
})
|
})
|
||||||
It("Returns an error if contract code is missing", func() {
|
It("Returns an error if contract code is missing", func() {
|
||||||
loadTrie(trieStateNodes, trieStorageNodes)
|
loadTrie(trieStateNodes, trieStorageNodes)
|
||||||
@ -288,6 +288,7 @@ var _ = Describe("PG-IPFS Validator", func() {
|
|||||||
err = v.ValidateStateTrie(stateRoot)
|
err = v.ValidateStateTrie(stateRoot)
|
||||||
Expect(err).To(HaveOccurred())
|
Expect(err).To(HaveOccurred())
|
||||||
Expect(err.Error()).To(ContainSubstring("missing trie node"))
|
Expect(err.Error()).To(ContainSubstring("missing trie node"))
|
||||||
|
Expect(err.Error()).To(ContainSubstring("path %x", missingStateNodePath))
|
||||||
})
|
})
|
||||||
It("Returns no error if the entire state trie can be validated", func() {
|
It("Returns no error if the entire state trie can be validated", func() {
|
||||||
loadTrie(trieStateNodes, nil)
|
loadTrie(trieStateNodes, nil)
|
||||||
@ -302,19 +303,20 @@ var _ = Describe("PG-IPFS Validator", func() {
|
|||||||
Expect(err).ToNot(HaveOccurred())
|
Expect(err).ToNot(HaveOccurred())
|
||||||
})
|
})
|
||||||
It("Returns an error the storage root node is missing", func() {
|
It("Returns an error the storage root node is missing", func() {
|
||||||
loadTrie(nil, missingRootStorageNodes)
|
loadTrie(trieStateNodes, missingRootStorageNodes)
|
||||||
err = v.ValidateStorageTrie(stateRoot, contractAddr, storageRoot)
|
err = v.ValidateStorageTrie(stateRoot, contractAddr, storageRoot)
|
||||||
Expect(err).To(HaveOccurred())
|
Expect(err).To(HaveOccurred())
|
||||||
Expect(err.Error()).To(ContainSubstring("missing trie node"))
|
Expect(err.Error()).To(ContainSubstring("missing trie node"))
|
||||||
})
|
})
|
||||||
It("Returns an error if the entire storage trie cannot be validated", func() {
|
It("Returns an error if the entire storage trie cannot be validated", func() {
|
||||||
loadTrie(nil, missingNodeStorageNodes)
|
loadTrie(trieStateNodes, missingNodeStorageNodes)
|
||||||
err = v.ValidateStorageTrie(stateRoot, contractAddr, storageRoot)
|
err = v.ValidateStorageTrie(stateRoot, contractAddr, storageRoot)
|
||||||
Expect(err).To(HaveOccurred())
|
Expect(err).To(HaveOccurred())
|
||||||
Expect(err.Error()).To(ContainSubstring("missing trie node"))
|
Expect(err.Error()).To(ContainSubstring("missing trie node"))
|
||||||
|
Expect(err.Error()).To(ContainSubstring("path %x", missingStorageNodePath))
|
||||||
})
|
})
|
||||||
It("Returns no error if the entire storage trie can be validated", func() {
|
It("Returns no error if the entire storage trie can be validated", func() {
|
||||||
loadTrie(nil, trieStorageNodes)
|
loadTrie(trieStateNodes, trieStorageNodes)
|
||||||
err = v.ValidateStorageTrie(stateRoot, contractAddr, storageRoot)
|
err = v.ValidateStorageTrie(stateRoot, contractAddr, storageRoot)
|
||||||
Expect(err).ToNot(HaveOccurred())
|
Expect(err).ToNot(HaveOccurred())
|
||||||
})
|
})
|
||||||
|
Loading…
Reference in New Issue
Block a user