From fa8d39807dc557d9a32a5c8e311ef1209b809e82 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet <3272758+gballet@users.noreply.github.com> Date: Tue, 14 Nov 2023 13:09:40 +0100 Subject: [PATCH] cmd, core, trie: verkle-capable `geth init` (#28270) This change allows the creation of a genesis block for verkle testnets. This makes for a chunk of code that is easier to review and still touches many discussion points. --- cmd/geth/chaincmd.go | 4 +- cmd/geth/dbcmd.go | 2 +- cmd/geth/snapshot.go | 8 +- cmd/geth/verkle.go | 6 +- cmd/utils/flags.go | 3 +- core/genesis.go | 23 ++- core/genesis_test.go | 66 +++++- core/state/database.go | 51 +++-- core/state/iterator.go | 2 +- core/state/state_object.go | 2 +- core/state/statedb.go | 2 +- core/state/trie_prefetcher.go | 4 +- core/types/hashes.go | 5 +- go.mod | 8 +- go.sum | 21 +- les/server_requests.go | 2 +- light/odr_test.go | 2 +- light/trie.go | 2 +- trie/database.go | 6 + trie/trienode/node.go | 2 +- trie/utils/verkle.go | 342 +++++++++++++++++++++++++++++++ trie/utils/verkle_test.go | 139 +++++++++++++ trie/verkle.go | 375 ++++++++++++++++++++++++++++++++++ trie/verkle_test.go | 97 +++++++++ 24 files changed, 1120 insertions(+), 54 deletions(-) create mode 100644 trie/utils/verkle.go create mode 100644 trie/utils/verkle_test.go create mode 100644 trie/verkle.go create mode 100644 trie/verkle_test.go diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index 5663963e3..b65827f5b 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -211,7 +211,7 @@ func initGenesis(ctx *cli.Context) error { } defer chaindb.Close() - triedb := utils.MakeTrieDatabase(ctx, chaindb, ctx.Bool(utils.CachePreimagesFlag.Name), false) + triedb := utils.MakeTrieDatabase(ctx, chaindb, ctx.Bool(utils.CachePreimagesFlag.Name), false, genesis.IsVerkle()) defer triedb.Close() _, hash, err := core.SetupGenesisBlockWithOverride(chaindb, triedb, genesis, &overrides) @@ -485,7 +485,7 @@ func dump(ctx *cli.Context) error { if err != nil { return err } - triedb := utils.MakeTrieDatabase(ctx, db, true, true) // always enable preimage lookup + triedb := utils.MakeTrieDatabase(ctx, db, true, true, false) // always enable preimage lookup defer triedb.Close() state, err := state.New(root, state.NewDatabaseWithNodeDB(db, triedb), nil) diff --git a/cmd/geth/dbcmd.go b/cmd/geth/dbcmd.go index ab2626c12..c60147b86 100644 --- a/cmd/geth/dbcmd.go +++ b/cmd/geth/dbcmd.go @@ -482,7 +482,7 @@ func dbDumpTrie(ctx *cli.Context) error { db := utils.MakeChainDatabase(ctx, stack, true) defer db.Close() - triedb := utils.MakeTrieDatabase(ctx, db, false, true) + triedb := utils.MakeTrieDatabase(ctx, db, false, true, false) defer triedb.Close() var ( diff --git a/cmd/geth/snapshot.go b/cmd/geth/snapshot.go index 25c6311c4..82beb4f2e 100644 --- a/cmd/geth/snapshot.go +++ b/cmd/geth/snapshot.go @@ -205,7 +205,7 @@ func verifyState(ctx *cli.Context) error { log.Error("Failed to load head block") return errors.New("no head block") } - triedb := utils.MakeTrieDatabase(ctx, chaindb, false, true) + triedb := utils.MakeTrieDatabase(ctx, chaindb, false, true, false) defer triedb.Close() snapConfig := snapshot.Config{ @@ -260,7 +260,7 @@ func traverseState(ctx *cli.Context) error { chaindb := utils.MakeChainDatabase(ctx, stack, true) defer chaindb.Close() - triedb := utils.MakeTrieDatabase(ctx, chaindb, false, true) + triedb := utils.MakeTrieDatabase(ctx, chaindb, false, true, false) defer triedb.Close() headBlock := rawdb.ReadHeadBlock(chaindb) @@ -369,7 +369,7 @@ func traverseRawState(ctx *cli.Context) error { chaindb := utils.MakeChainDatabase(ctx, stack, true) defer chaindb.Close() - triedb := utils.MakeTrieDatabase(ctx, chaindb, false, true) + triedb := utils.MakeTrieDatabase(ctx, chaindb, false, true, false) defer triedb.Close() headBlock := rawdb.ReadHeadBlock(chaindb) @@ -533,7 +533,7 @@ func dumpState(ctx *cli.Context) error { if err != nil { return err } - triedb := utils.MakeTrieDatabase(ctx, db, false, true) + triedb := utils.MakeTrieDatabase(ctx, db, false, true, false) defer triedb.Close() snapConfig := snapshot.Config{ diff --git a/cmd/geth/verkle.go b/cmd/geth/verkle.go index aa79889e8..420b063d8 100644 --- a/cmd/geth/verkle.go +++ b/cmd/geth/verkle.go @@ -84,7 +84,7 @@ func checkChildren(root verkle.VerkleNode, resolver verkle.NodeResolverFn) error return fmt.Errorf("could not find child %x in db: %w", childC, err) } // depth is set to 0, the tree isn't rebuilt so it's not a problem - childN, err := verkle.ParseNode(childS, 0, childC[:]) + childN, err := verkle.ParseNode(childS, 0) if err != nil { return fmt.Errorf("decode error child %x in db: %w", child.Commitment().Bytes(), err) } @@ -145,7 +145,7 @@ func verifyVerkle(ctx *cli.Context) error { if err != nil { return err } - root, err := verkle.ParseNode(serializedRoot, 0, rootC[:]) + root, err := verkle.ParseNode(serializedRoot, 0) if err != nil { return err } @@ -195,7 +195,7 @@ func expandVerkle(ctx *cli.Context) error { if err != nil { return err } - root, err := verkle.ParseNode(serializedRoot, 0, rootC[:]) + root, err := verkle.ParseNode(serializedRoot, 0) if err != nil { return err } diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index e9a7c7c11..8bbacac51 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -2212,9 +2212,10 @@ func MakeConsolePreloads(ctx *cli.Context) []string { } // MakeTrieDatabase constructs a trie database based on the configured scheme. -func MakeTrieDatabase(ctx *cli.Context, disk ethdb.Database, preimage bool, readOnly bool) *trie.Database { +func MakeTrieDatabase(ctx *cli.Context, disk ethdb.Database, preimage bool, readOnly bool, isVerkle bool) *trie.Database { config := &trie.Config{ Preimages: preimage, + IsVerkle: isVerkle, } scheme, err := rawdb.ParseStateScheme(ctx.String(StateSchemeFlag.Name), disk) if err != nil { diff --git a/core/genesis.go b/core/genesis.go index 1045815fa..60c2f9a8b 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -37,6 +37,7 @@ import ( "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" "github.com/ethereum/go-ethereum/trie" + "github.com/ethereum/go-ethereum/trie/triedb/pathdb" ) //go:generate go run github.com/fjl/gencodec -type Genesis -field-override genesisSpecMarshaling -out gen_genesis.go @@ -121,10 +122,20 @@ func (ga *GenesisAlloc) UnmarshalJSON(data []byte) error { } // hash computes the state root according to the genesis specification. -func (ga *GenesisAlloc) hash() (common.Hash, error) { +func (ga *GenesisAlloc) hash(isVerkle bool) (common.Hash, error) { + // If a genesis-time verkle trie is requested, create a trie config + // with the verkle trie enabled so that the tree can be initialized + // as such. + var config *trie.Config + if isVerkle { + config = &trie.Config{ + PathDB: pathdb.Defaults, + IsVerkle: true, + } + } // Create an ephemeral in-memory database for computing hash, // all the derived states will be discarded to not pollute disk. - db := state.NewDatabase(rawdb.NewMemoryDatabase()) + db := state.NewDatabaseWithConfig(rawdb.NewMemoryDatabase(), config) statedb, err := state.New(types.EmptyRootHash, db, nil) if err != nil { return common.Hash{}, err @@ -410,9 +421,15 @@ func (g *Genesis) configOrDefault(ghash common.Hash) *params.ChainConfig { } } +// IsVerkle indicates whether the state is already stored in a verkle +// tree at genesis time. +func (g *Genesis) IsVerkle() bool { + return g.Config.IsVerkle(new(big.Int).SetUint64(g.Number), g.Timestamp) +} + // ToBlock returns the genesis block according to genesis specification. func (g *Genesis) ToBlock() *types.Block { - root, err := g.Alloc.hash() + root, err := g.Alloc.hash(g.IsVerkle()) if err != nil { panic(err) } diff --git a/core/genesis_test.go b/core/genesis_test.go index fac88ff37..1d85b510c 100644 --- a/core/genesis_test.go +++ b/core/genesis_test.go @@ -17,6 +17,7 @@ package core import ( + "bytes" "encoding/json" "math/big" "reflect" @@ -231,7 +232,7 @@ func TestReadWriteGenesisAlloc(t *testing.T) { {1}: {Balance: big.NewInt(1), Storage: map[common.Hash]common.Hash{{1}: {1}}}, {2}: {Balance: big.NewInt(2), Storage: map[common.Hash]common.Hash{{2}: {2}}}, } - hash, _ = alloc.hash() + hash, _ = alloc.hash(false) ) blob, _ := json.Marshal(alloc) rawdb.WriteGenesisStateSpec(db, hash, blob) @@ -261,3 +262,66 @@ func newDbConfig(scheme string) *trie.Config { } return &trie.Config{PathDB: pathdb.Defaults} } + +func TestVerkleGenesisCommit(t *testing.T) { + var verkleTime uint64 = 0 + verkleConfig := ¶ms.ChainConfig{ + ChainID: big.NewInt(1), + HomesteadBlock: big.NewInt(0), + DAOForkBlock: nil, + DAOForkSupport: false, + EIP150Block: big.NewInt(0), + EIP155Block: big.NewInt(0), + EIP158Block: big.NewInt(0), + ByzantiumBlock: big.NewInt(0), + ConstantinopleBlock: big.NewInt(0), + PetersburgBlock: big.NewInt(0), + IstanbulBlock: big.NewInt(0), + MuirGlacierBlock: big.NewInt(0), + BerlinBlock: big.NewInt(0), + LondonBlock: big.NewInt(0), + ArrowGlacierBlock: big.NewInt(0), + GrayGlacierBlock: big.NewInt(0), + MergeNetsplitBlock: nil, + ShanghaiTime: &verkleTime, + CancunTime: &verkleTime, + PragueTime: &verkleTime, + VerkleTime: &verkleTime, + TerminalTotalDifficulty: big.NewInt(0), + TerminalTotalDifficultyPassed: true, + Ethash: nil, + Clique: nil, + } + + genesis := &Genesis{ + BaseFee: big.NewInt(params.InitialBaseFee), + Config: verkleConfig, + Timestamp: verkleTime, + Difficulty: big.NewInt(0), + Alloc: GenesisAlloc{ + {1}: {Balance: big.NewInt(1), Storage: map[common.Hash]common.Hash{{1}: {1}}}, + }, + } + + expected := common.Hex2Bytes("14398d42be3394ff8d50681816a4b7bf8d8283306f577faba2d5bc57498de23b") + got := genesis.ToBlock().Root().Bytes() + if !bytes.Equal(got, expected) { + t.Fatalf("invalid genesis state root, expected %x, got %x", expected, got) + } + + db := rawdb.NewMemoryDatabase() + triedb := trie.NewDatabase(db, &trie.Config{IsVerkle: true, PathDB: pathdb.Defaults}) + block := genesis.MustCommit(db, triedb) + if !bytes.Equal(block.Root().Bytes(), expected) { + t.Fatalf("invalid genesis state root, expected %x, got %x", expected, got) + } + + // Test that the trie is verkle + if !triedb.IsVerkle() { + t.Fatalf("expected trie to be verkle") + } + + if !rawdb.ExistsAccountTrieNode(db, nil) { + t.Fatal("could not find node") + } +} diff --git a/core/state/database.go b/core/state/database.go index 9467c8f72..b55f870d9 100644 --- a/core/state/database.go +++ b/core/state/database.go @@ -20,6 +20,7 @@ import ( "errors" "fmt" + "github.com/crate-crypto/go-ipa/banderwagon" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/common/lru" "github.com/ethereum/go-ethereum/core/rawdb" @@ -28,6 +29,7 @@ import ( "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/trie" "github.com/ethereum/go-ethereum/trie/trienode" + "github.com/ethereum/go-ethereum/trie/utils" ) const ( @@ -36,6 +38,12 @@ const ( // Cache size granted for caching clean code. codeCacheSize = 64 * 1024 * 1024 + + // commitmentSize is the size of commitment stored in cache. + commitmentSize = banderwagon.UncompressedSize + + // Cache item granted for caching commitment results. + commitmentCacheItems = 64 * 1024 * 1024 / (commitmentSize + common.AddressLength) ) // Database wraps access to tries and contract code. @@ -44,7 +52,7 @@ type Database interface { OpenTrie(root common.Hash) (Trie, error) // OpenStorageTrie opens the storage trie of an account. - OpenStorageTrie(stateRoot common.Hash, address common.Address, root common.Hash) (Trie, error) + OpenStorageTrie(stateRoot common.Hash, address common.Address, root common.Hash, trie Trie) (Trie, error) // CopyTrie returns an independent copy of the given trie. CopyTrie(Trie) Trie @@ -70,11 +78,6 @@ type Trie interface { // TODO(fjl): remove this when StateTrie is removed GetKey([]byte) []byte - // GetStorage returns the value for key stored in the trie. The value bytes - // must not be modified by the caller. If a node was not found in the database, - // a trie.MissingNodeError is returned. - GetStorage(addr common.Address, key []byte) ([]byte, error) - // GetAccount abstracts an account read from the trie. It retrieves the // account blob from the trie with provided account address and decodes it // with associated decoding algorithm. If the specified account is not in @@ -83,27 +86,32 @@ type Trie interface { // be returned. GetAccount(address common.Address) (*types.StateAccount, error) - // UpdateStorage associates key with value in the trie. If value has length zero, - // any existing value is deleted from the trie. The value bytes must not be modified - // by the caller while they are stored in the trie. If a node was not found in the - // database, a trie.MissingNodeError is returned. - UpdateStorage(addr common.Address, key, value []byte) error + // GetStorage returns the value for key stored in the trie. The value bytes + // must not be modified by the caller. If a node was not found in the database, + // a trie.MissingNodeError is returned. + GetStorage(addr common.Address, key []byte) ([]byte, error) // UpdateAccount abstracts an account write to the trie. It encodes the // provided account object with associated algorithm and then updates it // in the trie with provided address. UpdateAccount(address common.Address, account *types.StateAccount) error - // UpdateContractCode abstracts code write to the trie. It is expected - // to be moved to the stateWriter interface when the latter is ready. - UpdateContractCode(address common.Address, codeHash common.Hash, code []byte) error + // UpdateStorage associates key with value in the trie. If value has length zero, + // any existing value is deleted from the trie. The value bytes must not be modified + // by the caller while they are stored in the trie. If a node was not found in the + // database, a trie.MissingNodeError is returned. + UpdateStorage(addr common.Address, key, value []byte) error + + // DeleteAccount abstracts an account deletion from the trie. + DeleteAccount(address common.Address) error // DeleteStorage removes any existing value for key from the trie. If a node // was not found in the database, a trie.MissingNodeError is returned. DeleteStorage(addr common.Address, key []byte) error - // DeleteAccount abstracts an account deletion from the trie. - DeleteAccount(address common.Address) error + // UpdateContractCode abstracts code write to the trie. It is expected + // to be moved to the stateWriter interface when the latter is ready. + UpdateContractCode(address common.Address, codeHash common.Hash, code []byte) error // Hash returns the root hash of the trie. It does not write to the database and // can be used even if the trie doesn't have one. @@ -170,6 +178,9 @@ type cachingDB struct { // OpenTrie opens the main account trie at a specific root hash. func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) { + if db.triedb.IsVerkle() { + return trie.NewVerkleTrie(root, db.triedb, utils.NewPointCache(commitmentCacheItems)) + } tr, err := trie.NewStateTrie(trie.StateTrieID(root), db.triedb) if err != nil { return nil, err @@ -178,7 +189,13 @@ func (db *cachingDB) OpenTrie(root common.Hash) (Trie, error) { } // OpenStorageTrie opens the storage trie of an account. -func (db *cachingDB) OpenStorageTrie(stateRoot common.Hash, address common.Address, root common.Hash) (Trie, error) { +func (db *cachingDB) OpenStorageTrie(stateRoot common.Hash, address common.Address, root common.Hash, self Trie) (Trie, error) { + // In the verkle case, there is only one tree. But the two-tree structure + // is hardcoded in the codebase. So we need to return the same trie in this + // case. + if db.triedb.IsVerkle() { + return self, nil + } tr, err := trie.NewStateTrie(trie.StorageTrieID(stateRoot, crypto.Keccak256Hash(address.Bytes()), root), db.triedb) if err != nil { return nil, err diff --git a/core/state/iterator.go b/core/state/iterator.go index 683efd73d..dc84ce689 100644 --- a/core/state/iterator.go +++ b/core/state/iterator.go @@ -123,7 +123,7 @@ func (it *nodeIterator) step() error { address := common.BytesToAddress(preimage) // Traverse the storage slots belong to the account - dataTrie, err := it.state.db.OpenStorageTrie(it.state.originalRoot, address, account.Root) + dataTrie, err := it.state.db.OpenStorageTrie(it.state.originalRoot, address, account.Root, it.state.trie) if err != nil { return err } diff --git a/core/state/state_object.go b/core/state/state_object.go index d42d2c34d..fc66b4811 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -145,7 +145,7 @@ func (s *stateObject) getTrie() (Trie, error) { s.trie = s.db.prefetcher.trie(s.addrHash, s.data.Root) } if s.trie == nil { - tr, err := s.db.db.OpenStorageTrie(s.db.originalRoot, s.address, s.data.Root) + tr, err := s.db.db.OpenStorageTrie(s.db.originalRoot, s.address, s.data.Root, s.db.trie) if err != nil { return nil, err } diff --git a/core/state/statedb.go b/core/state/statedb.go index 195e463c2..674227857 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -998,7 +998,7 @@ func (s *StateDB) fastDeleteStorage(addrHash common.Hash, root common.Hash) (boo // employed when the associated state snapshot is not available. It iterates the // storage slots along with all internal trie nodes via trie directly. func (s *StateDB) slowDeleteStorage(addr common.Address, addrHash common.Hash, root common.Hash) (bool, common.StorageSize, map[common.Hash][]byte, *trienode.NodeSet, error) { - tr, err := s.db.OpenStorageTrie(s.originalRoot, addr, root) + tr, err := s.db.OpenStorageTrie(s.originalRoot, addr, root, s.trie) if err != nil { return false, 0, nil, nil, fmt.Errorf("failed to open storage trie, err: %w", err) } diff --git a/core/state/trie_prefetcher.go b/core/state/trie_prefetcher.go index 772c698dd..c2a49417d 100644 --- a/core/state/trie_prefetcher.go +++ b/core/state/trie_prefetcher.go @@ -305,7 +305,9 @@ func (sf *subfetcher) loop() { } sf.trie = trie } else { - trie, err := sf.db.OpenStorageTrie(sf.state, sf.addr, sf.root) + // The trie argument can be nil as verkle doesn't support prefetching + // yet. TODO FIX IT(rjl493456442), otherwise code will panic here. + trie, err := sf.db.OpenStorageTrie(sf.state, sf.addr, sf.root, nil) if err != nil { log.Warn("Trie prefetcher failed opening trie", "root", sf.root, "err", err) return diff --git a/core/types/hashes.go b/core/types/hashes.go index 3a787aa13..43e9130fd 100644 --- a/core/types/hashes.go +++ b/core/types/hashes.go @@ -23,7 +23,7 @@ import ( ) var ( - // EmptyRootHash is the known root hash of an empty trie. + // EmptyRootHash is the known root hash of an empty merkle trie. EmptyRootHash = common.HexToHash("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421") // EmptyUncleHash is the known hash of the empty uncle set. @@ -40,6 +40,9 @@ var ( // EmptyWithdrawalsHash is the known hash of the empty withdrawal set. EmptyWithdrawalsHash = common.HexToHash("56e81f171bcc55a6ff8345e692c0f86e5b48e01b996cadc001622fb5e363b421") + + // EmptyVerkleHash is the known hash of an empty verkle trie. + EmptyVerkleHash = common.Hash{} ) // TrieRootHash returns the hash itself if it's non-empty or the predefined diff --git a/go.mod b/go.mod index 385d5afdc..4d7ddcfc7 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/cockroachdb/errors v1.8.1 github.com/cockroachdb/pebble v0.0.0-20230928194634-aa077af62593 github.com/consensys/gnark-crypto v0.12.1 + github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233 github.com/crate-crypto/go-kzg-4844 v0.7.0 github.com/davecgh/go-spew v1.1.1 github.com/deckarep/golang-set/v2 v2.1.0 @@ -26,7 +27,7 @@ require ( github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 github.com/fsnotify/fsnotify v1.6.0 github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff - github.com/gballet/go-verkle v0.0.0-20230607174250-df487255f46b + github.com/gballet/go-verkle v0.1.1-0.20231031103413-a67434b50f46 github.com/go-stack/stack v1.8.1 github.com/gofrs/flock v0.8.1 github.com/golang-jwt/jwt/v4 v4.5.0 @@ -65,7 +66,7 @@ require ( go.uber.org/automaxprocs v1.5.2 golang.org/x/crypto v0.14.0 golang.org/x/exp v0.0.0-20230905200255-921286631fa9 - golang.org/x/sync v0.3.0 + golang.org/x/sync v0.4.0 golang.org/x/sys v0.13.0 golang.org/x/text v0.13.0 golang.org/x/time v0.3.0 @@ -89,7 +90,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/sts v1.23.2 // indirect github.com/aws/smithy-go v1.15.0 // indirect github.com/beorn7/perks v1.0.1 // indirect - github.com/bits-and-blooms/bitset v1.7.0 // indirect + github.com/bits-and-blooms/bitset v1.10.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f // indirect github.com/cockroachdb/redact v1.0.8 // indirect @@ -97,7 +98,6 @@ require ( github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect github.com/consensys/bavard v0.1.13 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect - github.com/crate-crypto/go-ipa v0.0.0-20230601170251-1830d0757c80 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 // indirect github.com/deepmap/oapi-codegen v1.6.0 // indirect github.com/dlclark/regexp2 v1.7.0 // indirect diff --git a/go.sum b/go.sum index cc38e7975..765a9da86 100644 --- a/go.sum +++ b/go.sum @@ -99,6 +99,8 @@ github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bits-and-blooms/bitset v1.7.0 h1:YjAGVd3XmtK9ktAbX8Zg2g2PwLIMjGREZJHlV4j7NEo= github.com/bits-and-blooms/bitset v1.7.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= +github.com/bits-and-blooms/bitset v1.10.0 h1:ePXTeiPEazB5+opbv5fr8umg2R/1NlzgDsyepwsSr88= +github.com/bits-and-blooms/bitset v1.10.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8= github.com/btcsuite/btcd/btcec/v2 v2.2.0 h1:fzn1qaOt32TuLjFlkzYSsBC35Q3KUjT1SwPxiMSCF5k= github.com/btcsuite/btcd/btcec/v2 v2.2.0/go.mod h1:U7MHm051Al6XmscBQ0BoNydpOTsFAn707034b5nY8zU= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U= @@ -145,8 +147,10 @@ github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= -github.com/crate-crypto/go-ipa v0.0.0-20230601170251-1830d0757c80 h1:DuBDHVjgGMPki7bAyh91+3cF1Vh34sAEdH8JQgbc2R0= -github.com/crate-crypto/go-ipa v0.0.0-20230601170251-1830d0757c80/go.mod h1:gzbVz57IDJgQ9rLQwfSk696JGWof8ftznEL9GoAv3NI= +github.com/crate-crypto/go-ipa v0.0.0-20230914135612-d1b03fcb8e58 h1:PwUlswsGOrLB677lW4XrlWLeszY3BaDGbvZ6dYk28tQ= +github.com/crate-crypto/go-ipa v0.0.0-20230914135612-d1b03fcb8e58/go.mod h1:J+gsi6D4peY0kyhaklyXFRVHOQWI2I5uU0c2+/90HYc= +github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233 h1:d28BXYi+wUpz1KBmiF9bWrjEMacUEREV6MBi2ODnrfQ= +github.com/crate-crypto/go-ipa v0.0.0-20231025140028-3c0104f4b233/go.mod h1:geZJZH3SzKCqnz5VT0q/DyIG/tvu/dZk+VIfXicupJs= github.com/crate-crypto/go-kzg-4844 v0.7.0 h1:C0vgZRk4q4EZ/JgPfzuSoxdCq3C3mOZMBShovmncxvA= github.com/crate-crypto/go-kzg-4844 v0.7.0/go.mod h1:1kMhvPgI0Ky3yIa+9lFySEBUBXkYxeOi8ZF1sYioxhc= github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= @@ -201,8 +205,10 @@ github.com/garslo/gogen v0.0.0-20170306192744-1d203ffc1f61/go.mod h1:Q0X6pkwTILD github.com/gavv/httpexpect v2.0.0+incompatible/go.mod h1:x+9tiU1YnrOvnB725RkpoLv1M62hOWzwo5OXotisrKc= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqGNY4FhTFhk+o9oFHGINQ/+vhlm8HFzi6znCI= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= -github.com/gballet/go-verkle v0.0.0-20230607174250-df487255f46b h1:vMT47RYsrftsHSTQhqXwC3BYflo38OLC3Y4LtXtLyU0= -github.com/gballet/go-verkle v0.0.0-20230607174250-df487255f46b/go.mod h1:CDncRYVRSDqwakm282WEkjfaAj1hxU/v5RXxk5nXOiI= +github.com/gballet/go-verkle v0.1.1-0.20231004173727-0a4e93ed640b h1:LHeiiSTL2FEGCP1ov6FqkikiViqygeVo1ZwJ1x3nYSE= +github.com/gballet/go-verkle v0.1.1-0.20231004173727-0a4e93ed640b/go.mod h1:7JamHhSTnnHDhcI3G8r4sWaD9XlleriqVlC3FeAQJKM= +github.com/gballet/go-verkle v0.1.1-0.20231031103413-a67434b50f46 h1:BAIP2GihuqhwdILrV+7GJel5lyPV3u1+PgzrWLc0TkE= +github.com/gballet/go-verkle v0.1.1-0.20231031103413-a67434b50f46/go.mod h1:QNpY22eby74jVhqH4WhDLDwxc/vqsern6pW+u2kbkpc= github.com/getkin/kin-openapi v0.53.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= @@ -418,7 +424,6 @@ github.com/labstack/echo/v4 v4.1.11/go.mod h1:i541M3Fj6f76NZtHSj7TXnyM8n2gaodfvf github.com/labstack/echo/v4 v4.2.1/go.mod h1:AA49e0DZ8kk5jTOOCKNuPR6oTnBS0dYiM4FW1e6jwpg= github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= -github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -713,9 +718,8 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= +golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -769,7 +773,6 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211020174200-9d6173849985/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/les/server_requests.go b/les/server_requests.go index 9a249f04c..cc5b60171 100644 --- a/les/server_requests.go +++ b/les/server_requests.go @@ -430,7 +430,7 @@ func handleGetProofs(msg Decoder) (serveRequestFn, uint64, uint64, error) { p.bumpInvalid() continue } - trie, err = statedb.OpenStorageTrie(root, address, account.Root) + trie, err = statedb.OpenStorageTrie(root, address, account.Root, nil) if trie == nil || err != nil { p.Log().Warn("Failed to open storage trie for proof", "block", header.Number, "hash", header.Hash(), "account", address, "root", account.Root, "err", err) continue diff --git a/light/odr_test.go b/light/odr_test.go index c415d73e7..de12f9b7e 100644 --- a/light/odr_test.go +++ b/light/odr_test.go @@ -89,7 +89,7 @@ func (odr *testOdr) Retrieve(ctx context.Context, req OdrRequest) error { t state.Trie ) if len(req.Id.AccountAddress) > 0 { - t, err = odr.serverState.OpenStorageTrie(req.Id.StateRoot, common.BytesToAddress(req.Id.AccountAddress), req.Id.Root) + t, err = odr.serverState.OpenStorageTrie(req.Id.StateRoot, common.BytesToAddress(req.Id.AccountAddress), req.Id.Root, nil) } else { t, err = odr.serverState.OpenTrie(req.Id.Root) } diff --git a/light/trie.go b/light/trie.go index 1847f1e71..1d93bdf41 100644 --- a/light/trie.go +++ b/light/trie.go @@ -55,7 +55,7 @@ func (db *odrDatabase) OpenTrie(root common.Hash) (state.Trie, error) { return &odrTrie{db: db, id: db.id}, nil } -func (db *odrDatabase) OpenStorageTrie(stateRoot common.Hash, address common.Address, root common.Hash) (state.Trie, error) { +func (db *odrDatabase) OpenStorageTrie(stateRoot common.Hash, address common.Address, root common.Hash, _ state.Trie) (state.Trie, error) { return &odrTrie{db: db, id: StorageTrieID(db.id, address, root)}, nil } diff --git a/trie/database.go b/trie/database.go index 1e59f0908..321b4f895 100644 --- a/trie/database.go +++ b/trie/database.go @@ -31,6 +31,7 @@ import ( // Config defines all necessary options for database. type Config struct { Preimages bool // Flag whether the preimage of node key is recorded + IsVerkle bool // Flag whether the db is holding a verkle tree HashDB *hashdb.Config // Configs for hash-based scheme PathDB *pathdb.Config // Configs for experimental path-based scheme } @@ -318,3 +319,8 @@ func (db *Database) SetBufferSize(size int) error { } return pdb.SetBufferSize(size) } + +// IsVerkle returns the indicator if the database is holding a verkle tree. +func (db *Database) IsVerkle() bool { + return db.config.IsVerkle +} diff --git a/trie/trienode/node.go b/trie/trienode/node.go index 98d5588b6..95315c2e9 100644 --- a/trie/trienode/node.go +++ b/trie/trienode/node.go @@ -39,7 +39,7 @@ func (n *Node) Size() int { // IsDeleted returns the indicator if the node is marked as deleted. func (n *Node) IsDeleted() bool { - return n.Hash == (common.Hash{}) + return len(n.Blob) == 0 } // New constructs a node with provided node information. diff --git a/trie/utils/verkle.go b/trie/utils/verkle.go new file mode 100644 index 000000000..ce059edc6 --- /dev/null +++ b/trie/utils/verkle.go @@ -0,0 +1,342 @@ +// Copyright 2023 go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package utils + +import ( + "encoding/binary" + "sync" + + "github.com/crate-crypto/go-ipa/bandersnatch/fr" + "github.com/ethereum/go-ethereum/common/lru" + "github.com/ethereum/go-ethereum/metrics" + "github.com/gballet/go-verkle" + "github.com/holiman/uint256" +) + +const ( + // The spec of verkle key encoding can be found here. + // https://notes.ethereum.org/@vbuterin/verkle_tree_eip#Tree-embedding + VersionLeafKey = 0 + BalanceLeafKey = 1 + NonceLeafKey = 2 + CodeKeccakLeafKey = 3 + CodeSizeLeafKey = 4 +) + +var ( + zero = uint256.NewInt(0) + verkleNodeWidthLog2 = 8 + headerStorageOffset = uint256.NewInt(64) + mainStorageOffsetLshVerkleNodeWidth = new(uint256.Int).Lsh(uint256.NewInt(256), 31-uint(verkleNodeWidthLog2)) + codeOffset = uint256.NewInt(128) + verkleNodeWidth = uint256.NewInt(256) + codeStorageDelta = uint256.NewInt(0).Sub(codeOffset, headerStorageOffset) + + index0Point *verkle.Point // pre-computed commitment of polynomial [2+256*64] + + // cacheHitGauge is the metric to track how many cache hit occurred. + cacheHitGauge = metrics.NewRegisteredGauge("trie/verkle/cache/hit", nil) + + // cacheMissGauge is the metric to track how many cache miss occurred. + cacheMissGauge = metrics.NewRegisteredGauge("trie/verkle/cache/miss", nil) +) + +func init() { + // The byte array is the Marshalled output of the point computed as such: + // + // var ( + // config = verkle.GetConfig() + // fr verkle.Fr + // ) + // verkle.FromLEBytes(&fr, []byte{2, 64}) + // point := config.CommitToPoly([]verkle.Fr{fr}, 1) + index0Point = new(verkle.Point) + err := index0Point.SetBytes([]byte{34, 25, 109, 242, 193, 5, 144, 224, 76, 52, 189, 92, 197, 126, 9, 145, 27, 152, 199, 130, 165, 3, 210, 27, 193, 131, 142, 28, 110, 26, 16, 191}) + if err != nil { + panic(err) + } +} + +// PointCache is the LRU cache for storing evaluated address commitment. +type PointCache struct { + lru lru.BasicLRU[string, *verkle.Point] + lock sync.RWMutex +} + +// NewPointCache returns the cache with specified size. +func NewPointCache(maxItems int) *PointCache { + return &PointCache{ + lru: lru.NewBasicLRU[string, *verkle.Point](maxItems), + } +} + +// Get returns the cached commitment for the specified address, or computing +// it on the flight. +func (c *PointCache) Get(addr []byte) *verkle.Point { + c.lock.Lock() + defer c.lock.Unlock() + + p, ok := c.lru.Get(string(addr)) + if ok { + cacheHitGauge.Inc(1) + return p + } + cacheMissGauge.Inc(1) + p = evaluateAddressPoint(addr) + c.lru.Add(string(addr), p) + return p +} + +// GetStem returns the first 31 bytes of the tree key as the tree stem. It only +// works for the account metadata whose treeIndex is 0. +func (c *PointCache) GetStem(addr []byte) []byte { + p := c.Get(addr) + return pointToHash(p, 0)[:31] +} + +// GetTreeKey performs both the work of the spec's get_tree_key function, and that +// of pedersen_hash: it builds the polynomial in pedersen_hash without having to +// create a mostly zero-filled buffer and "type cast" it to a 128-long 16-byte +// array. Since at most the first 5 coefficients of the polynomial will be non-zero, +// these 5 coefficients are created directly. +func GetTreeKey(address []byte, treeIndex *uint256.Int, subIndex byte) []byte { + if len(address) < 32 { + var aligned [32]byte + address = append(aligned[:32-len(address)], address...) + } + // poly = [2+256*64, address_le_low, address_le_high, tree_index_le_low, tree_index_le_high] + var poly [5]fr.Element + + // 32-byte address, interpreted as two little endian + // 16-byte numbers. + verkle.FromLEBytes(&poly[1], address[:16]) + verkle.FromLEBytes(&poly[2], address[16:]) + + // treeIndex must be interpreted as a 32-byte aligned little-endian integer. + // e.g: if treeIndex is 0xAABBCC, we need the byte representation to be 0xCCBBAA00...00. + // poly[3] = LE({CC,BB,AA,00...0}) (16 bytes), poly[4]=LE({00,00,...}) (16 bytes). + // + // To avoid unnecessary endianness conversions for go-ipa, we do some trick: + // - poly[3]'s byte representation is the same as the *top* 16 bytes (trieIndexBytes[16:]) of + // 32-byte aligned big-endian representation (BE({00,...,AA,BB,CC})). + // - poly[4]'s byte representation is the same as the *low* 16 bytes (trieIndexBytes[:16]) of + // the 32-byte aligned big-endian representation (BE({00,00,...}). + trieIndexBytes := treeIndex.Bytes32() + verkle.FromBytes(&poly[3], trieIndexBytes[16:]) + verkle.FromBytes(&poly[4], trieIndexBytes[:16]) + + cfg := verkle.GetConfig() + ret := cfg.CommitToPoly(poly[:], 0) + + // add a constant point corresponding to poly[0]=[2+256*64]. + ret.Add(ret, index0Point) + + return pointToHash(ret, subIndex) +} + +// GetTreeKeyWithEvaluatedAddress is basically identical to GetTreeKey, the only +// difference is a part of polynomial is already evaluated. +// +// Specifically, poly = [2+256*64, address_le_low, address_le_high] is already +// evaluated. +func GetTreeKeyWithEvaluatedAddress(evaluated *verkle.Point, treeIndex *uint256.Int, subIndex byte) []byte { + var poly [5]fr.Element + + poly[0].SetZero() + poly[1].SetZero() + poly[2].SetZero() + + // little-endian, 32-byte aligned treeIndex + var index [32]byte + for i := 0; i < len(treeIndex); i++ { + binary.LittleEndian.PutUint64(index[i*8:(i+1)*8], treeIndex[i]) + } + verkle.FromLEBytes(&poly[3], index[:16]) + verkle.FromLEBytes(&poly[4], index[16:]) + + cfg := verkle.GetConfig() + ret := cfg.CommitToPoly(poly[:], 0) + + // add the pre-evaluated address + ret.Add(ret, evaluated) + + return pointToHash(ret, subIndex) +} + +// VersionKey returns the verkle tree key of the version field for the specified account. +func VersionKey(address []byte) []byte { + return GetTreeKey(address, zero, VersionLeafKey) +} + +// BalanceKey returns the verkle tree key of the balance field for the specified account. +func BalanceKey(address []byte) []byte { + return GetTreeKey(address, zero, BalanceLeafKey) +} + +// NonceKey returns the verkle tree key of the nonce field for the specified account. +func NonceKey(address []byte) []byte { + return GetTreeKey(address, zero, NonceLeafKey) +} + +// CodeKeccakKey returns the verkle tree key of the code keccak field for +// the specified account. +func CodeKeccakKey(address []byte) []byte { + return GetTreeKey(address, zero, CodeKeccakLeafKey) +} + +// CodeSizeKey returns the verkle tree key of the code size field for the +// specified account. +func CodeSizeKey(address []byte) []byte { + return GetTreeKey(address, zero, CodeSizeLeafKey) +} + +func codeChunkIndex(chunk *uint256.Int) (*uint256.Int, byte) { + var ( + chunkOffset = new(uint256.Int).Add(codeOffset, chunk) + treeIndex = new(uint256.Int).Div(chunkOffset, verkleNodeWidth) + subIndexMod = new(uint256.Int).Mod(chunkOffset, verkleNodeWidth) + ) + var subIndex byte + if len(subIndexMod) != 0 { + subIndex = byte(subIndexMod[0]) + } + return treeIndex, subIndex +} + +// CodeChunkKey returns the verkle tree key of the code chunk for the +// specified account. +func CodeChunkKey(address []byte, chunk *uint256.Int) []byte { + treeIndex, subIndex := codeChunkIndex(chunk) + return GetTreeKey(address, treeIndex, subIndex) +} + +func storageIndex(bytes []byte) (*uint256.Int, byte) { + // If the storage slot is in the header, we need to add the header offset. + var key uint256.Int + key.SetBytes(bytes) + if key.Cmp(codeStorageDelta) < 0 { + // This addition is always safe; it can't ever overflow since pos + +package utils + +import ( + "bytes" + "testing" + + "github.com/gballet/go-verkle" + "github.com/holiman/uint256" +) + +func TestTreeKey(t *testing.T) { + var ( + address = []byte{0x01} + addressEval = evaluateAddressPoint(address) + smallIndex = uint256.NewInt(1) + largeIndex = uint256.NewInt(10000) + smallStorage = []byte{0x1} + largeStorage = bytes.Repeat([]byte{0xff}, 16) + ) + if !bytes.Equal(VersionKey(address), VersionKeyWithEvaluatedAddress(addressEval)) { + t.Fatal("Unmatched version key") + } + if !bytes.Equal(BalanceKey(address), BalanceKeyWithEvaluatedAddress(addressEval)) { + t.Fatal("Unmatched balance key") + } + if !bytes.Equal(NonceKey(address), NonceKeyWithEvaluatedAddress(addressEval)) { + t.Fatal("Unmatched nonce key") + } + if !bytes.Equal(CodeKeccakKey(address), CodeKeccakKeyWithEvaluatedAddress(addressEval)) { + t.Fatal("Unmatched code keccak key") + } + if !bytes.Equal(CodeSizeKey(address), CodeSizeKeyWithEvaluatedAddress(addressEval)) { + t.Fatal("Unmatched code size key") + } + if !bytes.Equal(CodeChunkKey(address, smallIndex), CodeChunkKeyWithEvaluatedAddress(addressEval, smallIndex)) { + t.Fatal("Unmatched code chunk key") + } + if !bytes.Equal(CodeChunkKey(address, largeIndex), CodeChunkKeyWithEvaluatedAddress(addressEval, largeIndex)) { + t.Fatal("Unmatched code chunk key") + } + if !bytes.Equal(StorageSlotKey(address, smallStorage), StorageSlotKeyWithEvaluatedAddress(addressEval, smallStorage)) { + t.Fatal("Unmatched storage slot key") + } + if !bytes.Equal(StorageSlotKey(address, largeStorage), StorageSlotKeyWithEvaluatedAddress(addressEval, largeStorage)) { + t.Fatal("Unmatched storage slot key") + } +} + +// goos: darwin +// goarch: amd64 +// pkg: github.com/ethereum/go-ethereum/trie/utils +// cpu: VirtualApple @ 2.50GHz +// BenchmarkTreeKey +// BenchmarkTreeKey-8 398731 2961 ns/op 32 B/op 1 allocs/op +func BenchmarkTreeKey(b *testing.B) { + // Initialize the IPA settings which can be pretty expensive. + verkle.GetConfig() + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + BalanceKey([]byte{0x01}) + } +} + +// goos: darwin +// goarch: amd64 +// pkg: github.com/ethereum/go-ethereum/trie/utils +// cpu: VirtualApple @ 2.50GHz +// BenchmarkTreeKeyWithEvaluation +// BenchmarkTreeKeyWithEvaluation-8 513855 2324 ns/op 32 B/op 1 allocs/op +func BenchmarkTreeKeyWithEvaluation(b *testing.B) { + // Initialize the IPA settings which can be pretty expensive. + verkle.GetConfig() + + addr := []byte{0x01} + eval := evaluateAddressPoint(addr) + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + BalanceKeyWithEvaluatedAddress(eval) + } +} + +// goos: darwin +// goarch: amd64 +// pkg: github.com/ethereum/go-ethereum/trie/utils +// cpu: VirtualApple @ 2.50GHz +// BenchmarkStorageKey +// BenchmarkStorageKey-8 230516 4584 ns/op 96 B/op 3 allocs/op +func BenchmarkStorageKey(b *testing.B) { + // Initialize the IPA settings which can be pretty expensive. + verkle.GetConfig() + + b.ReportAllocs() + b.ResetTimer() + + for i := 0; i < b.N; i++ { + StorageSlotKey([]byte{0x01}, bytes.Repeat([]byte{0xff}, 32)) + } +} + +// goos: darwin +// goarch: amd64 +// pkg: github.com/ethereum/go-ethereum/trie/utils +// cpu: VirtualApple @ 2.50GHz +// BenchmarkStorageKeyWithEvaluation +// BenchmarkStorageKeyWithEvaluation-8 320125 3753 ns/op 96 B/op 3 allocs/op +func BenchmarkStorageKeyWithEvaluation(b *testing.B) { + // Initialize the IPA settings which can be pretty expensive. + verkle.GetConfig() + + addr := []byte{0x01} + eval := evaluateAddressPoint(addr) + + b.ReportAllocs() + b.ResetTimer() + for i := 0; i < b.N; i++ { + StorageSlotKeyWithEvaluatedAddress(eval, bytes.Repeat([]byte{0xff}, 32)) + } +} diff --git a/trie/verkle.go b/trie/verkle.go new file mode 100644 index 000000000..89e2e5340 --- /dev/null +++ b/trie/verkle.go @@ -0,0 +1,375 @@ +// Copyright 2023 go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package trie + +import ( + "encoding/binary" + "errors" + "fmt" + "math/big" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/trie/trienode" + "github.com/ethereum/go-ethereum/trie/utils" + "github.com/gballet/go-verkle" + "github.com/holiman/uint256" +) + +var ( + zero [32]byte + errInvalidRootType = errors.New("invalid node type for root") +) + +// VerkleTrie is a wrapper around VerkleNode that implements the trie.Trie +// interface so that Verkle trees can be reused verbatim. +type VerkleTrie struct { + root verkle.VerkleNode + db *Database + cache *utils.PointCache + reader *trieReader +} + +// NewVerkleTrie constructs a verkle tree based on the specified root hash. +func NewVerkleTrie(root common.Hash, db *Database, cache *utils.PointCache) (*VerkleTrie, error) { + reader, err := newTrieReader(root, common.Hash{}, db) + if err != nil { + return nil, err + } + // Parse the root verkle node if it's not empty. + node := verkle.New() + if root != types.EmptyVerkleHash && root != types.EmptyRootHash { + blob, err := reader.node(nil, common.Hash{}) + if err != nil { + return nil, err + } + node, err = verkle.ParseNode(blob, 0) + if err != nil { + return nil, err + } + } + return &VerkleTrie{ + root: node, + db: db, + cache: cache, + reader: reader, + }, nil +} + +// GetKey returns the sha3 preimage of a hashed key that was previously used +// to store a value. +func (t *VerkleTrie) GetKey(key []byte) []byte { + return key +} + +// GetAccount implements state.Trie, retrieving the account with the specified +// account address. If the specified account is not in the verkle tree, nil will +// be returned. If the tree is corrupted, an error will be returned. +func (t *VerkleTrie) GetAccount(addr common.Address) (*types.StateAccount, error) { + var ( + acc = &types.StateAccount{} + values [][]byte + err error + ) + switch n := t.root.(type) { + case *verkle.InternalNode: + values, err = n.GetValuesAtStem(t.cache.GetStem(addr[:]), t.nodeResolver) + if err != nil { + return nil, fmt.Errorf("GetAccount (%x) error: %v", addr, err) + } + default: + return nil, errInvalidRootType + } + if values == nil { + return nil, nil + } + // Decode nonce in little-endian + if len(values[utils.NonceLeafKey]) > 0 { + acc.Nonce = binary.LittleEndian.Uint64(values[utils.NonceLeafKey]) + } + // Decode balance in little-endian + var balance [32]byte + copy(balance[:], values[utils.BalanceLeafKey]) + for i := 0; i < len(balance)/2; i++ { + balance[len(balance)-i-1], balance[i] = balance[i], balance[len(balance)-i-1] + } + acc.Balance = new(big.Int).SetBytes(balance[:]) + + // Decode codehash + acc.CodeHash = values[utils.CodeKeccakLeafKey] + + // TODO account.Root is leave as empty. How should we handle the legacy account? + return acc, nil +} + +// GetStorage implements state.Trie, retrieving the storage slot with the specified +// account address and storage key. If the specified slot is not in the verkle tree, +// nil will be returned. If the tree is corrupted, an error will be returned. +func (t *VerkleTrie) GetStorage(addr common.Address, key []byte) ([]byte, error) { + k := utils.StorageSlotKeyWithEvaluatedAddress(t.cache.Get(addr.Bytes()), key) + val, err := t.root.Get(k, t.nodeResolver) + if err != nil { + return nil, err + } + return common.TrimLeftZeroes(val), nil +} + +// UpdateAccount implements state.Trie, writing the provided account into the tree. +// If the tree is corrupted, an error will be returned. +func (t *VerkleTrie) UpdateAccount(addr common.Address, acc *types.StateAccount) error { + var ( + err error + nonce, balance [32]byte + values = make([][]byte, verkle.NodeWidth) + ) + values[utils.VersionLeafKey] = zero[:] + values[utils.CodeKeccakLeafKey] = acc.CodeHash[:] + + // Encode nonce in little-endian + binary.LittleEndian.PutUint64(nonce[:], acc.Nonce) + values[utils.NonceLeafKey] = nonce[:] + + // Encode balance in little-endian + bytes := acc.Balance.Bytes() + if len(bytes) > 0 { + for i, b := range bytes { + balance[len(bytes)-i-1] = b + } + } + values[utils.BalanceLeafKey] = balance[:] + + switch n := t.root.(type) { + case *verkle.InternalNode: + err = n.InsertValuesAtStem(t.cache.GetStem(addr[:]), values, t.nodeResolver) + if err != nil { + return fmt.Errorf("UpdateAccount (%x) error: %v", addr, err) + } + default: + return errInvalidRootType + } + // TODO figure out if the code size needs to be updated, too + return nil +} + +// UpdateStorage implements state.Trie, writing the provided storage slot into +// the tree. If the tree is corrupted, an error will be returned. +func (t *VerkleTrie) UpdateStorage(address common.Address, key, value []byte) error { + // Left padding the slot value to 32 bytes. + var v [32]byte + if len(value) >= 32 { + copy(v[:], value[:32]) + } else { + copy(v[32-len(value):], value[:]) + } + k := utils.StorageSlotKeyWithEvaluatedAddress(t.cache.Get(address.Bytes()), key) + return t.root.Insert(k, v[:], t.nodeResolver) +} + +// DeleteAccount implements state.Trie, deleting the specified account from the +// trie. If the account was not existent in the trie, no error will be returned. +// If the trie is corrupted, an error will be returned. +func (t *VerkleTrie) DeleteAccount(addr common.Address) error { + var ( + err error + values = make([][]byte, verkle.NodeWidth) + ) + for i := 0; i < verkle.NodeWidth; i++ { + values[i] = zero[:] + } + switch n := t.root.(type) { + case *verkle.InternalNode: + err = n.InsertValuesAtStem(t.cache.GetStem(addr.Bytes()), values, t.nodeResolver) + if err != nil { + return fmt.Errorf("DeleteAccount (%x) error: %v", addr, err) + } + default: + return errInvalidRootType + } + return nil +} + +// DeleteStorage implements state.Trie, deleting the specified storage slot from +// the trie. If the storage slot was not existent in the trie, no error will be +// returned. If the trie is corrupted, an error will be returned. +func (t *VerkleTrie) DeleteStorage(addr common.Address, key []byte) error { + var zero [32]byte + k := utils.StorageSlotKeyWithEvaluatedAddress(t.cache.Get(addr.Bytes()), key) + return t.root.Insert(k, zero[:], t.nodeResolver) +} + +// Hash returns the root hash of the tree. It does not write to the database and +// can be used even if the tree doesn't have one. +func (t *VerkleTrie) Hash() common.Hash { + return t.root.Commit().Bytes() +} + +// Commit writes all nodes to the tree's memory database. +func (t *VerkleTrie) Commit(_ bool) (common.Hash, *trienode.NodeSet, error) { + root, ok := t.root.(*verkle.InternalNode) + if !ok { + return common.Hash{}, nil, errors.New("unexpected root node type") + } + nodes, err := root.BatchSerialize() + if err != nil { + return common.Hash{}, nil, fmt.Errorf("serializing tree nodes: %s", err) + } + nodeset := trienode.NewNodeSet(common.Hash{}) + for _, node := range nodes { + // hash parameter is not used in pathdb + nodeset.AddNode(node.Path, trienode.New(common.Hash{}, node.SerializedBytes)) + } + // Serialize root commitment form + return t.Hash(), nodeset, nil +} + +// NodeIterator implements state.Trie, returning an iterator that returns +// nodes of the trie. Iteration starts at the key after the given start key. +// +// TODO(gballet, rjl493456442) implement it. +func (t *VerkleTrie) NodeIterator(startKey []byte) (NodeIterator, error) { + panic("not implemented") +} + +// Prove implements state.Trie, constructing a Merkle proof for key. The result +// contains all encoded nodes on the path to the value at key. The value itself +// is also included in the last node and can be retrieved by verifying the proof. +// +// If the trie does not contain a value for key, the returned proof contains all +// nodes of the longest existing prefix of the key (at least the root), ending +// with the node that proves the absence of the key. +// +// TODO(gballet, rjl493456442) implement it. +func (t *VerkleTrie) Prove(key []byte, proofDb ethdb.KeyValueWriter) error { + panic("not implemented") +} + +// Copy returns a deep-copied verkle tree. +func (t *VerkleTrie) Copy() *VerkleTrie { + return &VerkleTrie{ + root: t.root.Copy(), + db: t.db, + cache: t.cache, + reader: t.reader, + } +} + +// IsVerkle indicates if the trie is a Verkle trie. +func (t *VerkleTrie) IsVerkle() bool { + return true +} + +// ChunkedCode represents a sequence of 32-bytes chunks of code (31 bytes of which +// are actual code, and 1 byte is the pushdata offset). +type ChunkedCode []byte + +// Copy the values here so as to avoid an import cycle +const ( + PUSH1 = byte(0x60) + PUSH32 = byte(0x7f) +) + +// ChunkifyCode generates the chunked version of an array representing EVM bytecode +func ChunkifyCode(code []byte) ChunkedCode { + var ( + chunkOffset = 0 // offset in the chunk + chunkCount = len(code) / 31 + codeOffset = 0 // offset in the code + ) + if len(code)%31 != 0 { + chunkCount++ + } + chunks := make([]byte, chunkCount*32) + for i := 0; i < chunkCount; i++ { + // number of bytes to copy, 31 unless the end of the code has been reached. + end := 31 * (i + 1) + if len(code) < end { + end = len(code) + } + copy(chunks[i*32+1:], code[31*i:end]) // copy the code itself + + // chunk offset = taken from the last chunk. + if chunkOffset > 31 { + // skip offset calculation if push data covers the whole chunk + chunks[i*32] = 31 + chunkOffset = 1 + continue + } + chunks[32*i] = byte(chunkOffset) + chunkOffset = 0 + + // Check each instruction and update the offset it should be 0 unless + // a PUSH-N overflows. + for ; codeOffset < end; codeOffset++ { + if code[codeOffset] >= PUSH1 && code[codeOffset] <= PUSH32 { + codeOffset += int(code[codeOffset] - PUSH1 + 1) + if codeOffset+1 >= 31*(i+1) { + codeOffset++ + chunkOffset = codeOffset - 31*(i+1) + break + } + } + } + } + return chunks +} + +// UpdateContractCode implements state.Trie, writing the provided contract code +// into the trie. +func (t *VerkleTrie) UpdateContractCode(addr common.Address, codeHash common.Hash, code []byte) error { + var ( + chunks = ChunkifyCode(code) + values [][]byte + key []byte + err error + ) + for i, chunknr := 0, uint64(0); i < len(chunks); i, chunknr = i+32, chunknr+1 { + groupOffset := (chunknr + 128) % 256 + if groupOffset == 0 /* start of new group */ || chunknr == 0 /* first chunk in header group */ { + values = make([][]byte, verkle.NodeWidth) + key = utils.CodeChunkKeyWithEvaluatedAddress(t.cache.Get(addr.Bytes()), uint256.NewInt(chunknr)) + } + values[groupOffset] = chunks[i : i+32] + + // Reuse the calculated key to also update the code size. + if i == 0 { + cs := make([]byte, 32) + binary.LittleEndian.PutUint64(cs, uint64(len(code))) + values[utils.CodeSizeLeafKey] = cs + } + if groupOffset == 255 || len(chunks)-i <= 32 { + switch root := t.root.(type) { + case *verkle.InternalNode: + err = root.InsertValuesAtStem(key[:31], values, t.nodeResolver) + if err != nil { + return fmt.Errorf("UpdateContractCode (addr=%x) error: %w", addr[:], err) + } + default: + return errInvalidRootType + } + } + } + return nil +} + +func (t *VerkleTrie) ToDot() string { + return verkle.ToDot(t.root) +} + +func (t *VerkleTrie) nodeResolver(path []byte) ([]byte, error) { + return t.reader.node(path, common.Hash{}) +} diff --git a/trie/verkle_test.go b/trie/verkle_test.go new file mode 100644 index 000000000..44fb7dc29 --- /dev/null +++ b/trie/verkle_test.go @@ -0,0 +1,97 @@ +// Copyright 2023 go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package trie + +import ( + "bytes" + "math/big" + "reflect" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/trie/triedb/pathdb" + "github.com/ethereum/go-ethereum/trie/utils" +) + +var ( + accounts = map[common.Address]*types.StateAccount{ + common.Address{1}: { + Nonce: 100, + Balance: big.NewInt(100), + CodeHash: common.Hash{0x1}.Bytes(), + }, + common.Address{2}: { + Nonce: 200, + Balance: big.NewInt(200), + CodeHash: common.Hash{0x2}.Bytes(), + }, + } + storages = map[common.Address]map[common.Hash][]byte{ + common.Address{1}: { + common.Hash{10}: []byte{10}, + common.Hash{11}: []byte{11}, + common.MaxHash: []byte{0xff}, + }, + common.Address{2}: { + common.Hash{20}: []byte{20}, + common.Hash{21}: []byte{21}, + common.MaxHash: []byte{0xff}, + }, + } +) + +func TestVerkleTreeReadWrite(t *testing.T) { + db := NewDatabase(rawdb.NewMemoryDatabase(), &Config{ + IsVerkle: true, + PathDB: pathdb.Defaults, + }) + defer db.Close() + + tr, _ := NewVerkleTrie(types.EmptyVerkleHash, db, utils.NewPointCache(100)) + + for addr, acct := range accounts { + if err := tr.UpdateAccount(addr, acct); err != nil { + t.Fatalf("Failed to update account, %v", err) + } + for key, val := range storages[addr] { + if err := tr.UpdateStorage(addr, key.Bytes(), val); err != nil { + t.Fatalf("Failed to update account, %v", err) + } + } + } + + for addr, acct := range accounts { + stored, err := tr.GetAccount(addr) + if err != nil { + t.Fatalf("Failed to get account, %v", err) + } + if !reflect.DeepEqual(stored, acct) { + t.Fatal("account is not matched") + } + for key, val := range storages[addr] { + stored, err := tr.GetStorage(addr, key.Bytes()) + if err != nil { + t.Fatalf("Failed to get storage, %v", err) + } + if !bytes.Equal(stored, val) { + t.Fatal("storage is not matched") + } + } + } +}