From ced8041f72d2baf5cc0f958bfdde4395e9ae8a83 Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Mon, 24 Apr 2023 18:32:44 +0800 Subject: [PATCH 1/8] fix package oops --- trie_by_cid/trie/database_test.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/trie_by_cid/trie/database_test.go b/trie_by_cid/trie/database_test.go index fce88db..599cc58 100644 --- a/trie_by_cid/trie/database_test.go +++ b/trie_by_cid/trie/database_test.go @@ -20,13 +20,15 @@ import ( "testing" "github.com/ethereum/go-ethereum/ethdb/memorydb" + + "github.com/cerc-io/ipld-eth-statedb/trie_by_cid/trie" ) // Tests that the trie database returns a missing trie node error if attempting // to retrieve the meta root. func TestDatabaseMetarootFetch(t *testing.T) { - db := NewDatabase(memorydb.New()) - if _, err := db.Node(CidBytes(nil)); err == nil { + db := trie.NewDatabase(memorydb.New()) + if _, err := db.Node(trie.CidBytes(nil)); err == nil { t.Fatalf("metaroot retrieval succeeded") } } From 49177ee0ab50f6207519be22d5e0bf90048cd47f Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Tue, 11 Apr 2023 17:42:11 +0800 Subject: [PATCH 2/8] Refactor redundant DB code --- config.go | 90 ----------------- pgx.go | 13 +-- sqlx.go | 20 +--- state_database.go | 20 +--- statedb_test.go | 248 +++++----------------------------------------- 5 files changed, 32 insertions(+), 359 deletions(-) delete mode 100644 config.go diff --git a/config.go b/config.go deleted file mode 100644 index aba21df..0000000 --- a/config.go +++ /dev/null @@ -1,90 +0,0 @@ -package ipld_eth_statedb - -import ( - "context" - "fmt" - "time" - - "github.com/jackc/pgx/v4/pgxpool" - "github.com/jmoiron/sqlx" - - _ "github.com/lib/pq" -) - -type Config struct { - Hostname string - Port int - DatabaseName string - Username string - Password string - - ConnTimeout time.Duration - MaxConns int - MinConns int - MaxConnLifetime time.Duration - MaxConnIdleTime time.Duration - MaxIdle int -} - -// DbConnectionString constructs and returns the connection string from the config (for sqlx driver) -func (c Config) DbConnectionString() string { - if len(c.Username) > 0 && len(c.Password) > 0 { - return fmt.Sprintf("postgresql://%s:%s@%s:%d/%s?sslmode=disable", - c.Username, c.Password, c.Hostname, c.Port, c.DatabaseName) - } - if len(c.Username) > 0 && len(c.Password) == 0 { - return fmt.Sprintf("postgresql://%s@%s:%d/%s?sslmode=disable", - c.Username, c.Hostname, c.Port, c.DatabaseName) - } - return fmt.Sprintf("postgresql://%s:%d/%s?sslmode=disable", c.Hostname, c.Port, c.DatabaseName) -} - -// NewPGXPool returns a new pgx conn pool -func NewPGXPool(ctx context.Context, config Config) (*pgxpool.Pool, error) { - pgConf, err := makePGXConfig(config) - if err != nil { - return nil, err - } - return pgxpool.ConnectConfig(ctx, pgConf) -} - -// NewSQLXPool returns a new sqlx conn pool -func NewSQLXPool(ctx context.Context, config Config) (*sqlx.DB, error) { - db, err := sqlx.ConnectContext(ctx, "postgres", config.DbConnectionString()) - if err != nil { - return nil, err - } - return db, nil -} - -// makePGXConfig creates a pgxpool.Config from the provided Config -func makePGXConfig(config Config) (*pgxpool.Config, error) { - conf, err := pgxpool.ParseConfig("") - if err != nil { - return nil, err - } - - conf.ConnConfig.Config.Host = config.Hostname - conf.ConnConfig.Config.Port = uint16(config.Port) - conf.ConnConfig.Config.Database = config.DatabaseName - conf.ConnConfig.Config.User = config.Username - conf.ConnConfig.Config.Password = config.Password - - if config.ConnTimeout != 0 { - conf.ConnConfig.Config.ConnectTimeout = config.ConnTimeout - } - if config.MaxConns != 0 { - conf.MaxConns = int32(config.MaxConns) - } - if config.MinConns != 0 { - conf.MinConns = int32(config.MinConns) - } - if config.MaxConnLifetime != 0 { - conf.MaxConnLifetime = config.MaxConnLifetime - } - if config.MaxConnIdleTime != 0 { - conf.MaxConnIdleTime = config.MaxConnIdleTime - } - - return conf, nil -} diff --git a/pgx.go b/pgx.go index c780d20..e60a6d5 100644 --- a/pgx.go +++ b/pgx.go @@ -15,18 +15,9 @@ type PGXDriver struct { db *pgxpool.Pool } -// NewPGXDriver returns a new pgx driver for Postgres -func NewPGXDriver(ctx context.Context, config Config) (*PGXDriver, error) { - db, err := NewPGXPool(ctx, config) - if err != nil { - return nil, err - } - return &PGXDriver{ctx: ctx, db: db}, nil -} - // NewPGXDriverFromPool returns a new pgx driver for Postgres -func NewPGXDriverFromPool(ctx context.Context, db *pgxpool.Pool) (*PGXDriver, error) { - return &PGXDriver{ctx: ctx, db: db}, nil +func NewPGXDriverFromPool(ctx context.Context, db *pgxpool.Pool) *PGXDriver { + return &PGXDriver{ctx: ctx, db: db} } // QueryRow satisfies sql.Database diff --git a/sqlx.go b/sqlx.go index 92f4c03..2eed275 100644 --- a/sqlx.go +++ b/sqlx.go @@ -14,25 +14,9 @@ type SQLXDriver struct { db *sqlx.DB } -// NewSQLXDriver returns a new sqlx driver for Postgres -func NewSQLXDriver(ctx context.Context, config Config) (*SQLXDriver, error) { - db, err := NewSQLXPool(ctx, config) - if err != nil { - return nil, err - } - if config.MaxConns > 0 { - db.SetMaxOpenConns(config.MaxConns) - } - if config.MaxConnLifetime > 0 { - db.SetConnMaxLifetime(config.MaxConnLifetime) - } - db.SetMaxIdleConns(config.MaxIdle) - return &SQLXDriver{ctx: ctx, db: db}, nil -} - // NewSQLXDriverFromPool returns a new sqlx driver for Postgres -func NewSQLXDriverFromPool(ctx context.Context, db *sqlx.DB) (*SQLXDriver, error) { - return &SQLXDriver{ctx: ctx, db: db}, nil +func NewSQLXDriverFromPool(ctx context.Context, db *sqlx.DB) *SQLXDriver { + return &SQLXDriver{ctx: ctx, db: db} } // QueryRow satisfies sql.Database diff --git a/state_database.go b/state_database.go index ee115e2..cb2c7b4 100644 --- a/state_database.go +++ b/state_database.go @@ -8,8 +8,6 @@ import ( "github.com/VictoriaMetrics/fastcache" lru "github.com/hashicorp/golang-lru" - "github.com/jackc/pgx/v4/pgxpool" - "github.com/jmoiron/sqlx" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" @@ -48,24 +46,14 @@ type stateDatabase struct { codeCache *fastcache.Cache } -// NewStateDatabaseWithPgxPool returns a new Database implementation using the provided postgres connection pool -func NewStateDatabaseWithPgxPool(pgDb *pgxpool.Pool) (*stateDatabase, error) { +// NewStateDatabase returns a new Database implementation using the passed parameters +func NewStateDatabase(db Database) *stateDatabase { csc, _ := lru.New(codeSizeCacheSize) return &stateDatabase{ - db: NewPostgresDB(&PGXDriver{db: pgDb}), + db: db, codeSizeCache: csc, codeCache: fastcache.New(codeCacheSize), - }, nil -} - -// NewStateDatabaseWithSqlxPool returns a new Database implementation using the passed parameters -func NewStateDatabaseWithSqlxPool(db *sqlx.DB) (*stateDatabase, error) { - csc, _ := lru.New(codeSizeCacheSize) - return &stateDatabase{ - db: NewPostgresDB(&SQLXDriver{db: db}), - codeSizeCache: csc, - codeCache: fastcache.New(codeCacheSize), - }, nil + } } // ContractCode satisfies Database, it returns the contract code for a given codehash diff --git a/statedb_test.go b/statedb_test.go index 335522e..bb097b9 100644 --- a/statedb_test.go +++ b/statedb_test.go @@ -3,8 +3,6 @@ package ipld_eth_statedb_test import ( "context" "math/big" - "os" - "strconv" "testing" "github.com/lib/pq" @@ -15,6 +13,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" "github.com/ethereum/go-ethereum/statediff/indexer/ipld" statedb "github.com/cerc-io/ipld-eth-statedb" @@ -86,17 +85,12 @@ var ( ) func TestPGXSuite(t *testing.T) { - testConfig, err := getTestConfig() + testConfig, err := postgres.DefaultConfig.WithEnv() require.NoError(t, err) - pool, err := statedb.NewPGXPool(testCtx, testConfig) + pool, err := postgres.ConnectPGX(testCtx, testConfig) if err != nil { t.Fatal(err) } - driver, err := statedb.NewPGXDriverFromPool(context.Background(), pool) - if err != nil { - t.Fatal(err) - } - database := statedb.NewPostgresDB(driver) t.Cleanup(func() { tx, err := pool.Begin(testCtx) require.NoError(t, err) @@ -112,213 +106,23 @@ func TestPGXSuite(t *testing.T) { } require.NoError(t, tx.Commit(testCtx)) }) - require.NoError(t, insertHeaderCID(database, BlockHash.String(), BlockParentHash.String(), BlockNumber.Uint64())) - require.NoError(t, insertHeaderCID(database, BlockHash2.String(), BlockHash.String(), BlockNumber2)) - require.NoError(t, insertHeaderCID(database, BlockHash3.String(), BlockHash2.String(), BlockNumber3)) - require.NoError(t, insertHeaderCID(database, BlockHash4.String(), BlockHash3.String(), BlockNumber4)) - require.NoError(t, insertHeaderCID(database, NonCanonicalHash4.String(), BlockHash3.String(), BlockNumber4)) - require.NoError(t, insertHeaderCID(database, BlockHash5.String(), BlockHash4.String(), BlockNumber5)) - require.NoError(t, insertHeaderCID(database, NonCanonicalHash5.String(), NonCanonicalHash4.String(), BlockNumber5)) - require.NoError(t, insertHeaderCID(database, BlockHash6.String(), BlockHash5.String(), BlockNumber6)) - require.NoError(t, insertStateCID(database, stateModel{ - BlockNumber: BlockNumber.Uint64(), - BlockHash: BlockHash.String(), - LeafKey: AccountLeafKey.String(), - CID: AccountCID.String(), - Diff: true, - Balance: Account.Balance.Uint64(), - Nonce: Account.Nonce, - CodeHash: AccountCodeHash.String(), - StorageRoot: Account.Root.String(), - Removed: false, - })) - require.NoError(t, insertStateCID(database, stateModel{ - BlockNumber: BlockNumber4, - BlockHash: NonCanonicalHash4.String(), - LeafKey: AccountLeafKey.String(), - CID: AccountCID.String(), - Diff: true, - Balance: big.NewInt(123).Uint64(), - Nonce: Account.Nonce, - CodeHash: AccountCodeHash.String(), - StorageRoot: Account.Root.String(), - Removed: false, - })) - require.NoError(t, insertStateCID(database, stateModel{ - BlockNumber: BlockNumber5, - BlockHash: BlockHash5.String(), - LeafKey: AccountLeafKey.String(), - CID: RemovedNodeStateCID, - Diff: true, - Removed: true, - })) - require.NoError(t, insertStorageCID(database, storageModel{ - BlockNumber: BlockNumber.Uint64(), - BlockHash: BlockHash.String(), - LeafKey: AccountLeafKey.String(), - StorageLeafKey: StorageLeafKey.String(), - StorageCID: StorageCID.String(), - Diff: true, - Value: StoredValueRLP, - Removed: false, - })) - require.NoError(t, insertStorageCID(database, storageModel{ - BlockNumber: BlockNumber2, - BlockHash: BlockHash2.String(), - LeafKey: AccountLeafKey.String(), - StorageLeafKey: StorageLeafKey.String(), - StorageCID: RemovedNodeStorageCID, - Diff: true, - Value: []byte{}, - Removed: true, - })) - require.NoError(t, insertStorageCID(database, storageModel{ - BlockNumber: BlockNumber3, - BlockHash: BlockHash3.String(), - LeafKey: AccountLeafKey.String(), - StorageLeafKey: StorageLeafKey.String(), - StorageCID: StorageCID.String(), - Diff: true, - Value: StoredValueRLP2, - Removed: false, - })) - require.NoError(t, insertStorageCID(database, storageModel{ - BlockNumber: BlockNumber4, - BlockHash: NonCanonicalHash4.String(), - LeafKey: AccountLeafKey.String(), - StorageLeafKey: StorageLeafKey.String(), - StorageCID: StorageCID.String(), - Diff: true, - Value: NonCanonStoredValueRLP, - Removed: false, - })) - require.NoError(t, insertContractCode(database)) - db, err := statedb.NewStateDatabaseWithPgxPool(pool) + driver := statedb.NewPGXDriverFromPool(context.Background(), pool) + database := statedb.NewPostgresDB(driver) + insertSuiteData(t, database) + + db := statedb.NewStateDatabase(database) require.NoError(t, err) - - t.Run("Database", func(t *testing.T) { - size, err := db.ContractCodeSize(AccountCodeHash) - require.NoError(t, err) - require.Equal(t, len(AccountCode), size) - - code, err := db.ContractCode(AccountCodeHash) - require.NoError(t, err) - require.Equal(t, AccountCode, code) - - acct, err := db.StateAccount(AccountLeafKey, BlockHash) - require.NoError(t, err) - require.Equal(t, &Account, acct) - - acct2, err := db.StateAccount(AccountLeafKey, BlockHash2) - require.NoError(t, err) - require.Equal(t, &Account, acct2) - - acct3, err := db.StateAccount(AccountLeafKey, BlockHash3) - require.NoError(t, err) - require.Equal(t, &Account, acct3) - - // check that we don't get the non-canonical account - acct4, err := db.StateAccount(AccountLeafKey, BlockHash4) - require.NoError(t, err) - require.Equal(t, &Account, acct4) - - acct5, err := db.StateAccount(AccountLeafKey, BlockHash5) - require.NoError(t, err) - require.Nil(t, acct5) - - acct6, err := db.StateAccount(AccountLeafKey, BlockHash6) - require.NoError(t, err) - require.Nil(t, acct6) - - val, err := db.StorageValue(AccountLeafKey, StorageLeafKey, BlockHash) - require.NoError(t, err) - require.Equal(t, StoredValueRLP, val) - - val2, err := db.StorageValue(AccountLeafKey, StorageLeafKey, BlockHash2) - require.NoError(t, err) - require.Nil(t, val2) - - val3, err := db.StorageValue(AccountLeafKey, StorageLeafKey, BlockHash3) - require.NoError(t, err) - require.Equal(t, StoredValueRLP2, val3) - - // this checks that we don't get the non-canonical result - val4, err := db.StorageValue(AccountLeafKey, StorageLeafKey, BlockHash4) - require.NoError(t, err) - require.Equal(t, StoredValueRLP2, val4) - - // this checks that when the entire account was deleted, we return nil result for storage slot - val5, err := db.StorageValue(AccountLeafKey, StorageLeafKey, BlockHash5) - require.NoError(t, err) - require.Nil(t, val5) - - val6, err := db.StorageValue(AccountLeafKey, StorageLeafKey, BlockHash6) - require.NoError(t, err) - require.Nil(t, val6) - }) - - t.Run("StateDB", func(t *testing.T) { - sdb, err := statedb.New(BlockHash, db) - require.NoError(t, err) - - checkAccountUnchanged := func() { - require.Equal(t, Account.Balance, sdb.GetBalance(AccountAddress)) - require.Equal(t, Account.Nonce, sdb.GetNonce(AccountAddress)) - require.Equal(t, StoredValue, sdb.GetState(AccountAddress, StorageLeafKey)) - require.Equal(t, AccountCodeHash, sdb.GetCodeHash(AccountAddress)) - require.Equal(t, AccountCode, sdb.GetCode(AccountAddress)) - require.Equal(t, len(AccountCode), sdb.GetCodeSize(AccountAddress)) - } - - require.True(t, sdb.Exist(AccountAddress)) - checkAccountUnchanged() - - id := sdb.Snapshot() - - newStorage := crypto.Keccak256Hash([]byte{5, 4, 3, 2, 1}) - newCode := []byte{1, 3, 3, 7} - - sdb.SetBalance(AccountAddress, big.NewInt(300)) - sdb.AddBalance(AccountAddress, big.NewInt(200)) - sdb.SubBalance(AccountAddress, big.NewInt(100)) - sdb.SetNonce(AccountAddress, 42) - sdb.SetState(AccountAddress, StorageLeafKey, newStorage) - sdb.SetCode(AccountAddress, newCode) - - require.Equal(t, big.NewInt(400), sdb.GetBalance(AccountAddress)) - require.Equal(t, uint64(42), sdb.GetNonce(AccountAddress)) - require.Equal(t, newStorage, sdb.GetState(AccountAddress, StorageLeafKey)) - require.Equal(t, newCode, sdb.GetCode(AccountAddress)) - - sdb.AddSlotToAccessList(AccountAddress, StorageLeafKey) - require.True(t, sdb.AddressInAccessList(AccountAddress)) - hasAddr, hasSlot := sdb.SlotInAccessList(AccountAddress, StorageLeafKey) - require.True(t, hasAddr) - require.True(t, hasSlot) - - sdb.RevertToSnapshot(id) - - checkAccountUnchanged() - require.False(t, sdb.AddressInAccessList(AccountAddress)) - hasAddr, hasSlot = sdb.SlotInAccessList(AccountAddress, StorageLeafKey) - require.False(t, hasAddr) - require.False(t, hasSlot) - }) + testSuite(t, db) } func TestSQLXSuite(t *testing.T) { - testConfig, err := getTestConfig() + testConfig, err := postgres.DefaultConfig.WithEnv() require.NoError(t, err) - pool, err := statedb.NewSQLXPool(testCtx, testConfig) + pool, err := postgres.ConnectSQLX(testCtx, testConfig) if err != nil { t.Fatal(err) } - driver, err := statedb.NewSQLXDriverFromPool(context.Background(), pool) - if err != nil { - t.Fatal(err) - } - database := statedb.NewPostgresDB(driver) t.Cleanup(func() { tx, err := pool.Begin() require.NoError(t, err) @@ -334,6 +138,17 @@ func TestSQLXSuite(t *testing.T) { } require.NoError(t, tx.Commit()) }) + + driver := statedb.NewSQLXDriverFromPool(context.Background(), pool) + database := statedb.NewPostgresDB(driver) + insertSuiteData(t, database) + + db := statedb.NewStateDatabase(database) + require.NoError(t, err) + testSuite(t, db) +} + +func insertSuiteData(t *testing.T, database statedb.Database) { require.NoError(t, insertHeaderCID(database, BlockHash.String(), BlockParentHash.String(), BlockNumber.Uint64())) require.NoError(t, insertHeaderCID(database, BlockHash2.String(), BlockHash.String(), BlockNumber2)) require.NoError(t, insertHeaderCID(database, BlockHash3.String(), BlockHash2.String(), BlockNumber3)) @@ -415,10 +230,9 @@ func TestSQLXSuite(t *testing.T) { Removed: false, })) require.NoError(t, insertContractCode(database)) +} - db, err := statedb.NewStateDatabaseWithSqlxPool(pool) - require.NoError(t, err) - +func testSuite(t *testing.T, db statedb.StateDatabase) { t.Run("Database", func(t *testing.T) { size, err := db.ContractCodeSize(AccountCodeHash) require.NoError(t, err) @@ -648,17 +462,3 @@ func insertContractCode(db statedb.Database) error { _, err := db.Exec(testCtx, sql, BlockNumber.Uint64(), AccountCodeCID.String(), AccountCode) return err } - -func getTestConfig() (conf statedb.Config, err error) { - port, err := strconv.Atoi(os.Getenv("DATABASE_PORT")) - if err != nil { - return - } - return statedb.Config{ - Hostname: os.Getenv("DATABASE_HOSTNAME"), - DatabaseName: os.Getenv("DATABASE_NAME"), - Username: os.Getenv("DATABASE_USER"), - Password: os.Getenv("DATABASE_PASSWORD"), - Port: port, - }, nil -} From e3e4e1e41c1338555b854b66e2d54c69dc46cb76 Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Thu, 20 Apr 2023 17:29:53 +0800 Subject: [PATCH 3/8] Fix storage paths GetState takes a slot, not a leaf path --- state_object.go | 3 ++- statedb_test.go | 15 ++++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/state_object.go b/state_object.go index 31fbc81..b18c5bc 100644 --- a/state_object.go +++ b/state_object.go @@ -159,7 +159,8 @@ func (s *stateObject) GetCommittedState(db StateDatabase, key common.Hash) commo } // If no live objects are available, load from database start := time.Now() - enc, err := db.StorageValue(s.addrHash, key, s.blockHash) + keyHash := crypto.Keccak256Hash(key[:]) + enc, err := db.StorageValue(s.addrHash, keyHash, s.blockHash) if metrics.EnabledExpensive { s.db.StorageReads += time.Since(start) } diff --git a/statedb_test.go b/statedb_test.go index bb097b9..e26dd4f 100644 --- a/statedb_test.go +++ b/statedb_test.go @@ -64,7 +64,8 @@ var ( Root: common.Hash{}, } - StorageLeafKey = crypto.Keccak256Hash(common.HexToHash("0").Bytes()) + StorageSlot = common.HexToHash("0") + StorageLeafKey = crypto.Keccak256Hash(StorageSlot[:]) StoredValue = crypto.Keccak256Hash([]byte{1, 2, 3, 4, 5}) StoragePartialPath = []byte{0, 1, 0, 2, 0, 4} @@ -301,7 +302,7 @@ func testSuite(t *testing.T, db statedb.StateDatabase) { checkAccountUnchanged := func() { require.Equal(t, Account.Balance, sdb.GetBalance(AccountAddress)) require.Equal(t, Account.Nonce, sdb.GetNonce(AccountAddress)) - require.Equal(t, StoredValue, sdb.GetState(AccountAddress, StorageLeafKey)) + require.Equal(t, StoredValue, sdb.GetState(AccountAddress, StorageSlot)) require.Equal(t, AccountCodeHash, sdb.GetCodeHash(AccountAddress)) require.Equal(t, AccountCode, sdb.GetCode(AccountAddress)) require.Equal(t, len(AccountCode), sdb.GetCodeSize(AccountAddress)) @@ -319,17 +320,17 @@ func testSuite(t *testing.T, db statedb.StateDatabase) { sdb.AddBalance(AccountAddress, big.NewInt(200)) sdb.SubBalance(AccountAddress, big.NewInt(100)) sdb.SetNonce(AccountAddress, 42) - sdb.SetState(AccountAddress, StorageLeafKey, newStorage) + sdb.SetState(AccountAddress, StorageSlot, newStorage) sdb.SetCode(AccountAddress, newCode) require.Equal(t, big.NewInt(400), sdb.GetBalance(AccountAddress)) require.Equal(t, uint64(42), sdb.GetNonce(AccountAddress)) - require.Equal(t, newStorage, sdb.GetState(AccountAddress, StorageLeafKey)) + require.Equal(t, newStorage, sdb.GetState(AccountAddress, StorageSlot)) require.Equal(t, newCode, sdb.GetCode(AccountAddress)) - sdb.AddSlotToAccessList(AccountAddress, StorageLeafKey) + sdb.AddSlotToAccessList(AccountAddress, StorageSlot) require.True(t, sdb.AddressInAccessList(AccountAddress)) - hasAddr, hasSlot := sdb.SlotInAccessList(AccountAddress, StorageLeafKey) + hasAddr, hasSlot := sdb.SlotInAccessList(AccountAddress, StorageSlot) require.True(t, hasAddr) require.True(t, hasSlot) @@ -337,7 +338,7 @@ func testSuite(t *testing.T, db statedb.StateDatabase) { checkAccountUnchanged() require.False(t, sdb.AddressInAccessList(AccountAddress)) - hasAddr, hasSlot = sdb.SlotInAccessList(AccountAddress, StorageLeafKey) + hasAddr, hasSlot = sdb.SlotInAccessList(AccountAddress, StorageSlot) require.False(t, hasAddr) require.False(t, hasSlot) }) From 5790ff0439dda95b1a913fe8a51def5ea7aa6257 Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Thu, 20 Apr 2023 18:24:19 +0800 Subject: [PATCH 4/8] Add trie_by_cid/state.TryGetNode used in ipld-eth-server --- trie_by_cid/state/database.go | 1 + trie_by_cid/trie/encoding.go | 15 +++++++ trie_by_cid/trie/secure_trie.go | 8 ++++ trie_by_cid/trie/trie.go | 78 +++++++++++++++++++++++++++++++++ 4 files changed, 102 insertions(+) diff --git a/trie_by_cid/state/database.go b/trie_by_cid/state/database.go index 4c6a182..07fd020 100644 --- a/trie_by_cid/state/database.go +++ b/trie_by_cid/state/database.go @@ -42,6 +42,7 @@ type Database interface { // Trie is a Ethereum Merkle Patricia trie. type Trie interface { TryGet(key []byte) ([]byte, error) + TryGetNode(path []byte) ([]byte, int, error) TryGetAccount(key []byte) (*types.StateAccount, error) Hash() common.Hash NodeIterator(startKey []byte) trie.NodeIterator diff --git a/trie_by_cid/trie/encoding.go b/trie_by_cid/trie/encoding.go index 5871a66..381883a 100644 --- a/trie_by_cid/trie/encoding.go +++ b/trie_by_cid/trie/encoding.go @@ -16,6 +16,21 @@ package trie +// CompactToHex converts a compact encoded path to hex format +func CompactToHex(compact []byte) []byte { + if len(compact) == 0 { + return compact + } + base := keybytesToHex(compact) + // delete terminator flag + if base[0] < 2 { + base = base[:len(base)-1] + } + // apply odd flag + chop := 2 - base[0]&1 + return base[chop:] +} + func keybytesToHex(str []byte) []byte { l := len(str)*2 + 1 var nibbles = make([]byte, l) diff --git a/trie_by_cid/trie/secure_trie.go b/trie_by_cid/trie/secure_trie.go index 731b095..6e4b4ca 100644 --- a/trie_by_cid/trie/secure_trie.go +++ b/trie_by_cid/trie/secure_trie.go @@ -95,6 +95,14 @@ func (t *StateTrie) TryGetAccount(key []byte) (*types.StateAccount, error) { return &ret, err } +// TryGetNode attempts to retrieve a trie node by compact-encoded path. It is not +// possible to use keybyte-encoding as the path might contain odd nibbles. +// If the specified trie node is not in the trie, nil will be returned. +// If a trie node is not found in the database, a MissingNodeError is returned. +func (t *StateTrie) TryGetNode(path []byte) ([]byte, int, error) { + return t.trie.TryGetNode(path) +} + // Hash returns the root hash of StateTrie. It does not write to the // database and can be used even if the trie doesn't have one. func (t *StateTrie) Hash() common.Hash { diff --git a/trie_by_cid/trie/trie.go b/trie_by_cid/trie/trie.go index 6adb08b..f82c128 100644 --- a/trie_by_cid/trie/trie.go +++ b/trie_by_cid/trie/trie.go @@ -3,6 +3,7 @@ package trie import ( "bytes" + "errors" "fmt" "github.com/ethereum/go-ethereum/common" @@ -125,6 +126,83 @@ func (t *Trie) tryGet(origNode node, key []byte, pos int) (value []byte, newnode } } +// TryGetNode attempts to retrieve a trie node by compact-encoded path. It is not +// possible to use keybyte-encoding as the path might contain odd nibbles. +func (t *Trie) TryGetNode(path []byte) ([]byte, int, error) { + item, newroot, resolved, err := t.tryGetNode(t.root, CompactToHex(path), 0) + if err != nil { + return nil, resolved, err + } + if resolved > 0 { + t.root = newroot + } + if item == nil { + return nil, resolved, nil + } + return item, resolved, err +} + +func (t *Trie) tryGetNode(origNode node, path []byte, pos int) (item []byte, newnode node, resolved int, err error) { + // If non-existent path requested, abort + if origNode == nil { + return nil, nil, 0, nil + } + // If we reached the requested path, return the current node + if pos >= len(path) { + // Although we most probably have the original node expanded, encoding + // that into consensus form can be nasty (needs to cascade down) and + // time consuming. Instead, just pull the hash up from disk directly. + var hash hashNode + if node, ok := origNode.(hashNode); ok { + hash = node + } else { + hash, _ = origNode.cache() + } + if hash == nil { + return nil, origNode, 0, errors.New("non-consensus node") + } + blob, err := t.db.Node(hash) + return blob, origNode, 1, err + } + // Path still needs to be traversed, descend into children + switch n := (origNode).(type) { + case valueNode: + // Path prematurely ended, abort + return nil, nil, 0, nil + + case *shortNode: + if len(path)-pos < len(n.Key) || !bytes.Equal(n.Key, path[pos:pos+len(n.Key)]) { + // Path branches off from short node + return nil, n, 0, nil + } + item, newnode, resolved, err = t.tryGetNode(n.Val, path, pos+len(n.Key)) + if err == nil && resolved > 0 { + n = n.copy() + n.Val = newnode + } + return item, n, resolved, err + + case *fullNode: + item, newnode, resolved, err = t.tryGetNode(n.Children[path[pos]], path, pos+1) + if err == nil && resolved > 0 { + n = n.copy() + n.Children[path[pos]] = newnode + } + return item, n, resolved, err + + case hashNode: + child, err := t.resolveHash(n, path[:pos]) + if err != nil { + return nil, n, 1, err + } + item, newnode, resolved, err := t.tryGetNode(child, path, pos) + return item, newnode, resolved + 1, err + + default: + panic(fmt.Sprintf("%T: invalid node: %v", origNode, origNode)) + } +} + // resolveHash loads node from the underlying database with the provided // node hash and path prefix. func (t *Trie) resolveHash(n hashNode, prefix []byte) (node, error) { From 3625e2294982a25e6ba582af8409f7413222471d Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Thu, 20 Apr 2023 18:28:54 +0800 Subject: [PATCH 5/8] Reorganize packages * direct_by_leaf/ is original StateDB package * sql/ for SQL DB interfaces --- database.go | 28 ---------------- .../access_list.go | 2 +- journal.go => direct_by_leaf/journal.go | 2 +- sql.go => direct_by_leaf/sql.go | 2 +- .../state_database.go | 7 ++-- .../state_object.go | 2 +- statedb.go => direct_by_leaf/statedb.go | 2 +- .../statedb_test.go | 33 +++++++++---------- .../transient_storage.go | 2 +- interfaces.go => sql/interfaces.go | 14 ++------ pgx.go => sql/pgx.go | 2 +- sqlx.go => sql/sqlx.go | 2 +- 12 files changed, 30 insertions(+), 68 deletions(-) delete mode 100644 database.go rename access_list.go => direct_by_leaf/access_list.go (99%) rename journal.go => direct_by_leaf/journal.go (99%) rename sql.go => direct_by_leaf/sql.go (98%) rename state_database.go => direct_by_leaf/state_database.go (96%) rename state_object.go => direct_by_leaf/state_object.go (99%) rename statedb.go => direct_by_leaf/statedb.go (99%) rename statedb_test.go => direct_by_leaf/statedb_test.go (93%) rename transient_storage.go => direct_by_leaf/transient_storage.go (98%) rename interfaces.go => sql/interfaces.go (70%) rename pgx.go => sql/pgx.go (97%) rename sqlx.go => sql/sqlx.go (96%) diff --git a/database.go b/database.go deleted file mode 100644 index 35e6ace..0000000 --- a/database.go +++ /dev/null @@ -1,28 +0,0 @@ -package ipld_eth_statedb - -var _ Database = &DB{} - -// NewPostgresDB returns a postgres.DB using the provided driver -func NewPostgresDB(driver Driver) *DB { - return &DB{driver} -} - -// DB implements sql.Database using a configured driver and Postgres statement syntax -type DB struct { - Driver -} - -// GetContractCodeStmt satisfies the Statements interface -func (db *DB) GetContractCodeStmt() string { - return GetContractCodePgStr -} - -// GetStateAccountStmt satisfies the Statements interface -func (db *DB) GetStateAccountStmt() string { - return GetStateAccount -} - -// GetStorageSlotStmt satisfies the Statements interface -func (db *DB) GetStorageSlotStmt() string { - return GetStorageSlot -} diff --git a/access_list.go b/direct_by_leaf/access_list.go similarity index 99% rename from access_list.go rename to direct_by_leaf/access_list.go index 7aa2579..7e17a55 100644 --- a/access_list.go +++ b/direct_by_leaf/access_list.go @@ -1,4 +1,4 @@ -package ipld_eth_statedb +package state import ( "github.com/ethereum/go-ethereum/common" diff --git a/journal.go b/direct_by_leaf/journal.go similarity index 99% rename from journal.go rename to direct_by_leaf/journal.go index 8e0c655..0ea1fb5 100644 --- a/journal.go +++ b/direct_by_leaf/journal.go @@ -1,4 +1,4 @@ -package ipld_eth_statedb +package state import ( "math/big" diff --git a/sql.go b/direct_by_leaf/sql.go similarity index 98% rename from sql.go rename to direct_by_leaf/sql.go index 09b2bc8..0949cb4 100644 --- a/sql.go +++ b/direct_by_leaf/sql.go @@ -1,4 +1,4 @@ -package ipld_eth_statedb +package state const ( GetContractCodePgStr = `SELECT data FROM ipld.blocks WHERE key = $1` diff --git a/state_database.go b/direct_by_leaf/state_database.go similarity index 96% rename from state_database.go rename to direct_by_leaf/state_database.go index cb2c7b4..65204bc 100644 --- a/state_database.go +++ b/direct_by_leaf/state_database.go @@ -1,4 +1,4 @@ -package ipld_eth_statedb +package state import ( "context" @@ -14,6 +14,7 @@ import ( "github.com/ethereum/go-ethereum/statediff/indexer/ipld" util "github.com/cerc-io/ipld-eth-statedb/internal" + "github.com/cerc-io/ipld-eth-statedb/sql" ) const ( @@ -41,13 +42,13 @@ type StateDatabase interface { var _ StateDatabase = &stateDatabase{} type stateDatabase struct { - db Database + db sql.Database codeSizeCache *lru.Cache codeCache *fastcache.Cache } // NewStateDatabase returns a new Database implementation using the passed parameters -func NewStateDatabase(db Database) *stateDatabase { +func NewStateDatabase(db sql.Database) *stateDatabase { csc, _ := lru.New(codeSizeCacheSize) return &stateDatabase{ db: db, diff --git a/state_object.go b/direct_by_leaf/state_object.go similarity index 99% rename from state_object.go rename to direct_by_leaf/state_object.go index b18c5bc..d388589 100644 --- a/state_object.go +++ b/direct_by_leaf/state_object.go @@ -1,4 +1,4 @@ -package ipld_eth_statedb +package state import ( "bytes" diff --git a/statedb.go b/direct_by_leaf/statedb.go similarity index 99% rename from statedb.go rename to direct_by_leaf/statedb.go index 8933178..ff2c0fd 100644 --- a/statedb.go +++ b/direct_by_leaf/statedb.go @@ -1,4 +1,4 @@ -package ipld_eth_statedb +package state import ( "fmt" diff --git a/statedb_test.go b/direct_by_leaf/statedb_test.go similarity index 93% rename from statedb_test.go rename to direct_by_leaf/statedb_test.go index e26dd4f..ad46f0d 100644 --- a/statedb_test.go +++ b/direct_by_leaf/statedb_test.go @@ -1,4 +1,4 @@ -package ipld_eth_statedb_test +package state_test import ( "context" @@ -16,8 +16,9 @@ import ( "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" "github.com/ethereum/go-ethereum/statediff/indexer/ipld" - statedb "github.com/cerc-io/ipld-eth-statedb" + state "github.com/cerc-io/ipld-eth-statedb/direct_by_leaf" util "github.com/cerc-io/ipld-eth-statedb/internal" + "github.com/cerc-io/ipld-eth-statedb/sql" ) var ( @@ -52,7 +53,7 @@ var ( AccountPK, _ = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a") AccountAddress = crypto.PubkeyToAddress(AccountPK.PublicKey) //0x703c4b2bD70c169f5717101CaeE543299Fc946C7 - AccountLeafKey = crypto.Keccak256Hash(AccountAddress.Bytes()) + AccountLeafKey = crypto.Keccak256Hash(AccountAddress[:]) AccountCode = []byte{0, 1, 2, 3, 4, 5, 6, 7} AccountCodeHash = crypto.Keccak256Hash(AccountCode) @@ -73,7 +74,7 @@ var ( accountRLP, _ = rlp.EncodeToBytes(&Account) accountAndLeafRLP, _ = rlp.EncodeToBytes(&[]interface{}{AccountLeafKey, accountRLP}) AccountCID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, accountAndLeafRLP, multihash.KECCAK_256) - AccountCodeCID, _ = util.Keccak256ToCid(ipld.RawBinary, AccountCodeHash.Bytes()) + AccountCodeCID, _ = util.Keccak256ToCid(ipld.RawBinary, AccountCodeHash[:]) StoredValueRLP, _ = rlp.EncodeToBytes(StoredValue) StoredValueRLP2, _ = rlp.EncodeToBytes("something") @@ -108,11 +109,10 @@ func TestPGXSuite(t *testing.T) { require.NoError(t, tx.Commit(testCtx)) }) - driver := statedb.NewPGXDriverFromPool(context.Background(), pool) - database := statedb.NewPostgresDB(driver) + database := sql.NewPGXDriverFromPool(context.Background(), pool) insertSuiteData(t, database) - db := statedb.NewStateDatabase(database) + db := state.NewStateDatabase(database) require.NoError(t, err) testSuite(t, db) } @@ -140,16 +140,15 @@ func TestSQLXSuite(t *testing.T) { require.NoError(t, tx.Commit()) }) - driver := statedb.NewSQLXDriverFromPool(context.Background(), pool) - database := statedb.NewPostgresDB(driver) + database := sql.NewSQLXDriverFromPool(context.Background(), pool) insertSuiteData(t, database) - db := statedb.NewStateDatabase(database) + db := state.NewStateDatabase(database) require.NoError(t, err) testSuite(t, db) } -func insertSuiteData(t *testing.T, database statedb.Database) { +func insertSuiteData(t *testing.T, database sql.Database) { require.NoError(t, insertHeaderCID(database, BlockHash.String(), BlockParentHash.String(), BlockNumber.Uint64())) require.NoError(t, insertHeaderCID(database, BlockHash2.String(), BlockHash.String(), BlockNumber2)) require.NoError(t, insertHeaderCID(database, BlockHash3.String(), BlockHash2.String(), BlockNumber3)) @@ -233,7 +232,7 @@ func insertSuiteData(t *testing.T, database statedb.Database) { require.NoError(t, insertContractCode(database)) } -func testSuite(t *testing.T, db statedb.StateDatabase) { +func testSuite(t *testing.T, db state.StateDatabase) { t.Run("Database", func(t *testing.T) { size, err := db.ContractCodeSize(AccountCodeHash) require.NoError(t, err) @@ -296,7 +295,7 @@ func testSuite(t *testing.T, db statedb.StateDatabase) { }) t.Run("StateDB", func(t *testing.T) { - sdb, err := statedb.New(BlockHash, db) + sdb, err := state.New(BlockHash, db) require.NoError(t, err) checkAccountUnchanged := func() { @@ -344,7 +343,7 @@ func testSuite(t *testing.T, db statedb.StateDatabase) { }) } -func insertHeaderCID(db statedb.Database, blockHash, parentHash string, blockNumber uint64) error { +func insertHeaderCID(db sql.Database, blockHash, parentHash string, blockNumber uint64) error { cid, err := util.Keccak256ToCid(ipld.MEthHeader, common.HexToHash(blockHash).Bytes()) if err != nil { return err @@ -395,7 +394,7 @@ type stateModel struct { Removed bool } -func insertStateCID(db statedb.Database, cidModel stateModel) error { +func insertStateCID(db sql.Database, cidModel stateModel) error { sql := `INSERT INTO eth.state_cids ( block_number, header_id, @@ -434,7 +433,7 @@ type storageModel struct { Removed bool } -func insertStorageCID(db statedb.Database, cidModel storageModel) error { +func insertStorageCID(db sql.Database, cidModel storageModel) error { sql := `INSERT INTO eth.storage_cids ( block_number, header_id, @@ -458,7 +457,7 @@ func insertStorageCID(db statedb.Database, cidModel storageModel) error { return err } -func insertContractCode(db statedb.Database) error { +func insertContractCode(db sql.Database) error { sql := `INSERT INTO ipld.blocks (block_number, key, data) VALUES ($1, $2, $3)` _, err := db.Exec(testCtx, sql, BlockNumber.Uint64(), AccountCodeCID.String(), AccountCode) return err diff --git a/transient_storage.go b/direct_by_leaf/transient_storage.go similarity index 98% rename from transient_storage.go rename to direct_by_leaf/transient_storage.go index 91deec5..66e563e 100644 --- a/transient_storage.go +++ b/direct_by_leaf/transient_storage.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package ipld_eth_statedb +package state import ( "github.com/ethereum/go-ethereum/common" diff --git a/interfaces.go b/sql/interfaces.go similarity index 70% rename from interfaces.go rename to sql/interfaces.go index 66b0a98..448a647 100644 --- a/interfaces.go +++ b/sql/interfaces.go @@ -1,14 +1,11 @@ -package ipld_eth_statedb +package sql import ( "context" ) // Database interfaces to support multiple Postgres drivers -type Database interface { - Driver - Statements -} +type Database = Driver // Driver interface has all the methods required by a driver implementation to support the sql indexer type Driver interface { @@ -25,10 +22,3 @@ type ScannableRow interface { type Result interface { RowsAffected() (int64, error) } - -// Statements interface to accommodate different SQL query syntax -type Statements interface { - GetContractCodeStmt() string - GetStateAccountStmt() string - GetStorageSlotStmt() string -} diff --git a/pgx.go b/sql/pgx.go similarity index 97% rename from pgx.go rename to sql/pgx.go index e60a6d5..1ba3cc9 100644 --- a/pgx.go +++ b/sql/pgx.go @@ -1,4 +1,4 @@ -package ipld_eth_statedb +package sql import ( "context" diff --git a/sqlx.go b/sql/sqlx.go similarity index 96% rename from sqlx.go rename to sql/sqlx.go index 2eed275..700d8c2 100644 --- a/sqlx.go +++ b/sql/sqlx.go @@ -1,4 +1,4 @@ -package ipld_eth_statedb +package sql import ( "context" From 3f0e36c0a3917549b500d5c072ba366467f6ff7d Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Fri, 21 Apr 2023 00:10:11 +0800 Subject: [PATCH 6/8] Test NodeBlob --- trie_by_cid/trie/iterator_test.go | 114 +++++++++++++++++++----------- trie_by_cid/trie/util_test.go | 10 +++ 2 files changed, 82 insertions(+), 42 deletions(-) diff --git a/trie_by_cid/trie/iterator_test.go b/trie_by_cid/trie/iterator_test.go index 4df2ec4..eb1cf71 100644 --- a/trie_by_cid/trie/iterator_test.go +++ b/trie_by_cid/trie/iterator_test.go @@ -23,6 +23,7 @@ import ( "testing" "time" + "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" geth_trie "github.com/ethereum/go-ethereum/trie" @@ -41,18 +42,26 @@ var ( trieConfig = trie.Config{Cache: 256} ctx = context.Background() -) -var testdata1 = []kvs{ - {"barb", 0}, - {"bard", 1}, - {"bars", 2}, - {"bar", 3}, - {"fab", 4}, - {"food", 5}, - {"foos", 6}, - {"foo", 7}, -} + testdata0 = []kvs{ + {"one", 1}, + {"two", 2}, + {"three", 3}, + {"four", 4}, + {"five", 5}, + {"ten", 10}, + } + testdata1 = []kvs{ + {"barb", 0}, + {"bard", 1}, + {"bars", 2}, + {"bar", 3}, + {"fab", 4}, + {"food", 5}, + {"foos", 6}, + {"foo", 7}, + } +) func TestEmptyIterator(t *testing.T) { trie := trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase())) @@ -71,15 +80,7 @@ func TestIterator(t *testing.T) { edb := rawdb.NewMemoryDatabase() db := geth_trie.NewDatabase(edb) origtrie := geth_trie.NewEmpty(db) - vals := []kvs{ - {"one", 1}, - {"two", 2}, - {"three", 3}, - {"four", 4}, - {"five", 5}, - {"ten", 10}, - } - all, err := updateTrie(origtrie, vals) + all, err := updateTrie(origtrie, testdata0) if err != nil { t.Fatal(err) } @@ -103,22 +104,6 @@ func TestIterator(t *testing.T) { } } -func checkIteratorOrder(want []kvs, it *trie.Iterator) error { - for it.Next() { - if len(want) == 0 { - return fmt.Errorf("didn't expect any more values, got key %q", it.Key) - } - if !bytes.Equal(it.Key, []byte(want[0].k)) { - return fmt.Errorf("wrong key: got %q, want %q", it.Key, want[0].k) - } - want = want[1:] - } - if len(want) > 0 { - return fmt.Errorf("iterator ended early, want key %q", want[0]) - } - return nil -} - func TestIteratorSeek(t *testing.T) { edb := rawdb.NewMemoryDatabase() db := geth_trie.NewDatabase(edb) @@ -148,11 +133,56 @@ func TestIteratorSeek(t *testing.T) { } } -// returns a cache config with unique name (groupcache names are global) -func makeCacheConfig(t testing.TB) pgipfsethdb.CacheConfig { - return pgipfsethdb.CacheConfig{ - Name: t.Name(), - Size: 3000000, // 3MB - ExpiryDuration: time.Hour, +func checkIteratorOrder(want []kvs, it *trie.Iterator) error { + for it.Next() { + if len(want) == 0 { + return fmt.Errorf("didn't expect any more values, got key %q", it.Key) + } + if !bytes.Equal(it.Key, []byte(want[0].k)) { + return fmt.Errorf("wrong key: got %q, want %q", it.Key, want[0].k) + } + want = want[1:] + } + if len(want) > 0 { + return fmt.Errorf("iterator ended early, want key %q", want[0]) + } + return nil +} + +func TestIteratorNodeBlob(t *testing.T) { + edb := rawdb.NewMemoryDatabase() + db := geth_trie.NewDatabase(edb) + orig := geth_trie.NewEmpty(geth_trie.NewDatabase(rawdb.NewMemoryDatabase())) + if _, err := updateTrie(orig, testdata1); err != nil { + t.Fatal(err) + } + root := commitTrie(t, db, orig) + trie := indexTrie(t, edb, root) + + found := make(map[common.Hash][]byte) + it := trie.NodeIterator(nil) + for it.Next(true) { + if it.Hash() == (common.Hash{}) { + continue + } + found[it.Hash()] = it.NodeBlob() + } + + dbIter := edb.NewIterator(nil, nil) + defer dbIter.Release() + + var count int + for dbIter.Next() { + got, present := found[common.BytesToHash(dbIter.Key())] + if !present { + t.Fatalf("Missing trie node %v", dbIter.Key()) + } + if !bytes.Equal(got, dbIter.Value()) { + t.Fatalf("Unexpected trie node want %v got %v", dbIter.Value(), got) + } + count += 1 + } + if count != len(found) { + t.Fatal("Find extra trie node via iterator") } } diff --git a/trie_by_cid/trie/util_test.go b/trie_by_cid/trie/util_test.go index 4756314..fc84c6e 100644 --- a/trie_by_cid/trie/util_test.go +++ b/trie_by_cid/trie/util_test.go @@ -5,6 +5,7 @@ import ( "math/big" "math/rand" "testing" + "time" "github.com/jmoiron/sqlx" @@ -170,3 +171,12 @@ func TearDownDB(db *sqlx.DB) error { } return tx.Commit() } + +// returns a cache config with unique name (groupcache names are global) +func makeCacheConfig(t testing.TB) pgipfsethdb.CacheConfig { + return pgipfsethdb.CacheConfig{ + Name: t.Name(), + Size: 3000000, // 3MB + ExpiryDuration: time.Hour, + } +} From 88e7a394d0d2af3399a814fe1eed475555794deb Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Thu, 20 Apr 2023 23:53:37 +0800 Subject: [PATCH 7/8] Fix node resolution; use Cid struct as key was erroneously storing the cid in the fullNode's flag.cache --- internal/util.go | 2 +- trie_by_cid/state/database.go | 10 +++---- trie_by_cid/trie/database.go | 48 +++++++------------------------ trie_by_cid/trie/database_test.go | 2 +- trie_by_cid/trie/trie.go | 36 +++++++++++++++++------ 5 files changed, 45 insertions(+), 53 deletions(-) diff --git a/internal/util.go b/internal/util.go index 6c25c0f..a19df0a 100644 --- a/internal/util.go +++ b/internal/util.go @@ -1,4 +1,4 @@ -package util +package internal import ( "github.com/ipfs/go-cid" diff --git a/trie_by_cid/state/database.go b/trie_by_cid/state/database.go index 07fd020..30adbb7 100644 --- a/trie_by_cid/state/database.go +++ b/trie_by_cid/state/database.go @@ -10,6 +10,7 @@ import ( "github.com/ethereum/go-ethereum/statediff/indexer/ipld" lru "github.com/hashicorp/golang-lru" + "github.com/cerc-io/ipld-eth-statedb/internal" "github.com/cerc-io/ipld-eth-statedb/trie_by_cid/trie" ) @@ -97,11 +98,10 @@ func (db *cachingDB) ContractCode(codeHash common.Hash) ([]byte, error) { if code := db.codeCache.Get(nil, codeHash.Bytes()); len(code) > 0 { return code, nil } - // TODO - use non panicking - codeCID := ipld.Keccak256ToCid(ipld.RawBinary, codeHash.Bytes()) - // if err != nil { - // return nil, err - // } + codeCID, err := internal.Keccak256ToCid(ipld.RawBinary, codeHash.Bytes()) + if err != nil { + return nil, err + } code, err := db.db.DiskDB().Get(codeCID.Bytes()) if err != nil { return nil, err diff --git a/trie_by_cid/trie/database.go b/trie_by_cid/trie/database.go index 82bad23..d13cb49 100644 --- a/trie_by_cid/trie/database.go +++ b/trie_by_cid/trie/database.go @@ -22,12 +22,13 @@ import ( "github.com/VictoriaMetrics/fastcache" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/trie" + "github.com/ipfs/go-cid" ) -type CidBytes = []byte +type CidKey = cid.Cid -func isEmpty(key CidBytes) bool { - return len(key) == 0 +func isEmpty(key CidKey) bool { + return len(key.KeyString()) == 0 } // Database is an intermediate read-only layer between the trie data structures and @@ -73,57 +74,30 @@ func (db *Database) DiskDB() ethdb.KeyValueStore { return db.diskdb } -// node retrieves a cached trie node from memory, or returns nil if none can be -// found in the memory cache. -func (db *Database) node(key CidBytes) (node, error) { - // Retrieve the node from the clean cache if available - if db.cleans != nil { - if enc := db.cleans.Get(nil, key); enc != nil { - // The returned value from cache is in its own copy, - // safe to use mustDecodeNodeUnsafe for decoding. - return decodeNodeUnsafe(key, enc) - } - } - - // Content unavailable in memory, attempt to retrieve from disk - enc, err := db.diskdb.Get(key) - if err != nil { - return nil, err - } - if enc == nil { - return nil, nil - } - if db.cleans != nil { - db.cleans.Set(key, enc) - } - // The returned value from database is in its own copy, - // safe to use mustDecodeNodeUnsafe for decoding. - return decodeNodeUnsafe(key, enc) -} - -// Node retrieves an encoded cached trie node from memory. If it cannot be found -// cached, the method queries the persistent database for the content. -func (db *Database) Node(key CidBytes) ([]byte, error) { +// Node retrieves an encoded trie node by CID. If it cannot be found +// cached in memory, it queries the persistent database. +func (db *Database) Node(key CidKey) ([]byte, error) { // It doesn't make sense to retrieve the metaroot if isEmpty(key) { return nil, errors.New("not found") } + cidbytes := key.Bytes() // Retrieve the node from the clean cache if available if db.cleans != nil { - if enc := db.cleans.Get(nil, key); enc != nil { + if enc := db.cleans.Get(nil, cidbytes); enc != nil { return enc, nil } } // Content unavailable in memory, attempt to retrieve from disk - enc, err := db.diskdb.Get(key) + enc, err := db.diskdb.Get(cidbytes) if err != nil { return nil, err } if len(enc) != 0 { if db.cleans != nil { - db.cleans.Set(key[:], enc) + db.cleans.Set(cidbytes, enc) } return enc, nil } diff --git a/trie_by_cid/trie/database_test.go b/trie_by_cid/trie/database_test.go index 599cc58..a800135 100644 --- a/trie_by_cid/trie/database_test.go +++ b/trie_by_cid/trie/database_test.go @@ -28,7 +28,7 @@ import ( // to retrieve the meta root. func TestDatabaseMetarootFetch(t *testing.T) { db := trie.NewDatabase(memorydb.New()) - if _, err := db.Node(trie.CidBytes(nil)); err == nil { + if _, err := db.Node(trie.CidKey{}); err == nil { t.Fatalf("metaroot retrieval succeeded") } } diff --git a/trie_by_cid/trie/trie.go b/trie_by_cid/trie/trie.go index f82c128..78d917a 100644 --- a/trie_by_cid/trie/trie.go +++ b/trie_by_cid/trie/trie.go @@ -9,6 +9,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" + util "github.com/cerc-io/ipld-eth-statedb/internal" "github.com/ethereum/go-ethereum/statediff/indexer/ipld" ) @@ -161,7 +162,11 @@ func (t *Trie) tryGetNode(origNode node, path []byte, pos int) (item []byte, new if hash == nil { return nil, origNode, 0, errors.New("non-consensus node") } - blob, err := t.db.Node(hash) + cid, err := util.Keccak256ToCid(t.codec, hash) + if err != nil { + return nil, origNode, 0, err + } + blob, err := t.db.Node(cid) return blob, origNode, 1, err } // Path still needs to be traversed, descend into children @@ -206,8 +211,15 @@ func (t *Trie) tryGetNode(origNode node, path []byte, pos int) (item []byte, new // resolveHash loads node from the underlying database with the provided // node hash and path prefix. func (t *Trie) resolveHash(n hashNode, prefix []byte) (node, error) { - cid := ipld.Keccak256ToCid(t.codec, n) - node, err := t.db.node(cid.Bytes()) + cid, err := util.Keccak256ToCid(t.codec, n) + if err != nil { + return nil, err + } + enc, err := t.db.Node(cid) + if err != nil { + return nil, err + } + node, err := decodeNodeUnsafe(n, enc) if err != nil { return nil, err } @@ -220,8 +232,14 @@ func (t *Trie) resolveHash(n hashNode, prefix []byte) (node, error) { // resolveHash loads rlp-encoded node blob from the underlying database // with the provided node hash and path prefix. func (t *Trie) resolveBlob(n hashNode, prefix []byte) ([]byte, error) { - cid := ipld.Keccak256ToCid(t.codec, n) - blob, _ := t.db.Node(cid.Bytes()) + cid, err := util.Keccak256ToCid(t.codec, n) + if err != nil { + return nil, err + } + blob, err := t.db.Node(cid) + if err != nil { + return nil, err + } if len(blob) != 0 { return blob, nil } @@ -231,20 +249,20 @@ func (t *Trie) resolveBlob(n hashNode, prefix []byte) ([]byte, error) { // Hash returns the root hash of the trie. It does not write to the // database and can be used even if the trie doesn't have one. func (t *Trie) Hash() common.Hash { - hash, cached, _ := t.hashRoot() + hash, cached := t.hashRoot() t.root = cached return common.BytesToHash(hash.(hashNode)) } // hashRoot calculates the root hash of the given trie -func (t *Trie) hashRoot() (node, node, error) { +func (t *Trie) hashRoot() (node, node) { if t.root == nil { - return hashNode(emptyRoot.Bytes()), nil, nil + return hashNode(emptyRoot.Bytes()), nil } // If the number of changes is below 100, we let one thread handle it h := newHasher(t.unhashed >= 100) defer returnHasherToPool(h) hashed, cached := h.hash(t.root, true) t.unhashed = 0 - return hashed, cached, nil + return hashed, cached } From 4b2aba4fc740af1e5835a5df36d5d518e73599ef Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Mon, 24 Apr 2023 18:01:21 +0800 Subject: [PATCH 8/8] Add basic trie tests --- direct_by_leaf/statedb_test.go | 16 +-- trie_by_cid/trie/iterator_test.go | 10 +- trie_by_cid/trie/trie.go | 2 +- trie_by_cid/trie/trie_test.go | 173 ++++++++++++++++++++++++++++++ trie_by_cid/trie/util_test.go | 10 +- 5 files changed, 192 insertions(+), 19 deletions(-) create mode 100644 trie_by_cid/trie/trie_test.go diff --git a/direct_by_leaf/statedb_test.go b/direct_by_leaf/statedb_test.go index ad46f0d..03cd91b 100644 --- a/direct_by_leaf/statedb_test.go +++ b/direct_by_leaf/statedb_test.go @@ -97,10 +97,10 @@ func TestPGXSuite(t *testing.T) { tx, err := pool.Begin(testCtx) require.NoError(t, err) statements := []string{ - `DELETE FROM eth.header_cids`, - `DELETE FROM eth.state_cids`, - `DELETE FROM eth.storage_cids`, - `DELETE FROM ipld.blocks`, + `TRUNCATE eth.header_cids`, + `TRUNCATE eth.state_cids`, + `TRUNCATE eth.storage_cids`, + `TRUNCATE ipld.blocks`, } for _, stm := range statements { _, err = tx.Exec(testCtx, stm) @@ -128,10 +128,10 @@ func TestSQLXSuite(t *testing.T) { tx, err := pool.Begin() require.NoError(t, err) statements := []string{ - `DELETE FROM eth.header_cids`, - `DELETE FROM eth.state_cids`, - `DELETE FROM eth.storage_cids`, - `DELETE FROM ipld.blocks`, + `TRUNCATE eth.header_cids`, + `TRUNCATE eth.state_cids`, + `TRUNCATE eth.storage_cids`, + `TRUNCATE ipld.blocks`, } for _, stm := range statements { _, err = tx.Exec(stm) diff --git a/trie_by_cid/trie/iterator_test.go b/trie_by_cid/trie/iterator_test.go index eb1cf71..f7f0103 100644 --- a/trie_by_cid/trie/iterator_test.go +++ b/trie_by_cid/trie/iterator_test.go @@ -21,27 +21,19 @@ import ( "context" "fmt" "testing" - "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" geth_trie "github.com/ethereum/go-ethereum/trie" - pgipfsethdb "github.com/cerc-io/ipfs-ethdb/v5/postgres/v0" "github.com/cerc-io/ipld-eth-statedb/trie_by_cid/trie" ) var ( - cacheConfig = pgipfsethdb.CacheConfig{ - Name: "db", - Size: 3000000, // 3MB - ExpiryDuration: time.Hour, - } dbConfig, _ = postgres.DefaultConfig.WithEnv() trieConfig = trie.Config{Cache: 256} - - ctx = context.Background() + ctx = context.Background() testdata0 = []kvs{ {"one", 1}, diff --git a/trie_by_cid/trie/trie.go b/trie_by_cid/trie/trie.go index 78d917a..b2d8986 100644 --- a/trie_by_cid/trie/trie.go +++ b/trie_by_cid/trie/trie.go @@ -217,7 +217,7 @@ func (t *Trie) resolveHash(n hashNode, prefix []byte) (node, error) { } enc, err := t.db.Node(cid) if err != nil { - return nil, err + return nil, &MissingNodeError{Owner: t.owner, NodeHash: n, Path: prefix, err: err} } node, err := decodeNodeUnsafe(n, enc) if err != nil { diff --git a/trie_by_cid/trie/trie_test.go b/trie_by_cid/trie/trie_test.go new file mode 100644 index 0000000..0418409 --- /dev/null +++ b/trie_by_cid/trie/trie_test.go @@ -0,0 +1,173 @@ +// Copyright 2014 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package trie_test + +import ( + "fmt" + "math/big" + "math/rand" + "testing" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/rawdb" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/rlp" + geth_trie "github.com/ethereum/go-ethereum/trie" + + "github.com/cerc-io/ipld-eth-statedb/trie_by_cid/trie" +) + +func TestTrieEmpty(t *testing.T) { + trie := trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase())) + res := trie.Hash() + exp := types.EmptyRootHash + if res != exp { + t.Errorf("expected %x got %x", exp, res) + } +} + +func TestTrieMissingRoot(t *testing.T) { + root := common.HexToHash("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33") + tr, err := newStateTrie(root, trie.NewDatabase(rawdb.NewMemoryDatabase())) + if tr != nil { + t.Error("New returned non-nil trie for invalid root") + } + if _, ok := err.(*trie.MissingNodeError); !ok { + t.Errorf("New returned wrong error: %v", err) + } +} + +func TestTrieBasic(t *testing.T) { + edb := rawdb.NewMemoryDatabase() + db := geth_trie.NewDatabase(edb) + origtrie := geth_trie.NewEmpty(db) + origtrie.Update([]byte("foo"), packValue(842)) + expected := commitTrie(t, db, origtrie) + tr := indexTrie(t, edb, expected) + got := tr.Hash() + if expected != got { + t.Errorf("got %x expected %x", got, expected) + } + checkValue(t, tr, []byte("foo")) +} + +func TestTrieTiny(t *testing.T) { + // Create a realistic account trie to hash + _, accounts := makeAccounts(5) + edb := rawdb.NewMemoryDatabase() + db := geth_trie.NewDatabase(edb) + origtrie := geth_trie.NewEmpty(db) + + type testCase struct { + key, account []byte + root common.Hash + } + cases := []testCase{ + { + common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000001337"), + accounts[3], + common.HexToHash("8c6a85a4d9fda98feff88450299e574e5378e32391f75a055d470ac0653f1005"), + }, { + common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000001338"), + accounts[4], + common.HexToHash("ec63b967e98a5720e7f720482151963982890d82c9093c0d486b7eb8883a66b1"), + }, { + common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000001339"), + accounts[4], + common.HexToHash("0608c1d1dc3905fa22204c7a0e43644831c3b6d3def0f274be623a948197e64a"), + }, + } + for i, tc := range cases { + t.Run(fmt.Sprintf("case %d", i), func(t *testing.T) { + origtrie.Update(tc.key, tc.account) + trie := indexTrie(t, edb, commitTrie(t, db, origtrie)) + if exp, root := tc.root, trie.Hash(); exp != root { + t.Errorf("got %x, exp %x", root, exp) + } + checkValue(t, trie, tc.key) + }) + } +} + +func TestTrieMedium(t *testing.T) { + // Create a realistic account trie to hash + addresses, accounts := makeAccounts(1000) + edb := rawdb.NewMemoryDatabase() + db := geth_trie.NewDatabase(edb) + origtrie := geth_trie.NewEmpty(db) + var keys [][]byte + for i := 0; i < len(addresses); i++ { + key := crypto.Keccak256(addresses[i][:]) + if i%50 == 0 { + keys = append(keys, key) + } + origtrie.Update(key, accounts[i]) + } + tr := indexTrie(t, edb, commitTrie(t, db, origtrie)) + + root := tr.Hash() + exp := common.HexToHash("72f9d3f3fe1e1dd7b8936442e7642aef76371472d94319900790053c493f3fe6") + if exp != root { + t.Errorf("got %x, exp %x", root, exp) + } + + for _, key := range keys { + checkValue(t, tr, key) + } +} + +// Make deterministically random accounts +func makeAccounts(size int) (addresses [][20]byte, accounts [][]byte) { + random := rand.New(rand.NewSource(0)) + addresses = make([][20]byte, size) + for i := 0; i < len(addresses); i++ { + data := make([]byte, 20) + random.Read(data) + copy(addresses[i][:], data) + } + accounts = make([][]byte, len(addresses)) + for i := 0; i < len(accounts); i++ { + var ( + nonce = uint64(random.Int63()) + root = types.EmptyRootHash + code = crypto.Keccak256(nil) + ) + // The big.Rand function is not deterministic with regards to 64 vs 32 bit systems, + // and will consume different amount of data from the rand source. + // balance = new(big.Int).Rand(random, new(big.Int).Exp(common.Big2, common.Big256, nil)) + // Therefore, we instead just read via byte buffer + numBytes := random.Uint32() % 33 // [0, 32] bytes + balanceBytes := make([]byte, numBytes) + random.Read(balanceBytes) + balance := new(big.Int).SetBytes(balanceBytes) + acct := &types.StateAccount{Nonce: nonce, Balance: balance, Root: root, CodeHash: code} + data, _ := rlp.EncodeToBytes(acct) + accounts[i] = data + } + return addresses, accounts +} + +func checkValue(t *testing.T, tr *trie.Trie, key []byte) { + val, err := tr.TryGet(key) + if err != nil { + t.Fatalf("error getting node: %s", err) + } + if len(val) == 0 { + t.Errorf("failed to get value for %x", key) + } +} diff --git a/trie_by_cid/trie/util_test.go b/trie_by_cid/trie/util_test.go index fc84c6e..3272826 100644 --- a/trie_by_cid/trie/util_test.go +++ b/trie_by_cid/trie/util_test.go @@ -101,13 +101,21 @@ func indexTrie(t testing.TB, edb ethdb.Database, root common.Hash) *trie.Trie { ipfs_db := pgipfsethdb.NewDatabase(pg_db, makeCacheConfig(t)) sdb_db := state.NewDatabase(ipfs_db) - tr, err := trie.New(common.Hash{}, root, sdb_db.TrieDB(), ipld.MEthStateTrie) + tr, err := newStateTrie(root, sdb_db.TrieDB()) if err != nil { t.Fatal(err) } return tr } +func newStateTrie(root common.Hash, db *trie.Database) (*trie.Trie, error) { + tr, err := trie.New(common.Hash{}, root, db, ipld.MEthStateTrie) + if err != nil { + return nil, err + } + return tr, nil +} + // generates a random Geth LevelDB trie of n key-value pairs and corresponding value map func randomGethTrie(n int, db *geth_trie.Database) (*geth_trie.Trie, kvMap) { trie := geth_trie.NewEmpty(db)