Merge pull request #18 from cerc-io/roy/v5-dev
* Refactor redundant DB code * Fix storage paths GetState takes a slot, not a leaf path * Add trie_by_cid/state.TryGetNode used in ipld-eth-server * Reorganize packages * direct_by_leaf/ is original StateDB package * sql/ for SQL DB interfaces * Test NodeBlob * Fix node resolution; use Cid struct as key was erroneously storing the cid in the fullNode's flag.cache * Add basic trie tests
This commit is contained in:
GitHub
commit
0eba6888c1
90
config.go
90
config.go
@ -1,90 +0,0 @@
|
||||
package ipld_eth_statedb
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jackc/pgx/v4/pgxpool"
|
||||
"github.com/jmoiron/sqlx"
|
||||
|
||||
_ "github.com/lib/pq"
|
||||
)
|
||||
|
||||
type Config struct {
|
||||
Hostname string
|
||||
Port int
|
||||
DatabaseName string
|
||||
Username string
|
||||
Password string
|
||||
|
||||
ConnTimeout time.Duration
|
||||
MaxConns int
|
||||
MinConns int
|
||||
MaxConnLifetime time.Duration
|
||||
MaxConnIdleTime time.Duration
|
||||
MaxIdle int
|
||||
}
|
||||
|
||||
// DbConnectionString constructs and returns the connection string from the config (for sqlx driver)
|
||||
func (c Config) DbConnectionString() string {
|
||||
if len(c.Username) > 0 && len(c.Password) > 0 {
|
||||
return fmt.Sprintf("postgresql://%s:%s@%s:%d/%s?sslmode=disable",
|
||||
c.Username, c.Password, c.Hostname, c.Port, c.DatabaseName)
|
||||
}
|
||||
if len(c.Username) > 0 && len(c.Password) == 0 {
|
||||
return fmt.Sprintf("postgresql://%s@%s:%d/%s?sslmode=disable",
|
||||
c.Username, c.Hostname, c.Port, c.DatabaseName)
|
||||
}
|
||||
return fmt.Sprintf("postgresql://%s:%d/%s?sslmode=disable", c.Hostname, c.Port, c.DatabaseName)
|
||||
}
|
||||
|
||||
// NewPGXPool returns a new pgx conn pool
|
||||
func NewPGXPool(ctx context.Context, config Config) (*pgxpool.Pool, error) {
|
||||
pgConf, err := makePGXConfig(config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return pgxpool.ConnectConfig(ctx, pgConf)
|
||||
}
|
||||
|
||||
// NewSQLXPool returns a new sqlx conn pool
|
||||
func NewSQLXPool(ctx context.Context, config Config) (*sqlx.DB, error) {
|
||||
db, err := sqlx.ConnectContext(ctx, "postgres", config.DbConnectionString())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return db, nil
|
||||
}
|
||||
|
||||
// makePGXConfig creates a pgxpool.Config from the provided Config
|
||||
func makePGXConfig(config Config) (*pgxpool.Config, error) {
|
||||
conf, err := pgxpool.ParseConfig("")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
conf.ConnConfig.Config.Host = config.Hostname
|
||||
conf.ConnConfig.Config.Port = uint16(config.Port)
|
||||
conf.ConnConfig.Config.Database = config.DatabaseName
|
||||
conf.ConnConfig.Config.User = config.Username
|
||||
conf.ConnConfig.Config.Password = config.Password
|
||||
|
||||
if config.ConnTimeout != 0 {
|
||||
conf.ConnConfig.Config.ConnectTimeout = config.ConnTimeout
|
||||
}
|
||||
if config.MaxConns != 0 {
|
||||
conf.MaxConns = int32(config.MaxConns)
|
||||
}
|
||||
if config.MinConns != 0 {
|
||||
conf.MinConns = int32(config.MinConns)
|
||||
}
|
||||
if config.MaxConnLifetime != 0 {
|
||||
conf.MaxConnLifetime = config.MaxConnLifetime
|
||||
}
|
||||
if config.MaxConnIdleTime != 0 {
|
||||
conf.MaxConnIdleTime = config.MaxConnIdleTime
|
||||
}
|
||||
|
||||
return conf, nil
|
||||
}
|
||||
28
database.go
28
database.go
@ -1,28 +0,0 @@
|
||||
package ipld_eth_statedb
|
||||
|
||||
var _ Database = &DB{}
|
||||
|
||||
// NewPostgresDB returns a postgres.DB using the provided driver
|
||||
func NewPostgresDB(driver Driver) *DB {
|
||||
return &DB{driver}
|
||||
}
|
||||
|
||||
// DB implements sql.Database using a configured driver and Postgres statement syntax
|
||||
type DB struct {
|
||||
Driver
|
||||
}
|
||||
|
||||
// GetContractCodeStmt satisfies the Statements interface
|
||||
func (db *DB) GetContractCodeStmt() string {
|
||||
return GetContractCodePgStr
|
||||
}
|
||||
|
||||
// GetStateAccountStmt satisfies the Statements interface
|
||||
func (db *DB) GetStateAccountStmt() string {
|
||||
return GetStateAccount
|
||||
}
|
||||
|
||||
// GetStorageSlotStmt satisfies the Statements interface
|
||||
func (db *DB) GetStorageSlotStmt() string {
|
||||
return GetStorageSlot
|
||||
}
|
||||
@ -1,4 +1,4 @@
|
||||
package ipld_eth_statedb
|
||||
package state
|
||||
|
||||
import (
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
@ -1,4 +1,4 @@
|
||||
package ipld_eth_statedb
|
||||
package state
|
||||
|
||||
import (
|
||||
"math/big"
|
||||
@ -1,4 +1,4 @@
|
||||
package ipld_eth_statedb
|
||||
package state
|
||||
|
||||
const (
|
||||
GetContractCodePgStr = `SELECT data FROM ipld.blocks WHERE key = $1`
|
||||
@ -1,4 +1,4 @@
|
||||
package ipld_eth_statedb
|
||||
package state
|
||||
|
||||
import (
|
||||
"context"
|
||||
@ -8,14 +8,13 @@ import (
|
||||
|
||||
"github.com/VictoriaMetrics/fastcache"
|
||||
lru "github.com/hashicorp/golang-lru"
|
||||
"github.com/jackc/pgx/v4/pgxpool"
|
||||
"github.com/jmoiron/sqlx"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
|
||||
|
||||
util "github.com/cerc-io/ipld-eth-statedb/internal"
|
||||
"github.com/cerc-io/ipld-eth-statedb/sql"
|
||||
)
|
||||
|
||||
const (
|
||||
@ -43,29 +42,19 @@ type StateDatabase interface {
|
||||
var _ StateDatabase = &stateDatabase{}
|
||||
|
||||
type stateDatabase struct {
|
||||
db Database
|
||||
db sql.Database
|
||||
codeSizeCache *lru.Cache
|
||||
codeCache *fastcache.Cache
|
||||
}
|
||||
|
||||
// NewStateDatabaseWithPgxPool returns a new Database implementation using the provided postgres connection pool
|
||||
func NewStateDatabaseWithPgxPool(pgDb *pgxpool.Pool) (*stateDatabase, error) {
|
||||
// NewStateDatabase returns a new Database implementation using the passed parameters
|
||||
func NewStateDatabase(db sql.Database) *stateDatabase {
|
||||
csc, _ := lru.New(codeSizeCacheSize)
|
||||
return &stateDatabase{
|
||||
db: NewPostgresDB(&PGXDriver{db: pgDb}),
|
||||
db: db,
|
||||
codeSizeCache: csc,
|
||||
codeCache: fastcache.New(codeCacheSize),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// NewStateDatabaseWithSqlxPool returns a new Database implementation using the passed parameters
|
||||
func NewStateDatabaseWithSqlxPool(db *sqlx.DB) (*stateDatabase, error) {
|
||||
csc, _ := lru.New(codeSizeCacheSize)
|
||||
return &stateDatabase{
|
||||
db: NewPostgresDB(&SQLXDriver{db: db}),
|
||||
codeSizeCache: csc,
|
||||
codeCache: fastcache.New(codeCacheSize),
|
||||
}, nil
|
||||
}
|
||||
}
|
||||
|
||||
// ContractCode satisfies Database, it returns the contract code for a given codehash
|
||||
@ -1,4 +1,4 @@
|
||||
package ipld_eth_statedb
|
||||
package state
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
@ -159,7 +159,8 @@ func (s *stateObject) GetCommittedState(db StateDatabase, key common.Hash) commo
|
||||
}
|
||||
// If no live objects are available, load from database
|
||||
start := time.Now()
|
||||
enc, err := db.StorageValue(s.addrHash, key, s.blockHash)
|
||||
keyHash := crypto.Keccak256Hash(key[:])
|
||||
enc, err := db.StorageValue(s.addrHash, keyHash, s.blockHash)
|
||||
if metrics.EnabledExpensive {
|
||||
s.db.StorageReads += time.Since(start)
|
||||
}
|
||||
@ -1,4 +1,4 @@
|
||||
package ipld_eth_statedb
|
||||
package state
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
@ -1,10 +1,8 @@
|
||||
package ipld_eth_statedb_test
|
||||
package state_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"math/big"
|
||||
"os"
|
||||
"strconv"
|
||||
"testing"
|
||||
|
||||
"github.com/lib/pq"
|
||||
@ -15,10 +13,12 @@ import (
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/ethereum/go-ethereum/crypto"
|
||||
"github.com/ethereum/go-ethereum/rlp"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
|
||||
|
||||
statedb "github.com/cerc-io/ipld-eth-statedb"
|
||||
state "github.com/cerc-io/ipld-eth-statedb/direct_by_leaf"
|
||||
util "github.com/cerc-io/ipld-eth-statedb/internal"
|
||||
"github.com/cerc-io/ipld-eth-statedb/sql"
|
||||
)
|
||||
|
||||
var (
|
||||
@ -53,7 +53,7 @@ var (
|
||||
|
||||
AccountPK, _ = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a")
|
||||
AccountAddress = crypto.PubkeyToAddress(AccountPK.PublicKey) //0x703c4b2bD70c169f5717101CaeE543299Fc946C7
|
||||
AccountLeafKey = crypto.Keccak256Hash(AccountAddress.Bytes())
|
||||
AccountLeafKey = crypto.Keccak256Hash(AccountAddress[:])
|
||||
|
||||
AccountCode = []byte{0, 1, 2, 3, 4, 5, 6, 7}
|
||||
AccountCodeHash = crypto.Keccak256Hash(AccountCode)
|
||||
@ -65,7 +65,8 @@ var (
|
||||
Root: common.Hash{},
|
||||
}
|
||||
|
||||
StorageLeafKey = crypto.Keccak256Hash(common.HexToHash("0").Bytes())
|
||||
StorageSlot = common.HexToHash("0")
|
||||
StorageLeafKey = crypto.Keccak256Hash(StorageSlot[:])
|
||||
StoredValue = crypto.Keccak256Hash([]byte{1, 2, 3, 4, 5})
|
||||
StoragePartialPath = []byte{0, 1, 0, 2, 0, 4}
|
||||
|
||||
@ -73,7 +74,7 @@ var (
|
||||
accountRLP, _ = rlp.EncodeToBytes(&Account)
|
||||
accountAndLeafRLP, _ = rlp.EncodeToBytes(&[]interface{}{AccountLeafKey, accountRLP})
|
||||
AccountCID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, accountAndLeafRLP, multihash.KECCAK_256)
|
||||
AccountCodeCID, _ = util.Keccak256ToCid(ipld.RawBinary, AccountCodeHash.Bytes())
|
||||
AccountCodeCID, _ = util.Keccak256ToCid(ipld.RawBinary, AccountCodeHash[:])
|
||||
|
||||
StoredValueRLP, _ = rlp.EncodeToBytes(StoredValue)
|
||||
StoredValueRLP2, _ = rlp.EncodeToBytes("something")
|
||||
@ -86,25 +87,20 @@ var (
|
||||
)
|
||||
|
||||
func TestPGXSuite(t *testing.T) {
|
||||
testConfig, err := getTestConfig()
|
||||
testConfig, err := postgres.DefaultConfig.WithEnv()
|
||||
require.NoError(t, err)
|
||||
pool, err := statedb.NewPGXPool(testCtx, testConfig)
|
||||
pool, err := postgres.ConnectPGX(testCtx, testConfig)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
driver, err := statedb.NewPGXDriverFromPool(context.Background(), pool)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
database := statedb.NewPostgresDB(driver)
|
||||
t.Cleanup(func() {
|
||||
tx, err := pool.Begin(testCtx)
|
||||
require.NoError(t, err)
|
||||
statements := []string{
|
||||
`DELETE FROM eth.header_cids`,
|
||||
`DELETE FROM eth.state_cids`,
|
||||
`DELETE FROM eth.storage_cids`,
|
||||
`DELETE FROM ipld.blocks`,
|
||||
`TRUNCATE eth.header_cids`,
|
||||
`TRUNCATE eth.state_cids`,
|
||||
`TRUNCATE eth.storage_cids`,
|
||||
`TRUNCATE ipld.blocks`,
|
||||
}
|
||||
for _, stm := range statements {
|
||||
_, err = tx.Exec(testCtx, stm)
|
||||
@ -112,221 +108,30 @@ func TestPGXSuite(t *testing.T) {
|
||||
}
|
||||
require.NoError(t, tx.Commit(testCtx))
|
||||
})
|
||||
require.NoError(t, insertHeaderCID(database, BlockHash.String(), BlockParentHash.String(), BlockNumber.Uint64()))
|
||||
require.NoError(t, insertHeaderCID(database, BlockHash2.String(), BlockHash.String(), BlockNumber2))
|
||||
require.NoError(t, insertHeaderCID(database, BlockHash3.String(), BlockHash2.String(), BlockNumber3))
|
||||
require.NoError(t, insertHeaderCID(database, BlockHash4.String(), BlockHash3.String(), BlockNumber4))
|
||||
require.NoError(t, insertHeaderCID(database, NonCanonicalHash4.String(), BlockHash3.String(), BlockNumber4))
|
||||
require.NoError(t, insertHeaderCID(database, BlockHash5.String(), BlockHash4.String(), BlockNumber5))
|
||||
require.NoError(t, insertHeaderCID(database, NonCanonicalHash5.String(), NonCanonicalHash4.String(), BlockNumber5))
|
||||
require.NoError(t, insertHeaderCID(database, BlockHash6.String(), BlockHash5.String(), BlockNumber6))
|
||||
require.NoError(t, insertStateCID(database, stateModel{
|
||||
BlockNumber: BlockNumber.Uint64(),
|
||||
BlockHash: BlockHash.String(),
|
||||
LeafKey: AccountLeafKey.String(),
|
||||
CID: AccountCID.String(),
|
||||
Diff: true,
|
||||
Balance: Account.Balance.Uint64(),
|
||||
Nonce: Account.Nonce,
|
||||
CodeHash: AccountCodeHash.String(),
|
||||
StorageRoot: Account.Root.String(),
|
||||
Removed: false,
|
||||
}))
|
||||
require.NoError(t, insertStateCID(database, stateModel{
|
||||
BlockNumber: BlockNumber4,
|
||||
BlockHash: NonCanonicalHash4.String(),
|
||||
LeafKey: AccountLeafKey.String(),
|
||||
CID: AccountCID.String(),
|
||||
Diff: true,
|
||||
Balance: big.NewInt(123).Uint64(),
|
||||
Nonce: Account.Nonce,
|
||||
CodeHash: AccountCodeHash.String(),
|
||||
StorageRoot: Account.Root.String(),
|
||||
Removed: false,
|
||||
}))
|
||||
require.NoError(t, insertStateCID(database, stateModel{
|
||||
BlockNumber: BlockNumber5,
|
||||
BlockHash: BlockHash5.String(),
|
||||
LeafKey: AccountLeafKey.String(),
|
||||
CID: RemovedNodeStateCID,
|
||||
Diff: true,
|
||||
Removed: true,
|
||||
}))
|
||||
require.NoError(t, insertStorageCID(database, storageModel{
|
||||
BlockNumber: BlockNumber.Uint64(),
|
||||
BlockHash: BlockHash.String(),
|
||||
LeafKey: AccountLeafKey.String(),
|
||||
StorageLeafKey: StorageLeafKey.String(),
|
||||
StorageCID: StorageCID.String(),
|
||||
Diff: true,
|
||||
Value: StoredValueRLP,
|
||||
Removed: false,
|
||||
}))
|
||||
require.NoError(t, insertStorageCID(database, storageModel{
|
||||
BlockNumber: BlockNumber2,
|
||||
BlockHash: BlockHash2.String(),
|
||||
LeafKey: AccountLeafKey.String(),
|
||||
StorageLeafKey: StorageLeafKey.String(),
|
||||
StorageCID: RemovedNodeStorageCID,
|
||||
Diff: true,
|
||||
Value: []byte{},
|
||||
Removed: true,
|
||||
}))
|
||||
require.NoError(t, insertStorageCID(database, storageModel{
|
||||
BlockNumber: BlockNumber3,
|
||||
BlockHash: BlockHash3.String(),
|
||||
LeafKey: AccountLeafKey.String(),
|
||||
StorageLeafKey: StorageLeafKey.String(),
|
||||
StorageCID: StorageCID.String(),
|
||||
Diff: true,
|
||||
Value: StoredValueRLP2,
|
||||
Removed: false,
|
||||
}))
|
||||
require.NoError(t, insertStorageCID(database, storageModel{
|
||||
BlockNumber: BlockNumber4,
|
||||
BlockHash: NonCanonicalHash4.String(),
|
||||
LeafKey: AccountLeafKey.String(),
|
||||
StorageLeafKey: StorageLeafKey.String(),
|
||||
StorageCID: StorageCID.String(),
|
||||
Diff: true,
|
||||
Value: NonCanonStoredValueRLP,
|
||||
Removed: false,
|
||||
}))
|
||||
require.NoError(t, insertContractCode(database))
|
||||
|
||||
db, err := statedb.NewStateDatabaseWithPgxPool(pool)
|
||||
database := sql.NewPGXDriverFromPool(context.Background(), pool)
|
||||
insertSuiteData(t, database)
|
||||
|
||||
db := state.NewStateDatabase(database)
|
||||
require.NoError(t, err)
|
||||
|
||||
t.Run("Database", func(t *testing.T) {
|
||||
size, err := db.ContractCodeSize(AccountCodeHash)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, len(AccountCode), size)
|
||||
|
||||
code, err := db.ContractCode(AccountCodeHash)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, AccountCode, code)
|
||||
|
||||
acct, err := db.StateAccount(AccountLeafKey, BlockHash)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, &Account, acct)
|
||||
|
||||
acct2, err := db.StateAccount(AccountLeafKey, BlockHash2)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, &Account, acct2)
|
||||
|
||||
acct3, err := db.StateAccount(AccountLeafKey, BlockHash3)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, &Account, acct3)
|
||||
|
||||
// check that we don't get the non-canonical account
|
||||
acct4, err := db.StateAccount(AccountLeafKey, BlockHash4)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, &Account, acct4)
|
||||
|
||||
acct5, err := db.StateAccount(AccountLeafKey, BlockHash5)
|
||||
require.NoError(t, err)
|
||||
require.Nil(t, acct5)
|
||||
|
||||
acct6, err := db.StateAccount(AccountLeafKey, BlockHash6)
|
||||
require.NoError(t, err)
|
||||
require.Nil(t, acct6)
|
||||
|
||||
val, err := db.StorageValue(AccountLeafKey, StorageLeafKey, BlockHash)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, StoredValueRLP, val)
|
||||
|
||||
val2, err := db.StorageValue(AccountLeafKey, StorageLeafKey, BlockHash2)
|
||||
require.NoError(t, err)
|
||||
require.Nil(t, val2)
|
||||
|
||||
val3, err := db.StorageValue(AccountLeafKey, StorageLeafKey, BlockHash3)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, StoredValueRLP2, val3)
|
||||
|
||||
// this checks that we don't get the non-canonical result
|
||||
val4, err := db.StorageValue(AccountLeafKey, StorageLeafKey, BlockHash4)
|
||||
require.NoError(t, err)
|
||||
require.Equal(t, StoredValueRLP2, val4)
|
||||
|
||||
// this checks that when the entire account was deleted, we return nil result for storage slot
|
||||
val5, err := db.StorageValue(AccountLeafKey, StorageLeafKey, BlockHash5)
|
||||
require.NoError(t, err)
|
||||
require.Nil(t, val5)
|
||||
|
||||
val6, err := db.StorageValue(AccountLeafKey, StorageLeafKey, BlockHash6)
|
||||
require.NoError(t, err)
|
||||
require.Nil(t, val6)
|
||||
})
|
||||
|
||||
t.Run("StateDB", func(t *testing.T) {
|
||||
sdb, err := statedb.New(BlockHash, db)
|
||||
require.NoError(t, err)
|
||||
|
||||
checkAccountUnchanged := func() {
|
||||
require.Equal(t, Account.Balance, sdb.GetBalance(AccountAddress))
|
||||
require.Equal(t, Account.Nonce, sdb.GetNonce(AccountAddress))
|
||||
require.Equal(t, StoredValue, sdb.GetState(AccountAddress, StorageLeafKey))
|
||||
require.Equal(t, AccountCodeHash, sdb.GetCodeHash(AccountAddress))
|
||||
require.Equal(t, AccountCode, sdb.GetCode(AccountAddress))
|
||||
require.Equal(t, len(AccountCode), sdb.GetCodeSize(AccountAddress))
|
||||
}
|
||||
|
||||
require.True(t, sdb.Exist(AccountAddress))
|
||||
checkAccountUnchanged()
|
||||
|
||||
id := sdb.Snapshot()
|
||||
|
||||
newStorage := crypto.Keccak256Hash([]byte{5, 4, 3, 2, 1})
|
||||
newCode := []byte{1, 3, 3, 7}
|
||||
|
||||
sdb.SetBalance(AccountAddress, big.NewInt(300))
|
||||
sdb.AddBalance(AccountAddress, big.NewInt(200))
|
||||
sdb.SubBalance(AccountAddress, big.NewInt(100))
|
||||
sdb.SetNonce(AccountAddress, 42)
|
||||
sdb.SetState(AccountAddress, StorageLeafKey, newStorage)
|
||||
sdb.SetCode(AccountAddress, newCode)
|
||||
|
||||
require.Equal(t, big.NewInt(400), sdb.GetBalance(AccountAddress))
|
||||
require.Equal(t, uint64(42), sdb.GetNonce(AccountAddress))
|
||||
require.Equal(t, newStorage, sdb.GetState(AccountAddress, StorageLeafKey))
|
||||
require.Equal(t, newCode, sdb.GetCode(AccountAddress))
|
||||
|
||||
sdb.AddSlotToAccessList(AccountAddress, StorageLeafKey)
|
||||
require.True(t, sdb.AddressInAccessList(AccountAddress))
|
||||
hasAddr, hasSlot := sdb.SlotInAccessList(AccountAddress, StorageLeafKey)
|
||||
require.True(t, hasAddr)
|
||||
require.True(t, hasSlot)
|
||||
|
||||
sdb.RevertToSnapshot(id)
|
||||
|
||||
checkAccountUnchanged()
|
||||
require.False(t, sdb.AddressInAccessList(AccountAddress))
|
||||
hasAddr, hasSlot = sdb.SlotInAccessList(AccountAddress, StorageLeafKey)
|
||||
require.False(t, hasAddr)
|
||||
require.False(t, hasSlot)
|
||||
})
|
||||
testSuite(t, db)
|
||||
}
|
||||
|
||||
func TestSQLXSuite(t *testing.T) {
|
||||
testConfig, err := getTestConfig()
|
||||
testConfig, err := postgres.DefaultConfig.WithEnv()
|
||||
require.NoError(t, err)
|
||||
pool, err := statedb.NewSQLXPool(testCtx, testConfig)
|
||||
pool, err := postgres.ConnectSQLX(testCtx, testConfig)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
driver, err := statedb.NewSQLXDriverFromPool(context.Background(), pool)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
database := statedb.NewPostgresDB(driver)
|
||||
t.Cleanup(func() {
|
||||
tx, err := pool.Begin()
|
||||
require.NoError(t, err)
|
||||
statements := []string{
|
||||
`DELETE FROM eth.header_cids`,
|
||||
`DELETE FROM eth.state_cids`,
|
||||
`DELETE FROM eth.storage_cids`,
|
||||
`DELETE FROM ipld.blocks`,
|
||||
`TRUNCATE eth.header_cids`,
|
||||
`TRUNCATE eth.state_cids`,
|
||||
`TRUNCATE eth.storage_cids`,
|
||||
`TRUNCATE ipld.blocks`,
|
||||
}
|
||||
for _, stm := range statements {
|
||||
_, err = tx.Exec(stm)
|
||||
@ -334,6 +139,16 @@ func TestSQLXSuite(t *testing.T) {
|
||||
}
|
||||
require.NoError(t, tx.Commit())
|
||||
})
|
||||
|
||||
database := sql.NewSQLXDriverFromPool(context.Background(), pool)
|
||||
insertSuiteData(t, database)
|
||||
|
||||
db := state.NewStateDatabase(database)
|
||||
require.NoError(t, err)
|
||||
testSuite(t, db)
|
||||
}
|
||||
|
||||
func insertSuiteData(t *testing.T, database sql.Database) {
|
||||
require.NoError(t, insertHeaderCID(database, BlockHash.String(), BlockParentHash.String(), BlockNumber.Uint64()))
|
||||
require.NoError(t, insertHeaderCID(database, BlockHash2.String(), BlockHash.String(), BlockNumber2))
|
||||
require.NoError(t, insertHeaderCID(database, BlockHash3.String(), BlockHash2.String(), BlockNumber3))
|
||||
@ -415,10 +230,9 @@ func TestSQLXSuite(t *testing.T) {
|
||||
Removed: false,
|
||||
}))
|
||||
require.NoError(t, insertContractCode(database))
|
||||
}
|
||||
|
||||
db, err := statedb.NewStateDatabaseWithSqlxPool(pool)
|
||||
require.NoError(t, err)
|
||||
|
||||
func testSuite(t *testing.T, db state.StateDatabase) {
|
||||
t.Run("Database", func(t *testing.T) {
|
||||
size, err := db.ContractCodeSize(AccountCodeHash)
|
||||
require.NoError(t, err)
|
||||
@ -481,13 +295,13 @@ func TestSQLXSuite(t *testing.T) {
|
||||
})
|
||||
|
||||
t.Run("StateDB", func(t *testing.T) {
|
||||
sdb, err := statedb.New(BlockHash, db)
|
||||
sdb, err := state.New(BlockHash, db)
|
||||
require.NoError(t, err)
|
||||
|
||||
checkAccountUnchanged := func() {
|
||||
require.Equal(t, Account.Balance, sdb.GetBalance(AccountAddress))
|
||||
require.Equal(t, Account.Nonce, sdb.GetNonce(AccountAddress))
|
||||
require.Equal(t, StoredValue, sdb.GetState(AccountAddress, StorageLeafKey))
|
||||
require.Equal(t, StoredValue, sdb.GetState(AccountAddress, StorageSlot))
|
||||
require.Equal(t, AccountCodeHash, sdb.GetCodeHash(AccountAddress))
|
||||
require.Equal(t, AccountCode, sdb.GetCode(AccountAddress))
|
||||
require.Equal(t, len(AccountCode), sdb.GetCodeSize(AccountAddress))
|
||||
@ -505,17 +319,17 @@ func TestSQLXSuite(t *testing.T) {
|
||||
sdb.AddBalance(AccountAddress, big.NewInt(200))
|
||||
sdb.SubBalance(AccountAddress, big.NewInt(100))
|
||||
sdb.SetNonce(AccountAddress, 42)
|
||||
sdb.SetState(AccountAddress, StorageLeafKey, newStorage)
|
||||
sdb.SetState(AccountAddress, StorageSlot, newStorage)
|
||||
sdb.SetCode(AccountAddress, newCode)
|
||||
|
||||
require.Equal(t, big.NewInt(400), sdb.GetBalance(AccountAddress))
|
||||
require.Equal(t, uint64(42), sdb.GetNonce(AccountAddress))
|
||||
require.Equal(t, newStorage, sdb.GetState(AccountAddress, StorageLeafKey))
|
||||
require.Equal(t, newStorage, sdb.GetState(AccountAddress, StorageSlot))
|
||||
require.Equal(t, newCode, sdb.GetCode(AccountAddress))
|
||||
|
||||
sdb.AddSlotToAccessList(AccountAddress, StorageLeafKey)
|
||||
sdb.AddSlotToAccessList(AccountAddress, StorageSlot)
|
||||
require.True(t, sdb.AddressInAccessList(AccountAddress))
|
||||
hasAddr, hasSlot := sdb.SlotInAccessList(AccountAddress, StorageLeafKey)
|
||||
hasAddr, hasSlot := sdb.SlotInAccessList(AccountAddress, StorageSlot)
|
||||
require.True(t, hasAddr)
|
||||
require.True(t, hasSlot)
|
||||
|
||||
@ -523,13 +337,13 @@ func TestSQLXSuite(t *testing.T) {
|
||||
|
||||
checkAccountUnchanged()
|
||||
require.False(t, sdb.AddressInAccessList(AccountAddress))
|
||||
hasAddr, hasSlot = sdb.SlotInAccessList(AccountAddress, StorageLeafKey)
|
||||
hasAddr, hasSlot = sdb.SlotInAccessList(AccountAddress, StorageSlot)
|
||||
require.False(t, hasAddr)
|
||||
require.False(t, hasSlot)
|
||||
})
|
||||
}
|
||||
|
||||
func insertHeaderCID(db statedb.Database, blockHash, parentHash string, blockNumber uint64) error {
|
||||
func insertHeaderCID(db sql.Database, blockHash, parentHash string, blockNumber uint64) error {
|
||||
cid, err := util.Keccak256ToCid(ipld.MEthHeader, common.HexToHash(blockHash).Bytes())
|
||||
if err != nil {
|
||||
return err
|
||||
@ -580,7 +394,7 @@ type stateModel struct {
|
||||
Removed bool
|
||||
}
|
||||
|
||||
func insertStateCID(db statedb.Database, cidModel stateModel) error {
|
||||
func insertStateCID(db sql.Database, cidModel stateModel) error {
|
||||
sql := `INSERT INTO eth.state_cids (
|
||||
block_number,
|
||||
header_id,
|
||||
@ -619,7 +433,7 @@ type storageModel struct {
|
||||
Removed bool
|
||||
}
|
||||
|
||||
func insertStorageCID(db statedb.Database, cidModel storageModel) error {
|
||||
func insertStorageCID(db sql.Database, cidModel storageModel) error {
|
||||
sql := `INSERT INTO eth.storage_cids (
|
||||
block_number,
|
||||
header_id,
|
||||
@ -643,22 +457,8 @@ func insertStorageCID(db statedb.Database, cidModel storageModel) error {
|
||||
return err
|
||||
}
|
||||
|
||||
func insertContractCode(db statedb.Database) error {
|
||||
func insertContractCode(db sql.Database) error {
|
||||
sql := `INSERT INTO ipld.blocks (block_number, key, data) VALUES ($1, $2, $3)`
|
||||
_, err := db.Exec(testCtx, sql, BlockNumber.Uint64(), AccountCodeCID.String(), AccountCode)
|
||||
return err
|
||||
}
|
||||
|
||||
func getTestConfig() (conf statedb.Config, err error) {
|
||||
port, err := strconv.Atoi(os.Getenv("DATABASE_PORT"))
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
return statedb.Config{
|
||||
Hostname: os.Getenv("DATABASE_HOSTNAME"),
|
||||
DatabaseName: os.Getenv("DATABASE_NAME"),
|
||||
Username: os.Getenv("DATABASE_USER"),
|
||||
Password: os.Getenv("DATABASE_PASSWORD"),
|
||||
Port: port,
|
||||
}, nil
|
||||
}
|
||||
@ -14,7 +14,7 @@
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package ipld_eth_statedb
|
||||
package state
|
||||
|
||||
import (
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
@ -1,4 +1,4 @@
|
||||
package util
|
||||
package internal
|
||||
|
||||
import (
|
||||
"github.com/ipfs/go-cid"
|
||||
|
||||
@ -1,14 +1,11 @@
|
||||
package ipld_eth_statedb
|
||||
package sql
|
||||
|
||||
import (
|
||||
"context"
|
||||
)
|
||||
|
||||
// Database interfaces to support multiple Postgres drivers
|
||||
type Database interface {
|
||||
Driver
|
||||
Statements
|
||||
}
|
||||
type Database = Driver
|
||||
|
||||
// Driver interface has all the methods required by a driver implementation to support the sql indexer
|
||||
type Driver interface {
|
||||
@ -25,10 +22,3 @@ type ScannableRow interface {
|
||||
type Result interface {
|
||||
RowsAffected() (int64, error)
|
||||
}
|
||||
|
||||
// Statements interface to accommodate different SQL query syntax
|
||||
type Statements interface {
|
||||
GetContractCodeStmt() string
|
||||
GetStateAccountStmt() string
|
||||
GetStorageSlotStmt() string
|
||||
}
|
||||
@ -1,4 +1,4 @@
|
||||
package ipld_eth_statedb
|
||||
package sql
|
||||
|
||||
import (
|
||||
"context"
|
||||
@ -15,18 +15,9 @@ type PGXDriver struct {
|
||||
db *pgxpool.Pool
|
||||
}
|
||||
|
||||
// NewPGXDriver returns a new pgx driver for Postgres
|
||||
func NewPGXDriver(ctx context.Context, config Config) (*PGXDriver, error) {
|
||||
db, err := NewPGXPool(ctx, config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &PGXDriver{ctx: ctx, db: db}, nil
|
||||
}
|
||||
|
||||
// NewPGXDriverFromPool returns a new pgx driver for Postgres
|
||||
func NewPGXDriverFromPool(ctx context.Context, db *pgxpool.Pool) (*PGXDriver, error) {
|
||||
return &PGXDriver{ctx: ctx, db: db}, nil
|
||||
func NewPGXDriverFromPool(ctx context.Context, db *pgxpool.Pool) *PGXDriver {
|
||||
return &PGXDriver{ctx: ctx, db: db}
|
||||
}
|
||||
|
||||
// QueryRow satisfies sql.Database
|
||||
@ -1,4 +1,4 @@
|
||||
package ipld_eth_statedb
|
||||
package sql
|
||||
|
||||
import (
|
||||
"context"
|
||||
@ -14,25 +14,9 @@ type SQLXDriver struct {
|
||||
db *sqlx.DB
|
||||
}
|
||||
|
||||
// NewSQLXDriver returns a new sqlx driver for Postgres
|
||||
func NewSQLXDriver(ctx context.Context, config Config) (*SQLXDriver, error) {
|
||||
db, err := NewSQLXPool(ctx, config)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if config.MaxConns > 0 {
|
||||
db.SetMaxOpenConns(config.MaxConns)
|
||||
}
|
||||
if config.MaxConnLifetime > 0 {
|
||||
db.SetConnMaxLifetime(config.MaxConnLifetime)
|
||||
}
|
||||
db.SetMaxIdleConns(config.MaxIdle)
|
||||
return &SQLXDriver{ctx: ctx, db: db}, nil
|
||||
}
|
||||
|
||||
// NewSQLXDriverFromPool returns a new sqlx driver for Postgres
|
||||
func NewSQLXDriverFromPool(ctx context.Context, db *sqlx.DB) (*SQLXDriver, error) {
|
||||
return &SQLXDriver{ctx: ctx, db: db}, nil
|
||||
func NewSQLXDriverFromPool(ctx context.Context, db *sqlx.DB) *SQLXDriver {
|
||||
return &SQLXDriver{ctx: ctx, db: db}
|
||||
}
|
||||
|
||||
// QueryRow satisfies sql.Database
|
||||
@ -10,6 +10,7 @@ import (
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
|
||||
lru "github.com/hashicorp/golang-lru"
|
||||
|
||||
"github.com/cerc-io/ipld-eth-statedb/internal"
|
||||
"github.com/cerc-io/ipld-eth-statedb/trie_by_cid/trie"
|
||||
)
|
||||
|
||||
@ -42,6 +43,7 @@ type Database interface {
|
||||
// Trie is a Ethereum Merkle Patricia trie.
|
||||
type Trie interface {
|
||||
TryGet(key []byte) ([]byte, error)
|
||||
TryGetNode(path []byte) ([]byte, int, error)
|
||||
TryGetAccount(key []byte) (*types.StateAccount, error)
|
||||
Hash() common.Hash
|
||||
NodeIterator(startKey []byte) trie.NodeIterator
|
||||
@ -96,11 +98,10 @@ func (db *cachingDB) ContractCode(codeHash common.Hash) ([]byte, error) {
|
||||
if code := db.codeCache.Get(nil, codeHash.Bytes()); len(code) > 0 {
|
||||
return code, nil
|
||||
}
|
||||
// TODO - use non panicking
|
||||
codeCID := ipld.Keccak256ToCid(ipld.RawBinary, codeHash.Bytes())
|
||||
// if err != nil {
|
||||
// return nil, err
|
||||
// }
|
||||
codeCID, err := internal.Keccak256ToCid(ipld.RawBinary, codeHash.Bytes())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
code, err := db.db.DiskDB().Get(codeCID.Bytes())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
||||
@ -22,12 +22,13 @@ import (
|
||||
"github.com/VictoriaMetrics/fastcache"
|
||||
"github.com/ethereum/go-ethereum/ethdb"
|
||||
"github.com/ethereum/go-ethereum/trie"
|
||||
"github.com/ipfs/go-cid"
|
||||
)
|
||||
|
||||
type CidBytes = []byte
|
||||
type CidKey = cid.Cid
|
||||
|
||||
func isEmpty(key CidBytes) bool {
|
||||
return len(key) == 0
|
||||
func isEmpty(key CidKey) bool {
|
||||
return len(key.KeyString()) == 0
|
||||
}
|
||||
|
||||
// Database is an intermediate read-only layer between the trie data structures and
|
||||
@ -73,57 +74,30 @@ func (db *Database) DiskDB() ethdb.KeyValueStore {
|
||||
return db.diskdb
|
||||
}
|
||||
|
||||
// node retrieves a cached trie node from memory, or returns nil if none can be
|
||||
// found in the memory cache.
|
||||
func (db *Database) node(key CidBytes) (node, error) {
|
||||
// Retrieve the node from the clean cache if available
|
||||
if db.cleans != nil {
|
||||
if enc := db.cleans.Get(nil, key); enc != nil {
|
||||
// The returned value from cache is in its own copy,
|
||||
// safe to use mustDecodeNodeUnsafe for decoding.
|
||||
return decodeNodeUnsafe(key, enc)
|
||||
}
|
||||
}
|
||||
|
||||
// Content unavailable in memory, attempt to retrieve from disk
|
||||
enc, err := db.diskdb.Get(key)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if enc == nil {
|
||||
return nil, nil
|
||||
}
|
||||
if db.cleans != nil {
|
||||
db.cleans.Set(key, enc)
|
||||
}
|
||||
// The returned value from database is in its own copy,
|
||||
// safe to use mustDecodeNodeUnsafe for decoding.
|
||||
return decodeNodeUnsafe(key, enc)
|
||||
}
|
||||
|
||||
// Node retrieves an encoded cached trie node from memory. If it cannot be found
|
||||
// cached, the method queries the persistent database for the content.
|
||||
func (db *Database) Node(key CidBytes) ([]byte, error) {
|
||||
// Node retrieves an encoded trie node by CID. If it cannot be found
|
||||
// cached in memory, it queries the persistent database.
|
||||
func (db *Database) Node(key CidKey) ([]byte, error) {
|
||||
// It doesn't make sense to retrieve the metaroot
|
||||
if isEmpty(key) {
|
||||
return nil, errors.New("not found")
|
||||
}
|
||||
cidbytes := key.Bytes()
|
||||
// Retrieve the node from the clean cache if available
|
||||
if db.cleans != nil {
|
||||
if enc := db.cleans.Get(nil, key); enc != nil {
|
||||
if enc := db.cleans.Get(nil, cidbytes); enc != nil {
|
||||
return enc, nil
|
||||
}
|
||||
}
|
||||
|
||||
// Content unavailable in memory, attempt to retrieve from disk
|
||||
enc, err := db.diskdb.Get(key)
|
||||
enc, err := db.diskdb.Get(cidbytes)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if len(enc) != 0 {
|
||||
if db.cleans != nil {
|
||||
db.cleans.Set(key[:], enc)
|
||||
db.cleans.Set(cidbytes, enc)
|
||||
}
|
||||
return enc, nil
|
||||
}
|
||||
|
||||
@ -20,13 +20,15 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/ethereum/go-ethereum/ethdb/memorydb"
|
||||
|
||||
"github.com/cerc-io/ipld-eth-statedb/trie_by_cid/trie"
|
||||
)
|
||||
|
||||
// Tests that the trie database returns a missing trie node error if attempting
|
||||
// to retrieve the meta root.
|
||||
func TestDatabaseMetarootFetch(t *testing.T) {
|
||||
db := NewDatabase(memorydb.New())
|
||||
if _, err := db.Node(CidBytes(nil)); err == nil {
|
||||
db := trie.NewDatabase(memorydb.New())
|
||||
if _, err := db.Node(trie.CidKey{}); err == nil {
|
||||
t.Fatalf("metaroot retrieval succeeded")
|
||||
}
|
||||
}
|
||||
|
||||
@ -16,6 +16,21 @@
|
||||
|
||||
package trie
|
||||
|
||||
// CompactToHex converts a compact encoded path to hex format
|
||||
func CompactToHex(compact []byte) []byte {
|
||||
if len(compact) == 0 {
|
||||
return compact
|
||||
}
|
||||
base := keybytesToHex(compact)
|
||||
// delete terminator flag
|
||||
if base[0] < 2 {
|
||||
base = base[:len(base)-1]
|
||||
}
|
||||
// apply odd flag
|
||||
chop := 2 - base[0]&1
|
||||
return base[chop:]
|
||||
}
|
||||
|
||||
func keybytesToHex(str []byte) []byte {
|
||||
l := len(str)*2 + 1
|
||||
var nibbles = make([]byte, l)
|
||||
|
||||
@ -21,39 +21,40 @@ import (
|
||||
"context"
|
||||
"fmt"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/core/rawdb"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
|
||||
geth_trie "github.com/ethereum/go-ethereum/trie"
|
||||
|
||||
pgipfsethdb "github.com/cerc-io/ipfs-ethdb/v5/postgres/v0"
|
||||
"github.com/cerc-io/ipld-eth-statedb/trie_by_cid/trie"
|
||||
)
|
||||
|
||||
var (
|
||||
cacheConfig = pgipfsethdb.CacheConfig{
|
||||
Name: "db",
|
||||
Size: 3000000, // 3MB
|
||||
ExpiryDuration: time.Hour,
|
||||
}
|
||||
dbConfig, _ = postgres.DefaultConfig.WithEnv()
|
||||
trieConfig = trie.Config{Cache: 256}
|
||||
ctx = context.Background()
|
||||
|
||||
ctx = context.Background()
|
||||
testdata0 = []kvs{
|
||||
{"one", 1},
|
||||
{"two", 2},
|
||||
{"three", 3},
|
||||
{"four", 4},
|
||||
{"five", 5},
|
||||
{"ten", 10},
|
||||
}
|
||||
testdata1 = []kvs{
|
||||
{"barb", 0},
|
||||
{"bard", 1},
|
||||
{"bars", 2},
|
||||
{"bar", 3},
|
||||
{"fab", 4},
|
||||
{"food", 5},
|
||||
{"foos", 6},
|
||||
{"foo", 7},
|
||||
}
|
||||
)
|
||||
|
||||
var testdata1 = []kvs{
|
||||
{"barb", 0},
|
||||
{"bard", 1},
|
||||
{"bars", 2},
|
||||
{"bar", 3},
|
||||
{"fab", 4},
|
||||
{"food", 5},
|
||||
{"foos", 6},
|
||||
{"foo", 7},
|
||||
}
|
||||
|
||||
func TestEmptyIterator(t *testing.T) {
|
||||
trie := trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase()))
|
||||
iter := trie.NodeIterator(nil)
|
||||
@ -71,15 +72,7 @@ func TestIterator(t *testing.T) {
|
||||
edb := rawdb.NewMemoryDatabase()
|
||||
db := geth_trie.NewDatabase(edb)
|
||||
origtrie := geth_trie.NewEmpty(db)
|
||||
vals := []kvs{
|
||||
{"one", 1},
|
||||
{"two", 2},
|
||||
{"three", 3},
|
||||
{"four", 4},
|
||||
{"five", 5},
|
||||
{"ten", 10},
|
||||
}
|
||||
all, err := updateTrie(origtrie, vals)
|
||||
all, err := updateTrie(origtrie, testdata0)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -103,22 +96,6 @@ func TestIterator(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func checkIteratorOrder(want []kvs, it *trie.Iterator) error {
|
||||
for it.Next() {
|
||||
if len(want) == 0 {
|
||||
return fmt.Errorf("didn't expect any more values, got key %q", it.Key)
|
||||
}
|
||||
if !bytes.Equal(it.Key, []byte(want[0].k)) {
|
||||
return fmt.Errorf("wrong key: got %q, want %q", it.Key, want[0].k)
|
||||
}
|
||||
want = want[1:]
|
||||
}
|
||||
if len(want) > 0 {
|
||||
return fmt.Errorf("iterator ended early, want key %q", want[0])
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestIteratorSeek(t *testing.T) {
|
||||
edb := rawdb.NewMemoryDatabase()
|
||||
db := geth_trie.NewDatabase(edb)
|
||||
@ -148,11 +125,56 @@ func TestIteratorSeek(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
// returns a cache config with unique name (groupcache names are global)
|
||||
func makeCacheConfig(t testing.TB) pgipfsethdb.CacheConfig {
|
||||
return pgipfsethdb.CacheConfig{
|
||||
Name: t.Name(),
|
||||
Size: 3000000, // 3MB
|
||||
ExpiryDuration: time.Hour,
|
||||
func checkIteratorOrder(want []kvs, it *trie.Iterator) error {
|
||||
for it.Next() {
|
||||
if len(want) == 0 {
|
||||
return fmt.Errorf("didn't expect any more values, got key %q", it.Key)
|
||||
}
|
||||
if !bytes.Equal(it.Key, []byte(want[0].k)) {
|
||||
return fmt.Errorf("wrong key: got %q, want %q", it.Key, want[0].k)
|
||||
}
|
||||
want = want[1:]
|
||||
}
|
||||
if len(want) > 0 {
|
||||
return fmt.Errorf("iterator ended early, want key %q", want[0])
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func TestIteratorNodeBlob(t *testing.T) {
|
||||
edb := rawdb.NewMemoryDatabase()
|
||||
db := geth_trie.NewDatabase(edb)
|
||||
orig := geth_trie.NewEmpty(geth_trie.NewDatabase(rawdb.NewMemoryDatabase()))
|
||||
if _, err := updateTrie(orig, testdata1); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
root := commitTrie(t, db, orig)
|
||||
trie := indexTrie(t, edb, root)
|
||||
|
||||
found := make(map[common.Hash][]byte)
|
||||
it := trie.NodeIterator(nil)
|
||||
for it.Next(true) {
|
||||
if it.Hash() == (common.Hash{}) {
|
||||
continue
|
||||
}
|
||||
found[it.Hash()] = it.NodeBlob()
|
||||
}
|
||||
|
||||
dbIter := edb.NewIterator(nil, nil)
|
||||
defer dbIter.Release()
|
||||
|
||||
var count int
|
||||
for dbIter.Next() {
|
||||
got, present := found[common.BytesToHash(dbIter.Key())]
|
||||
if !present {
|
||||
t.Fatalf("Missing trie node %v", dbIter.Key())
|
||||
}
|
||||
if !bytes.Equal(got, dbIter.Value()) {
|
||||
t.Fatalf("Unexpected trie node want %v got %v", dbIter.Value(), got)
|
||||
}
|
||||
count += 1
|
||||
}
|
||||
if count != len(found) {
|
||||
t.Fatal("Find extra trie node via iterator")
|
||||
}
|
||||
}
|
||||
|
||||
@ -95,6 +95,14 @@ func (t *StateTrie) TryGetAccount(key []byte) (*types.StateAccount, error) {
|
||||
return &ret, err
|
||||
}
|
||||
|
||||
// TryGetNode attempts to retrieve a trie node by compact-encoded path. It is not
|
||||
// possible to use keybyte-encoding as the path might contain odd nibbles.
|
||||
// If the specified trie node is not in the trie, nil will be returned.
|
||||
// If a trie node is not found in the database, a MissingNodeError is returned.
|
||||
func (t *StateTrie) TryGetNode(path []byte) ([]byte, int, error) {
|
||||
return t.trie.TryGetNode(path)
|
||||
}
|
||||
|
||||
// Hash returns the root hash of StateTrie. It does not write to the
|
||||
// database and can be used even if the trie doesn't have one.
|
||||
func (t *StateTrie) Hash() common.Hash {
|
||||
|
||||
@ -3,11 +3,13 @@ package trie
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"errors"
|
||||
"fmt"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/crypto"
|
||||
|
||||
util "github.com/cerc-io/ipld-eth-statedb/internal"
|
||||
"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
|
||||
)
|
||||
|
||||
@ -125,11 +127,99 @@ func (t *Trie) tryGet(origNode node, key []byte, pos int) (value []byte, newnode
|
||||
}
|
||||
}
|
||||
|
||||
// TryGetNode attempts to retrieve a trie node by compact-encoded path. It is not
|
||||
// possible to use keybyte-encoding as the path might contain odd nibbles.
|
||||
func (t *Trie) TryGetNode(path []byte) ([]byte, int, error) {
|
||||
item, newroot, resolved, err := t.tryGetNode(t.root, CompactToHex(path), 0)
|
||||
if err != nil {
|
||||
return nil, resolved, err
|
||||
}
|
||||
if resolved > 0 {
|
||||
t.root = newroot
|
||||
}
|
||||
if item == nil {
|
||||
return nil, resolved, nil
|
||||
}
|
||||
return item, resolved, err
|
||||
}
|
||||
|
||||
func (t *Trie) tryGetNode(origNode node, path []byte, pos int) (item []byte, newnode node, resolved int, err error) {
|
||||
// If non-existent path requested, abort
|
||||
if origNode == nil {
|
||||
return nil, nil, 0, nil
|
||||
}
|
||||
// If we reached the requested path, return the current node
|
||||
if pos >= len(path) {
|
||||
// Although we most probably have the original node expanded, encoding
|
||||
// that into consensus form can be nasty (needs to cascade down) and
|
||||
// time consuming. Instead, just pull the hash up from disk directly.
|
||||
var hash hashNode
|
||||
if node, ok := origNode.(hashNode); ok {
|
||||
hash = node
|
||||
} else {
|
||||
hash, _ = origNode.cache()
|
||||
}
|
||||
if hash == nil {
|
||||
return nil, origNode, 0, errors.New("non-consensus node")
|
||||
}
|
||||
cid, err := util.Keccak256ToCid(t.codec, hash)
|
||||
if err != nil {
|
||||
return nil, origNode, 0, err
|
||||
}
|
||||
blob, err := t.db.Node(cid)
|
||||
return blob, origNode, 1, err
|
||||
}
|
||||
// Path still needs to be traversed, descend into children
|
||||
switch n := (origNode).(type) {
|
||||
case valueNode:
|
||||
// Path prematurely ended, abort
|
||||
return nil, nil, 0, nil
|
||||
|
||||
case *shortNode:
|
||||
if len(path)-pos < len(n.Key) || !bytes.Equal(n.Key, path[pos:pos+len(n.Key)]) {
|
||||
// Path branches off from short node
|
||||
return nil, n, 0, nil
|
||||
}
|
||||
item, newnode, resolved, err = t.tryGetNode(n.Val, path, pos+len(n.Key))
|
||||
if err == nil && resolved > 0 {
|
||||
n = n.copy()
|
||||
n.Val = newnode
|
||||
}
|
||||
return item, n, resolved, err
|
||||
|
||||
case *fullNode:
|
||||
item, newnode, resolved, err = t.tryGetNode(n.Children[path[pos]], path, pos+1)
|
||||
if err == nil && resolved > 0 {
|
||||
n = n.copy()
|
||||
n.Children[path[pos]] = newnode
|
||||
}
|
||||
return item, n, resolved, err
|
||||
|
||||
case hashNode:
|
||||
child, err := t.resolveHash(n, path[:pos])
|
||||
if err != nil {
|
||||
return nil, n, 1, err
|
||||
}
|
||||
item, newnode, resolved, err := t.tryGetNode(child, path, pos)
|
||||
return item, newnode, resolved + 1, err
|
||||
|
||||
default:
|
||||
panic(fmt.Sprintf("%T: invalid node: %v", origNode, origNode))
|
||||
}
|
||||
}
|
||||
|
||||
// resolveHash loads node from the underlying database with the provided
|
||||
// node hash and path prefix.
|
||||
func (t *Trie) resolveHash(n hashNode, prefix []byte) (node, error) {
|
||||
cid := ipld.Keccak256ToCid(t.codec, n)
|
||||
node, err := t.db.node(cid.Bytes())
|
||||
cid, err := util.Keccak256ToCid(t.codec, n)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
enc, err := t.db.Node(cid)
|
||||
if err != nil {
|
||||
return nil, &MissingNodeError{Owner: t.owner, NodeHash: n, Path: prefix, err: err}
|
||||
}
|
||||
node, err := decodeNodeUnsafe(n, enc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
@ -142,8 +232,14 @@ func (t *Trie) resolveHash(n hashNode, prefix []byte) (node, error) {
|
||||
// resolveHash loads rlp-encoded node blob from the underlying database
|
||||
// with the provided node hash and path prefix.
|
||||
func (t *Trie) resolveBlob(n hashNode, prefix []byte) ([]byte, error) {
|
||||
cid := ipld.Keccak256ToCid(t.codec, n)
|
||||
blob, _ := t.db.Node(cid.Bytes())
|
||||
cid, err := util.Keccak256ToCid(t.codec, n)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
blob, err := t.db.Node(cid)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(blob) != 0 {
|
||||
return blob, nil
|
||||
}
|
||||
@ -153,20 +249,20 @@ func (t *Trie) resolveBlob(n hashNode, prefix []byte) ([]byte, error) {
|
||||
// Hash returns the root hash of the trie. It does not write to the
|
||||
// database and can be used even if the trie doesn't have one.
|
||||
func (t *Trie) Hash() common.Hash {
|
||||
hash, cached, _ := t.hashRoot()
|
||||
hash, cached := t.hashRoot()
|
||||
t.root = cached
|
||||
return common.BytesToHash(hash.(hashNode))
|
||||
}
|
||||
|
||||
// hashRoot calculates the root hash of the given trie
|
||||
func (t *Trie) hashRoot() (node, node, error) {
|
||||
func (t *Trie) hashRoot() (node, node) {
|
||||
if t.root == nil {
|
||||
return hashNode(emptyRoot.Bytes()), nil, nil
|
||||
return hashNode(emptyRoot.Bytes()), nil
|
||||
}
|
||||
// If the number of changes is below 100, we let one thread handle it
|
||||
h := newHasher(t.unhashed >= 100)
|
||||
defer returnHasherToPool(h)
|
||||
hashed, cached := h.hash(t.root, true)
|
||||
t.unhashed = 0
|
||||
return hashed, cached, nil
|
||||
return hashed, cached
|
||||
}
|
||||
|
||||
173
trie_by_cid/trie/trie_test.go
Normal file
173
trie_by_cid/trie/trie_test.go
Normal file
@ -0,0 +1,173 @@
|
||||
// Copyright 2014 The go-ethereum Authors
|
||||
// This file is part of the go-ethereum library.
|
||||
//
|
||||
// The go-ethereum library is free software: you can redistribute it and/or modify
|
||||
// it under the terms of the GNU Lesser General Public License as published by
|
||||
// the Free Software Foundation, either version 3 of the License, or
|
||||
// (at your option) any later version.
|
||||
//
|
||||
// The go-ethereum library is distributed in the hope that it will be useful,
|
||||
// but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
// GNU Lesser General Public License for more details.
|
||||
//
|
||||
// You should have received a copy of the GNU Lesser General Public License
|
||||
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
package trie_test
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"math/big"
|
||||
"math/rand"
|
||||
"testing"
|
||||
|
||||
"github.com/ethereum/go-ethereum/common"
|
||||
"github.com/ethereum/go-ethereum/core/rawdb"
|
||||
"github.com/ethereum/go-ethereum/core/types"
|
||||
"github.com/ethereum/go-ethereum/crypto"
|
||||
"github.com/ethereum/go-ethereum/rlp"
|
||||
geth_trie "github.com/ethereum/go-ethereum/trie"
|
||||
|
||||
"github.com/cerc-io/ipld-eth-statedb/trie_by_cid/trie"
|
||||
)
|
||||
|
||||
func TestTrieEmpty(t *testing.T) {
|
||||
trie := trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase()))
|
||||
res := trie.Hash()
|
||||
exp := types.EmptyRootHash
|
||||
if res != exp {
|
||||
t.Errorf("expected %x got %x", exp, res)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrieMissingRoot(t *testing.T) {
|
||||
root := common.HexToHash("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33")
|
||||
tr, err := newStateTrie(root, trie.NewDatabase(rawdb.NewMemoryDatabase()))
|
||||
if tr != nil {
|
||||
t.Error("New returned non-nil trie for invalid root")
|
||||
}
|
||||
if _, ok := err.(*trie.MissingNodeError); !ok {
|
||||
t.Errorf("New returned wrong error: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrieBasic(t *testing.T) {
|
||||
edb := rawdb.NewMemoryDatabase()
|
||||
db := geth_trie.NewDatabase(edb)
|
||||
origtrie := geth_trie.NewEmpty(db)
|
||||
origtrie.Update([]byte("foo"), packValue(842))
|
||||
expected := commitTrie(t, db, origtrie)
|
||||
tr := indexTrie(t, edb, expected)
|
||||
got := tr.Hash()
|
||||
if expected != got {
|
||||
t.Errorf("got %x expected %x", got, expected)
|
||||
}
|
||||
checkValue(t, tr, []byte("foo"))
|
||||
}
|
||||
|
||||
func TestTrieTiny(t *testing.T) {
|
||||
// Create a realistic account trie to hash
|
||||
_, accounts := makeAccounts(5)
|
||||
edb := rawdb.NewMemoryDatabase()
|
||||
db := geth_trie.NewDatabase(edb)
|
||||
origtrie := geth_trie.NewEmpty(db)
|
||||
|
||||
type testCase struct {
|
||||
key, account []byte
|
||||
root common.Hash
|
||||
}
|
||||
cases := []testCase{
|
||||
{
|
||||
common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000001337"),
|
||||
accounts[3],
|
||||
common.HexToHash("8c6a85a4d9fda98feff88450299e574e5378e32391f75a055d470ac0653f1005"),
|
||||
}, {
|
||||
common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000001338"),
|
||||
accounts[4],
|
||||
common.HexToHash("ec63b967e98a5720e7f720482151963982890d82c9093c0d486b7eb8883a66b1"),
|
||||
}, {
|
||||
common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000001339"),
|
||||
accounts[4],
|
||||
common.HexToHash("0608c1d1dc3905fa22204c7a0e43644831c3b6d3def0f274be623a948197e64a"),
|
||||
},
|
||||
}
|
||||
for i, tc := range cases {
|
||||
t.Run(fmt.Sprintf("case %d", i), func(t *testing.T) {
|
||||
origtrie.Update(tc.key, tc.account)
|
||||
trie := indexTrie(t, edb, commitTrie(t, db, origtrie))
|
||||
if exp, root := tc.root, trie.Hash(); exp != root {
|
||||
t.Errorf("got %x, exp %x", root, exp)
|
||||
}
|
||||
checkValue(t, trie, tc.key)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestTrieMedium(t *testing.T) {
|
||||
// Create a realistic account trie to hash
|
||||
addresses, accounts := makeAccounts(1000)
|
||||
edb := rawdb.NewMemoryDatabase()
|
||||
db := geth_trie.NewDatabase(edb)
|
||||
origtrie := geth_trie.NewEmpty(db)
|
||||
var keys [][]byte
|
||||
for i := 0; i < len(addresses); i++ {
|
||||
key := crypto.Keccak256(addresses[i][:])
|
||||
if i%50 == 0 {
|
||||
keys = append(keys, key)
|
||||
}
|
||||
origtrie.Update(key, accounts[i])
|
||||
}
|
||||
tr := indexTrie(t, edb, commitTrie(t, db, origtrie))
|
||||
|
||||
root := tr.Hash()
|
||||
exp := common.HexToHash("72f9d3f3fe1e1dd7b8936442e7642aef76371472d94319900790053c493f3fe6")
|
||||
if exp != root {
|
||||
t.Errorf("got %x, exp %x", root, exp)
|
||||
}
|
||||
|
||||
for _, key := range keys {
|
||||
checkValue(t, tr, key)
|
||||
}
|
||||
}
|
||||
|
||||
// Make deterministically random accounts
|
||||
func makeAccounts(size int) (addresses [][20]byte, accounts [][]byte) {
|
||||
random := rand.New(rand.NewSource(0))
|
||||
addresses = make([][20]byte, size)
|
||||
for i := 0; i < len(addresses); i++ {
|
||||
data := make([]byte, 20)
|
||||
random.Read(data)
|
||||
copy(addresses[i][:], data)
|
||||
}
|
||||
accounts = make([][]byte, len(addresses))
|
||||
for i := 0; i < len(accounts); i++ {
|
||||
var (
|
||||
nonce = uint64(random.Int63())
|
||||
root = types.EmptyRootHash
|
||||
code = crypto.Keccak256(nil)
|
||||
)
|
||||
// The big.Rand function is not deterministic with regards to 64 vs 32 bit systems,
|
||||
// and will consume different amount of data from the rand source.
|
||||
// balance = new(big.Int).Rand(random, new(big.Int).Exp(common.Big2, common.Big256, nil))
|
||||
// Therefore, we instead just read via byte buffer
|
||||
numBytes := random.Uint32() % 33 // [0, 32] bytes
|
||||
balanceBytes := make([]byte, numBytes)
|
||||
random.Read(balanceBytes)
|
||||
balance := new(big.Int).SetBytes(balanceBytes)
|
||||
acct := &types.StateAccount{Nonce: nonce, Balance: balance, Root: root, CodeHash: code}
|
||||
data, _ := rlp.EncodeToBytes(acct)
|
||||
accounts[i] = data
|
||||
}
|
||||
return addresses, accounts
|
||||
}
|
||||
|
||||
func checkValue(t *testing.T, tr *trie.Trie, key []byte) {
|
||||
val, err := tr.TryGet(key)
|
||||
if err != nil {
|
||||
t.Fatalf("error getting node: %s", err)
|
||||
}
|
||||
if len(val) == 0 {
|
||||
t.Errorf("failed to get value for %x", key)
|
||||
}
|
||||
}
|
||||
@ -5,6 +5,7 @@ import (
|
||||
"math/big"
|
||||
"math/rand"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/jmoiron/sqlx"
|
||||
|
||||
@ -100,13 +101,21 @@ func indexTrie(t testing.TB, edb ethdb.Database, root common.Hash) *trie.Trie {
|
||||
|
||||
ipfs_db := pgipfsethdb.NewDatabase(pg_db, makeCacheConfig(t))
|
||||
sdb_db := state.NewDatabase(ipfs_db)
|
||||
tr, err := trie.New(common.Hash{}, root, sdb_db.TrieDB(), ipld.MEthStateTrie)
|
||||
tr, err := newStateTrie(root, sdb_db.TrieDB())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return tr
|
||||
}
|
||||
|
||||
func newStateTrie(root common.Hash, db *trie.Database) (*trie.Trie, error) {
|
||||
tr, err := trie.New(common.Hash{}, root, db, ipld.MEthStateTrie)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return tr, nil
|
||||
}
|
||||
|
||||
// generates a random Geth LevelDB trie of n key-value pairs and corresponding value map
|
||||
func randomGethTrie(n int, db *geth_trie.Database) (*geth_trie.Trie, kvMap) {
|
||||
trie := geth_trie.NewEmpty(db)
|
||||
@ -170,3 +179,12 @@ func TearDownDB(db *sqlx.DB) error {
|
||||
}
|
||||
return tx.Commit()
|
||||
}
|
||||
|
||||
// returns a cache config with unique name (groupcache names are global)
|
||||
func makeCacheConfig(t testing.TB) pgipfsethdb.CacheConfig {
|
||||
return pgipfsethdb.CacheConfig{
|
||||
Name: t.Name(),
|
||||
Size: 3000000, // 3MB
|
||||
ExpiryDuration: time.Hour,
|
||||
}
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user