Merge pull request #18 from cerc-io/roy/v5-dev

* Refactor redundant DB code

* Fix storage paths

GetState takes a slot, not a leaf path

* Add trie_by_cid/state.TryGetNode

used in ipld-eth-server

* Reorganize packages

* direct_by_leaf/ is original StateDB package
* sql/ for SQL DB interfaces

* Test NodeBlob

* Fix node resolution; use Cid struct as key

was erroneously storing the cid in the fullNode's flag.cache

* Add basic trie tests
This commit is contained in:
Roy Crihfield 2023-04-25 15:01:12 +08:00 committed by GitHub
commit 0eba6888c1
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 484 additions and 538 deletions

View File

@ -1,90 +0,0 @@
package ipld_eth_statedb
import (
"context"
"fmt"
"time"
"github.com/jackc/pgx/v4/pgxpool"
"github.com/jmoiron/sqlx"
_ "github.com/lib/pq"
)
type Config struct {
Hostname string
Port int
DatabaseName string
Username string
Password string
ConnTimeout time.Duration
MaxConns int
MinConns int
MaxConnLifetime time.Duration
MaxConnIdleTime time.Duration
MaxIdle int
}
// DbConnectionString constructs and returns the connection string from the config (for sqlx driver)
func (c Config) DbConnectionString() string {
if len(c.Username) > 0 && len(c.Password) > 0 {
return fmt.Sprintf("postgresql://%s:%s@%s:%d/%s?sslmode=disable",
c.Username, c.Password, c.Hostname, c.Port, c.DatabaseName)
}
if len(c.Username) > 0 && len(c.Password) == 0 {
return fmt.Sprintf("postgresql://%s@%s:%d/%s?sslmode=disable",
c.Username, c.Hostname, c.Port, c.DatabaseName)
}
return fmt.Sprintf("postgresql://%s:%d/%s?sslmode=disable", c.Hostname, c.Port, c.DatabaseName)
}
// NewPGXPool returns a new pgx conn pool
func NewPGXPool(ctx context.Context, config Config) (*pgxpool.Pool, error) {
pgConf, err := makePGXConfig(config)
if err != nil {
return nil, err
}
return pgxpool.ConnectConfig(ctx, pgConf)
}
// NewSQLXPool returns a new sqlx conn pool
func NewSQLXPool(ctx context.Context, config Config) (*sqlx.DB, error) {
db, err := sqlx.ConnectContext(ctx, "postgres", config.DbConnectionString())
if err != nil {
return nil, err
}
return db, nil
}
// makePGXConfig creates a pgxpool.Config from the provided Config
func makePGXConfig(config Config) (*pgxpool.Config, error) {
conf, err := pgxpool.ParseConfig("")
if err != nil {
return nil, err
}
conf.ConnConfig.Config.Host = config.Hostname
conf.ConnConfig.Config.Port = uint16(config.Port)
conf.ConnConfig.Config.Database = config.DatabaseName
conf.ConnConfig.Config.User = config.Username
conf.ConnConfig.Config.Password = config.Password
if config.ConnTimeout != 0 {
conf.ConnConfig.Config.ConnectTimeout = config.ConnTimeout
}
if config.MaxConns != 0 {
conf.MaxConns = int32(config.MaxConns)
}
if config.MinConns != 0 {
conf.MinConns = int32(config.MinConns)
}
if config.MaxConnLifetime != 0 {
conf.MaxConnLifetime = config.MaxConnLifetime
}
if config.MaxConnIdleTime != 0 {
conf.MaxConnIdleTime = config.MaxConnIdleTime
}
return conf, nil
}

View File

@ -1,28 +0,0 @@
package ipld_eth_statedb
var _ Database = &DB{}
// NewPostgresDB returns a postgres.DB using the provided driver
func NewPostgresDB(driver Driver) *DB {
return &DB{driver}
}
// DB implements sql.Database using a configured driver and Postgres statement syntax
type DB struct {
Driver
}
// GetContractCodeStmt satisfies the Statements interface
func (db *DB) GetContractCodeStmt() string {
return GetContractCodePgStr
}
// GetStateAccountStmt satisfies the Statements interface
func (db *DB) GetStateAccountStmt() string {
return GetStateAccount
}
// GetStorageSlotStmt satisfies the Statements interface
func (db *DB) GetStorageSlotStmt() string {
return GetStorageSlot
}

View File

@ -1,4 +1,4 @@
package ipld_eth_statedb
package state
import (
"github.com/ethereum/go-ethereum/common"

View File

@ -1,4 +1,4 @@
package ipld_eth_statedb
package state
import (
"math/big"

View File

@ -1,4 +1,4 @@
package ipld_eth_statedb
package state
const (
GetContractCodePgStr = `SELECT data FROM ipld.blocks WHERE key = $1`

View File

@ -1,4 +1,4 @@
package ipld_eth_statedb
package state
import (
"context"
@ -8,14 +8,13 @@ import (
"github.com/VictoriaMetrics/fastcache"
lru "github.com/hashicorp/golang-lru"
"github.com/jackc/pgx/v4/pgxpool"
"github.com/jmoiron/sqlx"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
util "github.com/cerc-io/ipld-eth-statedb/internal"
"github.com/cerc-io/ipld-eth-statedb/sql"
)
const (
@ -43,29 +42,19 @@ type StateDatabase interface {
var _ StateDatabase = &stateDatabase{}
type stateDatabase struct {
db Database
db sql.Database
codeSizeCache *lru.Cache
codeCache *fastcache.Cache
}
// NewStateDatabaseWithPgxPool returns a new Database implementation using the provided postgres connection pool
func NewStateDatabaseWithPgxPool(pgDb *pgxpool.Pool) (*stateDatabase, error) {
// NewStateDatabase returns a new Database implementation using the passed parameters
func NewStateDatabase(db sql.Database) *stateDatabase {
csc, _ := lru.New(codeSizeCacheSize)
return &stateDatabase{
db: NewPostgresDB(&PGXDriver{db: pgDb}),
db: db,
codeSizeCache: csc,
codeCache: fastcache.New(codeCacheSize),
}, nil
}
// NewStateDatabaseWithSqlxPool returns a new Database implementation using the passed parameters
func NewStateDatabaseWithSqlxPool(db *sqlx.DB) (*stateDatabase, error) {
csc, _ := lru.New(codeSizeCacheSize)
return &stateDatabase{
db: NewPostgresDB(&SQLXDriver{db: db}),
codeSizeCache: csc,
codeCache: fastcache.New(codeCacheSize),
}, nil
}
}
// ContractCode satisfies Database, it returns the contract code for a given codehash

View File

@ -1,4 +1,4 @@
package ipld_eth_statedb
package state
import (
"bytes"
@ -159,7 +159,8 @@ func (s *stateObject) GetCommittedState(db StateDatabase, key common.Hash) commo
}
// If no live objects are available, load from database
start := time.Now()
enc, err := db.StorageValue(s.addrHash, key, s.blockHash)
keyHash := crypto.Keccak256Hash(key[:])
enc, err := db.StorageValue(s.addrHash, keyHash, s.blockHash)
if metrics.EnabledExpensive {
s.db.StorageReads += time.Since(start)
}

View File

@ -1,4 +1,4 @@
package ipld_eth_statedb
package state
import (
"fmt"

View File

@ -1,10 +1,8 @@
package ipld_eth_statedb_test
package state_test
import (
"context"
"math/big"
"os"
"strconv"
"testing"
"github.com/lib/pq"
@ -15,10 +13,12 @@ import (
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/rlp"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
statedb "github.com/cerc-io/ipld-eth-statedb"
state "github.com/cerc-io/ipld-eth-statedb/direct_by_leaf"
util "github.com/cerc-io/ipld-eth-statedb/internal"
"github.com/cerc-io/ipld-eth-statedb/sql"
)
var (
@ -53,7 +53,7 @@ var (
AccountPK, _ = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a")
AccountAddress = crypto.PubkeyToAddress(AccountPK.PublicKey) //0x703c4b2bD70c169f5717101CaeE543299Fc946C7
AccountLeafKey = crypto.Keccak256Hash(AccountAddress.Bytes())
AccountLeafKey = crypto.Keccak256Hash(AccountAddress[:])
AccountCode = []byte{0, 1, 2, 3, 4, 5, 6, 7}
AccountCodeHash = crypto.Keccak256Hash(AccountCode)
@ -65,7 +65,8 @@ var (
Root: common.Hash{},
}
StorageLeafKey = crypto.Keccak256Hash(common.HexToHash("0").Bytes())
StorageSlot = common.HexToHash("0")
StorageLeafKey = crypto.Keccak256Hash(StorageSlot[:])
StoredValue = crypto.Keccak256Hash([]byte{1, 2, 3, 4, 5})
StoragePartialPath = []byte{0, 1, 0, 2, 0, 4}
@ -73,7 +74,7 @@ var (
accountRLP, _ = rlp.EncodeToBytes(&Account)
accountAndLeafRLP, _ = rlp.EncodeToBytes(&[]interface{}{AccountLeafKey, accountRLP})
AccountCID, _ = ipld.RawdataToCid(ipld.MEthStateTrie, accountAndLeafRLP, multihash.KECCAK_256)
AccountCodeCID, _ = util.Keccak256ToCid(ipld.RawBinary, AccountCodeHash.Bytes())
AccountCodeCID, _ = util.Keccak256ToCid(ipld.RawBinary, AccountCodeHash[:])
StoredValueRLP, _ = rlp.EncodeToBytes(StoredValue)
StoredValueRLP2, _ = rlp.EncodeToBytes("something")
@ -86,25 +87,20 @@ var (
)
func TestPGXSuite(t *testing.T) {
testConfig, err := getTestConfig()
testConfig, err := postgres.DefaultConfig.WithEnv()
require.NoError(t, err)
pool, err := statedb.NewPGXPool(testCtx, testConfig)
pool, err := postgres.ConnectPGX(testCtx, testConfig)
if err != nil {
t.Fatal(err)
}
driver, err := statedb.NewPGXDriverFromPool(context.Background(), pool)
if err != nil {
t.Fatal(err)
}
database := statedb.NewPostgresDB(driver)
t.Cleanup(func() {
tx, err := pool.Begin(testCtx)
require.NoError(t, err)
statements := []string{
`DELETE FROM eth.header_cids`,
`DELETE FROM eth.state_cids`,
`DELETE FROM eth.storage_cids`,
`DELETE FROM ipld.blocks`,
`TRUNCATE eth.header_cids`,
`TRUNCATE eth.state_cids`,
`TRUNCATE eth.storage_cids`,
`TRUNCATE ipld.blocks`,
}
for _, stm := range statements {
_, err = tx.Exec(testCtx, stm)
@ -112,221 +108,30 @@ func TestPGXSuite(t *testing.T) {
}
require.NoError(t, tx.Commit(testCtx))
})
require.NoError(t, insertHeaderCID(database, BlockHash.String(), BlockParentHash.String(), BlockNumber.Uint64()))
require.NoError(t, insertHeaderCID(database, BlockHash2.String(), BlockHash.String(), BlockNumber2))
require.NoError(t, insertHeaderCID(database, BlockHash3.String(), BlockHash2.String(), BlockNumber3))
require.NoError(t, insertHeaderCID(database, BlockHash4.String(), BlockHash3.String(), BlockNumber4))
require.NoError(t, insertHeaderCID(database, NonCanonicalHash4.String(), BlockHash3.String(), BlockNumber4))
require.NoError(t, insertHeaderCID(database, BlockHash5.String(), BlockHash4.String(), BlockNumber5))
require.NoError(t, insertHeaderCID(database, NonCanonicalHash5.String(), NonCanonicalHash4.String(), BlockNumber5))
require.NoError(t, insertHeaderCID(database, BlockHash6.String(), BlockHash5.String(), BlockNumber6))
require.NoError(t, insertStateCID(database, stateModel{
BlockNumber: BlockNumber.Uint64(),
BlockHash: BlockHash.String(),
LeafKey: AccountLeafKey.String(),
CID: AccountCID.String(),
Diff: true,
Balance: Account.Balance.Uint64(),
Nonce: Account.Nonce,
CodeHash: AccountCodeHash.String(),
StorageRoot: Account.Root.String(),
Removed: false,
}))
require.NoError(t, insertStateCID(database, stateModel{
BlockNumber: BlockNumber4,
BlockHash: NonCanonicalHash4.String(),
LeafKey: AccountLeafKey.String(),
CID: AccountCID.String(),
Diff: true,
Balance: big.NewInt(123).Uint64(),
Nonce: Account.Nonce,
CodeHash: AccountCodeHash.String(),
StorageRoot: Account.Root.String(),
Removed: false,
}))
require.NoError(t, insertStateCID(database, stateModel{
BlockNumber: BlockNumber5,
BlockHash: BlockHash5.String(),
LeafKey: AccountLeafKey.String(),
CID: RemovedNodeStateCID,
Diff: true,
Removed: true,
}))
require.NoError(t, insertStorageCID(database, storageModel{
BlockNumber: BlockNumber.Uint64(),
BlockHash: BlockHash.String(),
LeafKey: AccountLeafKey.String(),
StorageLeafKey: StorageLeafKey.String(),
StorageCID: StorageCID.String(),
Diff: true,
Value: StoredValueRLP,
Removed: false,
}))
require.NoError(t, insertStorageCID(database, storageModel{
BlockNumber: BlockNumber2,
BlockHash: BlockHash2.String(),
LeafKey: AccountLeafKey.String(),
StorageLeafKey: StorageLeafKey.String(),
StorageCID: RemovedNodeStorageCID,
Diff: true,
Value: []byte{},
Removed: true,
}))
require.NoError(t, insertStorageCID(database, storageModel{
BlockNumber: BlockNumber3,
BlockHash: BlockHash3.String(),
LeafKey: AccountLeafKey.String(),
StorageLeafKey: StorageLeafKey.String(),
StorageCID: StorageCID.String(),
Diff: true,
Value: StoredValueRLP2,
Removed: false,
}))
require.NoError(t, insertStorageCID(database, storageModel{
BlockNumber: BlockNumber4,
BlockHash: NonCanonicalHash4.String(),
LeafKey: AccountLeafKey.String(),
StorageLeafKey: StorageLeafKey.String(),
StorageCID: StorageCID.String(),
Diff: true,
Value: NonCanonStoredValueRLP,
Removed: false,
}))
require.NoError(t, insertContractCode(database))
db, err := statedb.NewStateDatabaseWithPgxPool(pool)
database := sql.NewPGXDriverFromPool(context.Background(), pool)
insertSuiteData(t, database)
db := state.NewStateDatabase(database)
require.NoError(t, err)
t.Run("Database", func(t *testing.T) {
size, err := db.ContractCodeSize(AccountCodeHash)
require.NoError(t, err)
require.Equal(t, len(AccountCode), size)
code, err := db.ContractCode(AccountCodeHash)
require.NoError(t, err)
require.Equal(t, AccountCode, code)
acct, err := db.StateAccount(AccountLeafKey, BlockHash)
require.NoError(t, err)
require.Equal(t, &Account, acct)
acct2, err := db.StateAccount(AccountLeafKey, BlockHash2)
require.NoError(t, err)
require.Equal(t, &Account, acct2)
acct3, err := db.StateAccount(AccountLeafKey, BlockHash3)
require.NoError(t, err)
require.Equal(t, &Account, acct3)
// check that we don't get the non-canonical account
acct4, err := db.StateAccount(AccountLeafKey, BlockHash4)
require.NoError(t, err)
require.Equal(t, &Account, acct4)
acct5, err := db.StateAccount(AccountLeafKey, BlockHash5)
require.NoError(t, err)
require.Nil(t, acct5)
acct6, err := db.StateAccount(AccountLeafKey, BlockHash6)
require.NoError(t, err)
require.Nil(t, acct6)
val, err := db.StorageValue(AccountLeafKey, StorageLeafKey, BlockHash)
require.NoError(t, err)
require.Equal(t, StoredValueRLP, val)
val2, err := db.StorageValue(AccountLeafKey, StorageLeafKey, BlockHash2)
require.NoError(t, err)
require.Nil(t, val2)
val3, err := db.StorageValue(AccountLeafKey, StorageLeafKey, BlockHash3)
require.NoError(t, err)
require.Equal(t, StoredValueRLP2, val3)
// this checks that we don't get the non-canonical result
val4, err := db.StorageValue(AccountLeafKey, StorageLeafKey, BlockHash4)
require.NoError(t, err)
require.Equal(t, StoredValueRLP2, val4)
// this checks that when the entire account was deleted, we return nil result for storage slot
val5, err := db.StorageValue(AccountLeafKey, StorageLeafKey, BlockHash5)
require.NoError(t, err)
require.Nil(t, val5)
val6, err := db.StorageValue(AccountLeafKey, StorageLeafKey, BlockHash6)
require.NoError(t, err)
require.Nil(t, val6)
})
t.Run("StateDB", func(t *testing.T) {
sdb, err := statedb.New(BlockHash, db)
require.NoError(t, err)
checkAccountUnchanged := func() {
require.Equal(t, Account.Balance, sdb.GetBalance(AccountAddress))
require.Equal(t, Account.Nonce, sdb.GetNonce(AccountAddress))
require.Equal(t, StoredValue, sdb.GetState(AccountAddress, StorageLeafKey))
require.Equal(t, AccountCodeHash, sdb.GetCodeHash(AccountAddress))
require.Equal(t, AccountCode, sdb.GetCode(AccountAddress))
require.Equal(t, len(AccountCode), sdb.GetCodeSize(AccountAddress))
}
require.True(t, sdb.Exist(AccountAddress))
checkAccountUnchanged()
id := sdb.Snapshot()
newStorage := crypto.Keccak256Hash([]byte{5, 4, 3, 2, 1})
newCode := []byte{1, 3, 3, 7}
sdb.SetBalance(AccountAddress, big.NewInt(300))
sdb.AddBalance(AccountAddress, big.NewInt(200))
sdb.SubBalance(AccountAddress, big.NewInt(100))
sdb.SetNonce(AccountAddress, 42)
sdb.SetState(AccountAddress, StorageLeafKey, newStorage)
sdb.SetCode(AccountAddress, newCode)
require.Equal(t, big.NewInt(400), sdb.GetBalance(AccountAddress))
require.Equal(t, uint64(42), sdb.GetNonce(AccountAddress))
require.Equal(t, newStorage, sdb.GetState(AccountAddress, StorageLeafKey))
require.Equal(t, newCode, sdb.GetCode(AccountAddress))
sdb.AddSlotToAccessList(AccountAddress, StorageLeafKey)
require.True(t, sdb.AddressInAccessList(AccountAddress))
hasAddr, hasSlot := sdb.SlotInAccessList(AccountAddress, StorageLeafKey)
require.True(t, hasAddr)
require.True(t, hasSlot)
sdb.RevertToSnapshot(id)
checkAccountUnchanged()
require.False(t, sdb.AddressInAccessList(AccountAddress))
hasAddr, hasSlot = sdb.SlotInAccessList(AccountAddress, StorageLeafKey)
require.False(t, hasAddr)
require.False(t, hasSlot)
})
testSuite(t, db)
}
func TestSQLXSuite(t *testing.T) {
testConfig, err := getTestConfig()
testConfig, err := postgres.DefaultConfig.WithEnv()
require.NoError(t, err)
pool, err := statedb.NewSQLXPool(testCtx, testConfig)
pool, err := postgres.ConnectSQLX(testCtx, testConfig)
if err != nil {
t.Fatal(err)
}
driver, err := statedb.NewSQLXDriverFromPool(context.Background(), pool)
if err != nil {
t.Fatal(err)
}
database := statedb.NewPostgresDB(driver)
t.Cleanup(func() {
tx, err := pool.Begin()
require.NoError(t, err)
statements := []string{
`DELETE FROM eth.header_cids`,
`DELETE FROM eth.state_cids`,
`DELETE FROM eth.storage_cids`,
`DELETE FROM ipld.blocks`,
`TRUNCATE eth.header_cids`,
`TRUNCATE eth.state_cids`,
`TRUNCATE eth.storage_cids`,
`TRUNCATE ipld.blocks`,
}
for _, stm := range statements {
_, err = tx.Exec(stm)
@ -334,6 +139,16 @@ func TestSQLXSuite(t *testing.T) {
}
require.NoError(t, tx.Commit())
})
database := sql.NewSQLXDriverFromPool(context.Background(), pool)
insertSuiteData(t, database)
db := state.NewStateDatabase(database)
require.NoError(t, err)
testSuite(t, db)
}
func insertSuiteData(t *testing.T, database sql.Database) {
require.NoError(t, insertHeaderCID(database, BlockHash.String(), BlockParentHash.String(), BlockNumber.Uint64()))
require.NoError(t, insertHeaderCID(database, BlockHash2.String(), BlockHash.String(), BlockNumber2))
require.NoError(t, insertHeaderCID(database, BlockHash3.String(), BlockHash2.String(), BlockNumber3))
@ -415,10 +230,9 @@ func TestSQLXSuite(t *testing.T) {
Removed: false,
}))
require.NoError(t, insertContractCode(database))
}
db, err := statedb.NewStateDatabaseWithSqlxPool(pool)
require.NoError(t, err)
func testSuite(t *testing.T, db state.StateDatabase) {
t.Run("Database", func(t *testing.T) {
size, err := db.ContractCodeSize(AccountCodeHash)
require.NoError(t, err)
@ -481,13 +295,13 @@ func TestSQLXSuite(t *testing.T) {
})
t.Run("StateDB", func(t *testing.T) {
sdb, err := statedb.New(BlockHash, db)
sdb, err := state.New(BlockHash, db)
require.NoError(t, err)
checkAccountUnchanged := func() {
require.Equal(t, Account.Balance, sdb.GetBalance(AccountAddress))
require.Equal(t, Account.Nonce, sdb.GetNonce(AccountAddress))
require.Equal(t, StoredValue, sdb.GetState(AccountAddress, StorageLeafKey))
require.Equal(t, StoredValue, sdb.GetState(AccountAddress, StorageSlot))
require.Equal(t, AccountCodeHash, sdb.GetCodeHash(AccountAddress))
require.Equal(t, AccountCode, sdb.GetCode(AccountAddress))
require.Equal(t, len(AccountCode), sdb.GetCodeSize(AccountAddress))
@ -505,17 +319,17 @@ func TestSQLXSuite(t *testing.T) {
sdb.AddBalance(AccountAddress, big.NewInt(200))
sdb.SubBalance(AccountAddress, big.NewInt(100))
sdb.SetNonce(AccountAddress, 42)
sdb.SetState(AccountAddress, StorageLeafKey, newStorage)
sdb.SetState(AccountAddress, StorageSlot, newStorage)
sdb.SetCode(AccountAddress, newCode)
require.Equal(t, big.NewInt(400), sdb.GetBalance(AccountAddress))
require.Equal(t, uint64(42), sdb.GetNonce(AccountAddress))
require.Equal(t, newStorage, sdb.GetState(AccountAddress, StorageLeafKey))
require.Equal(t, newStorage, sdb.GetState(AccountAddress, StorageSlot))
require.Equal(t, newCode, sdb.GetCode(AccountAddress))
sdb.AddSlotToAccessList(AccountAddress, StorageLeafKey)
sdb.AddSlotToAccessList(AccountAddress, StorageSlot)
require.True(t, sdb.AddressInAccessList(AccountAddress))
hasAddr, hasSlot := sdb.SlotInAccessList(AccountAddress, StorageLeafKey)
hasAddr, hasSlot := sdb.SlotInAccessList(AccountAddress, StorageSlot)
require.True(t, hasAddr)
require.True(t, hasSlot)
@ -523,13 +337,13 @@ func TestSQLXSuite(t *testing.T) {
checkAccountUnchanged()
require.False(t, sdb.AddressInAccessList(AccountAddress))
hasAddr, hasSlot = sdb.SlotInAccessList(AccountAddress, StorageLeafKey)
hasAddr, hasSlot = sdb.SlotInAccessList(AccountAddress, StorageSlot)
require.False(t, hasAddr)
require.False(t, hasSlot)
})
}
func insertHeaderCID(db statedb.Database, blockHash, parentHash string, blockNumber uint64) error {
func insertHeaderCID(db sql.Database, blockHash, parentHash string, blockNumber uint64) error {
cid, err := util.Keccak256ToCid(ipld.MEthHeader, common.HexToHash(blockHash).Bytes())
if err != nil {
return err
@ -580,7 +394,7 @@ type stateModel struct {
Removed bool
}
func insertStateCID(db statedb.Database, cidModel stateModel) error {
func insertStateCID(db sql.Database, cidModel stateModel) error {
sql := `INSERT INTO eth.state_cids (
block_number,
header_id,
@ -619,7 +433,7 @@ type storageModel struct {
Removed bool
}
func insertStorageCID(db statedb.Database, cidModel storageModel) error {
func insertStorageCID(db sql.Database, cidModel storageModel) error {
sql := `INSERT INTO eth.storage_cids (
block_number,
header_id,
@ -643,22 +457,8 @@ func insertStorageCID(db statedb.Database, cidModel storageModel) error {
return err
}
func insertContractCode(db statedb.Database) error {
func insertContractCode(db sql.Database) error {
sql := `INSERT INTO ipld.blocks (block_number, key, data) VALUES ($1, $2, $3)`
_, err := db.Exec(testCtx, sql, BlockNumber.Uint64(), AccountCodeCID.String(), AccountCode)
return err
}
func getTestConfig() (conf statedb.Config, err error) {
port, err := strconv.Atoi(os.Getenv("DATABASE_PORT"))
if err != nil {
return
}
return statedb.Config{
Hostname: os.Getenv("DATABASE_HOSTNAME"),
DatabaseName: os.Getenv("DATABASE_NAME"),
Username: os.Getenv("DATABASE_USER"),
Password: os.Getenv("DATABASE_PASSWORD"),
Port: port,
}, nil
}

View File

@ -14,7 +14,7 @@
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package ipld_eth_statedb
package state
import (
"github.com/ethereum/go-ethereum/common"

View File

@ -1,4 +1,4 @@
package util
package internal
import (
"github.com/ipfs/go-cid"

View File

@ -1,14 +1,11 @@
package ipld_eth_statedb
package sql
import (
"context"
)
// Database interfaces to support multiple Postgres drivers
type Database interface {
Driver
Statements
}
type Database = Driver
// Driver interface has all the methods required by a driver implementation to support the sql indexer
type Driver interface {
@ -25,10 +22,3 @@ type ScannableRow interface {
type Result interface {
RowsAffected() (int64, error)
}
// Statements interface to accommodate different SQL query syntax
type Statements interface {
GetContractCodeStmt() string
GetStateAccountStmt() string
GetStorageSlotStmt() string
}

View File

@ -1,4 +1,4 @@
package ipld_eth_statedb
package sql
import (
"context"
@ -15,18 +15,9 @@ type PGXDriver struct {
db *pgxpool.Pool
}
// NewPGXDriver returns a new pgx driver for Postgres
func NewPGXDriver(ctx context.Context, config Config) (*PGXDriver, error) {
db, err := NewPGXPool(ctx, config)
if err != nil {
return nil, err
}
return &PGXDriver{ctx: ctx, db: db}, nil
}
// NewPGXDriverFromPool returns a new pgx driver for Postgres
func NewPGXDriverFromPool(ctx context.Context, db *pgxpool.Pool) (*PGXDriver, error) {
return &PGXDriver{ctx: ctx, db: db}, nil
func NewPGXDriverFromPool(ctx context.Context, db *pgxpool.Pool) *PGXDriver {
return &PGXDriver{ctx: ctx, db: db}
}
// QueryRow satisfies sql.Database

View File

@ -1,4 +1,4 @@
package ipld_eth_statedb
package sql
import (
"context"
@ -14,25 +14,9 @@ type SQLXDriver struct {
db *sqlx.DB
}
// NewSQLXDriver returns a new sqlx driver for Postgres
func NewSQLXDriver(ctx context.Context, config Config) (*SQLXDriver, error) {
db, err := NewSQLXPool(ctx, config)
if err != nil {
return nil, err
}
if config.MaxConns > 0 {
db.SetMaxOpenConns(config.MaxConns)
}
if config.MaxConnLifetime > 0 {
db.SetConnMaxLifetime(config.MaxConnLifetime)
}
db.SetMaxIdleConns(config.MaxIdle)
return &SQLXDriver{ctx: ctx, db: db}, nil
}
// NewSQLXDriverFromPool returns a new sqlx driver for Postgres
func NewSQLXDriverFromPool(ctx context.Context, db *sqlx.DB) (*SQLXDriver, error) {
return &SQLXDriver{ctx: ctx, db: db}, nil
func NewSQLXDriverFromPool(ctx context.Context, db *sqlx.DB) *SQLXDriver {
return &SQLXDriver{ctx: ctx, db: db}
}
// QueryRow satisfies sql.Database

View File

@ -10,6 +10,7 @@ import (
"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
lru "github.com/hashicorp/golang-lru"
"github.com/cerc-io/ipld-eth-statedb/internal"
"github.com/cerc-io/ipld-eth-statedb/trie_by_cid/trie"
)
@ -42,6 +43,7 @@ type Database interface {
// Trie is a Ethereum Merkle Patricia trie.
type Trie interface {
TryGet(key []byte) ([]byte, error)
TryGetNode(path []byte) ([]byte, int, error)
TryGetAccount(key []byte) (*types.StateAccount, error)
Hash() common.Hash
NodeIterator(startKey []byte) trie.NodeIterator
@ -96,11 +98,10 @@ func (db *cachingDB) ContractCode(codeHash common.Hash) ([]byte, error) {
if code := db.codeCache.Get(nil, codeHash.Bytes()); len(code) > 0 {
return code, nil
}
// TODO - use non panicking
codeCID := ipld.Keccak256ToCid(ipld.RawBinary, codeHash.Bytes())
// if err != nil {
// return nil, err
// }
codeCID, err := internal.Keccak256ToCid(ipld.RawBinary, codeHash.Bytes())
if err != nil {
return nil, err
}
code, err := db.db.DiskDB().Get(codeCID.Bytes())
if err != nil {
return nil, err

View File

@ -22,12 +22,13 @@ import (
"github.com/VictoriaMetrics/fastcache"
"github.com/ethereum/go-ethereum/ethdb"
"github.com/ethereum/go-ethereum/trie"
"github.com/ipfs/go-cid"
)
type CidBytes = []byte
type CidKey = cid.Cid
func isEmpty(key CidBytes) bool {
return len(key) == 0
func isEmpty(key CidKey) bool {
return len(key.KeyString()) == 0
}
// Database is an intermediate read-only layer between the trie data structures and
@ -73,57 +74,30 @@ func (db *Database) DiskDB() ethdb.KeyValueStore {
return db.diskdb
}
// node retrieves a cached trie node from memory, or returns nil if none can be
// found in the memory cache.
func (db *Database) node(key CidBytes) (node, error) {
// Retrieve the node from the clean cache if available
if db.cleans != nil {
if enc := db.cleans.Get(nil, key); enc != nil {
// The returned value from cache is in its own copy,
// safe to use mustDecodeNodeUnsafe for decoding.
return decodeNodeUnsafe(key, enc)
}
}
// Content unavailable in memory, attempt to retrieve from disk
enc, err := db.diskdb.Get(key)
if err != nil {
return nil, err
}
if enc == nil {
return nil, nil
}
if db.cleans != nil {
db.cleans.Set(key, enc)
}
// The returned value from database is in its own copy,
// safe to use mustDecodeNodeUnsafe for decoding.
return decodeNodeUnsafe(key, enc)
}
// Node retrieves an encoded cached trie node from memory. If it cannot be found
// cached, the method queries the persistent database for the content.
func (db *Database) Node(key CidBytes) ([]byte, error) {
// Node retrieves an encoded trie node by CID. If it cannot be found
// cached in memory, it queries the persistent database.
func (db *Database) Node(key CidKey) ([]byte, error) {
// It doesn't make sense to retrieve the metaroot
if isEmpty(key) {
return nil, errors.New("not found")
}
cidbytes := key.Bytes()
// Retrieve the node from the clean cache if available
if db.cleans != nil {
if enc := db.cleans.Get(nil, key); enc != nil {
if enc := db.cleans.Get(nil, cidbytes); enc != nil {
return enc, nil
}
}
// Content unavailable in memory, attempt to retrieve from disk
enc, err := db.diskdb.Get(key)
enc, err := db.diskdb.Get(cidbytes)
if err != nil {
return nil, err
}
if len(enc) != 0 {
if db.cleans != nil {
db.cleans.Set(key[:], enc)
db.cleans.Set(cidbytes, enc)
}
return enc, nil
}

View File

@ -20,13 +20,15 @@ import (
"testing"
"github.com/ethereum/go-ethereum/ethdb/memorydb"
"github.com/cerc-io/ipld-eth-statedb/trie_by_cid/trie"
)
// Tests that the trie database returns a missing trie node error if attempting
// to retrieve the meta root.
func TestDatabaseMetarootFetch(t *testing.T) {
db := NewDatabase(memorydb.New())
if _, err := db.Node(CidBytes(nil)); err == nil {
db := trie.NewDatabase(memorydb.New())
if _, err := db.Node(trie.CidKey{}); err == nil {
t.Fatalf("metaroot retrieval succeeded")
}
}

View File

@ -16,6 +16,21 @@
package trie
// CompactToHex converts a compact encoded path to hex format
func CompactToHex(compact []byte) []byte {
if len(compact) == 0 {
return compact
}
base := keybytesToHex(compact)
// delete terminator flag
if base[0] < 2 {
base = base[:len(base)-1]
}
// apply odd flag
chop := 2 - base[0]&1
return base[chop:]
}
func keybytesToHex(str []byte) []byte {
l := len(str)*2 + 1
var nibbles = make([]byte, l)

View File

@ -21,39 +21,40 @@ import (
"context"
"fmt"
"testing"
"time"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
geth_trie "github.com/ethereum/go-ethereum/trie"
pgipfsethdb "github.com/cerc-io/ipfs-ethdb/v5/postgres/v0"
"github.com/cerc-io/ipld-eth-statedb/trie_by_cid/trie"
)
var (
cacheConfig = pgipfsethdb.CacheConfig{
Name: "db",
Size: 3000000, // 3MB
ExpiryDuration: time.Hour,
}
dbConfig, _ = postgres.DefaultConfig.WithEnv()
trieConfig = trie.Config{Cache: 256}
ctx = context.Background()
ctx = context.Background()
testdata0 = []kvs{
{"one", 1},
{"two", 2},
{"three", 3},
{"four", 4},
{"five", 5},
{"ten", 10},
}
testdata1 = []kvs{
{"barb", 0},
{"bard", 1},
{"bars", 2},
{"bar", 3},
{"fab", 4},
{"food", 5},
{"foos", 6},
{"foo", 7},
}
)
var testdata1 = []kvs{
{"barb", 0},
{"bard", 1},
{"bars", 2},
{"bar", 3},
{"fab", 4},
{"food", 5},
{"foos", 6},
{"foo", 7},
}
func TestEmptyIterator(t *testing.T) {
trie := trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase()))
iter := trie.NodeIterator(nil)
@ -71,15 +72,7 @@ func TestIterator(t *testing.T) {
edb := rawdb.NewMemoryDatabase()
db := geth_trie.NewDatabase(edb)
origtrie := geth_trie.NewEmpty(db)
vals := []kvs{
{"one", 1},
{"two", 2},
{"three", 3},
{"four", 4},
{"five", 5},
{"ten", 10},
}
all, err := updateTrie(origtrie, vals)
all, err := updateTrie(origtrie, testdata0)
if err != nil {
t.Fatal(err)
}
@ -103,22 +96,6 @@ func TestIterator(t *testing.T) {
}
}
func checkIteratorOrder(want []kvs, it *trie.Iterator) error {
for it.Next() {
if len(want) == 0 {
return fmt.Errorf("didn't expect any more values, got key %q", it.Key)
}
if !bytes.Equal(it.Key, []byte(want[0].k)) {
return fmt.Errorf("wrong key: got %q, want %q", it.Key, want[0].k)
}
want = want[1:]
}
if len(want) > 0 {
return fmt.Errorf("iterator ended early, want key %q", want[0])
}
return nil
}
func TestIteratorSeek(t *testing.T) {
edb := rawdb.NewMemoryDatabase()
db := geth_trie.NewDatabase(edb)
@ -148,11 +125,56 @@ func TestIteratorSeek(t *testing.T) {
}
}
// returns a cache config with unique name (groupcache names are global)
func makeCacheConfig(t testing.TB) pgipfsethdb.CacheConfig {
return pgipfsethdb.CacheConfig{
Name: t.Name(),
Size: 3000000, // 3MB
ExpiryDuration: time.Hour,
func checkIteratorOrder(want []kvs, it *trie.Iterator) error {
for it.Next() {
if len(want) == 0 {
return fmt.Errorf("didn't expect any more values, got key %q", it.Key)
}
if !bytes.Equal(it.Key, []byte(want[0].k)) {
return fmt.Errorf("wrong key: got %q, want %q", it.Key, want[0].k)
}
want = want[1:]
}
if len(want) > 0 {
return fmt.Errorf("iterator ended early, want key %q", want[0])
}
return nil
}
func TestIteratorNodeBlob(t *testing.T) {
edb := rawdb.NewMemoryDatabase()
db := geth_trie.NewDatabase(edb)
orig := geth_trie.NewEmpty(geth_trie.NewDatabase(rawdb.NewMemoryDatabase()))
if _, err := updateTrie(orig, testdata1); err != nil {
t.Fatal(err)
}
root := commitTrie(t, db, orig)
trie := indexTrie(t, edb, root)
found := make(map[common.Hash][]byte)
it := trie.NodeIterator(nil)
for it.Next(true) {
if it.Hash() == (common.Hash{}) {
continue
}
found[it.Hash()] = it.NodeBlob()
}
dbIter := edb.NewIterator(nil, nil)
defer dbIter.Release()
var count int
for dbIter.Next() {
got, present := found[common.BytesToHash(dbIter.Key())]
if !present {
t.Fatalf("Missing trie node %v", dbIter.Key())
}
if !bytes.Equal(got, dbIter.Value()) {
t.Fatalf("Unexpected trie node want %v got %v", dbIter.Value(), got)
}
count += 1
}
if count != len(found) {
t.Fatal("Find extra trie node via iterator")
}
}

View File

@ -95,6 +95,14 @@ func (t *StateTrie) TryGetAccount(key []byte) (*types.StateAccount, error) {
return &ret, err
}
// TryGetNode attempts to retrieve a trie node by compact-encoded path. It is not
// possible to use keybyte-encoding as the path might contain odd nibbles.
// If the specified trie node is not in the trie, nil will be returned.
// If a trie node is not found in the database, a MissingNodeError is returned.
func (t *StateTrie) TryGetNode(path []byte) ([]byte, int, error) {
return t.trie.TryGetNode(path)
}
// Hash returns the root hash of StateTrie. It does not write to the
// database and can be used even if the trie doesn't have one.
func (t *StateTrie) Hash() common.Hash {

View File

@ -3,11 +3,13 @@ package trie
import (
"bytes"
"errors"
"fmt"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/crypto"
util "github.com/cerc-io/ipld-eth-statedb/internal"
"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
)
@ -125,11 +127,99 @@ func (t *Trie) tryGet(origNode node, key []byte, pos int) (value []byte, newnode
}
}
// TryGetNode attempts to retrieve a trie node by compact-encoded path. It is not
// possible to use keybyte-encoding as the path might contain odd nibbles.
func (t *Trie) TryGetNode(path []byte) ([]byte, int, error) {
item, newroot, resolved, err := t.tryGetNode(t.root, CompactToHex(path), 0)
if err != nil {
return nil, resolved, err
}
if resolved > 0 {
t.root = newroot
}
if item == nil {
return nil, resolved, nil
}
return item, resolved, err
}
func (t *Trie) tryGetNode(origNode node, path []byte, pos int) (item []byte, newnode node, resolved int, err error) {
// If non-existent path requested, abort
if origNode == nil {
return nil, nil, 0, nil
}
// If we reached the requested path, return the current node
if pos >= len(path) {
// Although we most probably have the original node expanded, encoding
// that into consensus form can be nasty (needs to cascade down) and
// time consuming. Instead, just pull the hash up from disk directly.
var hash hashNode
if node, ok := origNode.(hashNode); ok {
hash = node
} else {
hash, _ = origNode.cache()
}
if hash == nil {
return nil, origNode, 0, errors.New("non-consensus node")
}
cid, err := util.Keccak256ToCid(t.codec, hash)
if err != nil {
return nil, origNode, 0, err
}
blob, err := t.db.Node(cid)
return blob, origNode, 1, err
}
// Path still needs to be traversed, descend into children
switch n := (origNode).(type) {
case valueNode:
// Path prematurely ended, abort
return nil, nil, 0, nil
case *shortNode:
if len(path)-pos < len(n.Key) || !bytes.Equal(n.Key, path[pos:pos+len(n.Key)]) {
// Path branches off from short node
return nil, n, 0, nil
}
item, newnode, resolved, err = t.tryGetNode(n.Val, path, pos+len(n.Key))
if err == nil && resolved > 0 {
n = n.copy()
n.Val = newnode
}
return item, n, resolved, err
case *fullNode:
item, newnode, resolved, err = t.tryGetNode(n.Children[path[pos]], path, pos+1)
if err == nil && resolved > 0 {
n = n.copy()
n.Children[path[pos]] = newnode
}
return item, n, resolved, err
case hashNode:
child, err := t.resolveHash(n, path[:pos])
if err != nil {
return nil, n, 1, err
}
item, newnode, resolved, err := t.tryGetNode(child, path, pos)
return item, newnode, resolved + 1, err
default:
panic(fmt.Sprintf("%T: invalid node: %v", origNode, origNode))
}
}
// resolveHash loads node from the underlying database with the provided
// node hash and path prefix.
func (t *Trie) resolveHash(n hashNode, prefix []byte) (node, error) {
cid := ipld.Keccak256ToCid(t.codec, n)
node, err := t.db.node(cid.Bytes())
cid, err := util.Keccak256ToCid(t.codec, n)
if err != nil {
return nil, err
}
enc, err := t.db.Node(cid)
if err != nil {
return nil, &MissingNodeError{Owner: t.owner, NodeHash: n, Path: prefix, err: err}
}
node, err := decodeNodeUnsafe(n, enc)
if err != nil {
return nil, err
}
@ -142,8 +232,14 @@ func (t *Trie) resolveHash(n hashNode, prefix []byte) (node, error) {
// resolveHash loads rlp-encoded node blob from the underlying database
// with the provided node hash and path prefix.
func (t *Trie) resolveBlob(n hashNode, prefix []byte) ([]byte, error) {
cid := ipld.Keccak256ToCid(t.codec, n)
blob, _ := t.db.Node(cid.Bytes())
cid, err := util.Keccak256ToCid(t.codec, n)
if err != nil {
return nil, err
}
blob, err := t.db.Node(cid)
if err != nil {
return nil, err
}
if len(blob) != 0 {
return blob, nil
}
@ -153,20 +249,20 @@ func (t *Trie) resolveBlob(n hashNode, prefix []byte) ([]byte, error) {
// Hash returns the root hash of the trie. It does not write to the
// database and can be used even if the trie doesn't have one.
func (t *Trie) Hash() common.Hash {
hash, cached, _ := t.hashRoot()
hash, cached := t.hashRoot()
t.root = cached
return common.BytesToHash(hash.(hashNode))
}
// hashRoot calculates the root hash of the given trie
func (t *Trie) hashRoot() (node, node, error) {
func (t *Trie) hashRoot() (node, node) {
if t.root == nil {
return hashNode(emptyRoot.Bytes()), nil, nil
return hashNode(emptyRoot.Bytes()), nil
}
// If the number of changes is below 100, we let one thread handle it
h := newHasher(t.unhashed >= 100)
defer returnHasherToPool(h)
hashed, cached := h.hash(t.root, true)
t.unhashed = 0
return hashed, cached, nil
return hashed, cached
}

View File

@ -0,0 +1,173 @@
// Copyright 2014 The go-ethereum Authors
// This file is part of the go-ethereum library.
//
// The go-ethereum library is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// The go-ethereum library is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with the go-ethereum library. If not, see <http://www.gnu.org/licenses/>.
package trie_test
import (
"fmt"
"math/big"
"math/rand"
"testing"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/rawdb"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/rlp"
geth_trie "github.com/ethereum/go-ethereum/trie"
"github.com/cerc-io/ipld-eth-statedb/trie_by_cid/trie"
)
func TestTrieEmpty(t *testing.T) {
trie := trie.NewEmpty(trie.NewDatabase(rawdb.NewMemoryDatabase()))
res := trie.Hash()
exp := types.EmptyRootHash
if res != exp {
t.Errorf("expected %x got %x", exp, res)
}
}
func TestTrieMissingRoot(t *testing.T) {
root := common.HexToHash("0beec7b5ea3f0fdbc95d0dd47f3c5bc275da8a33")
tr, err := newStateTrie(root, trie.NewDatabase(rawdb.NewMemoryDatabase()))
if tr != nil {
t.Error("New returned non-nil trie for invalid root")
}
if _, ok := err.(*trie.MissingNodeError); !ok {
t.Errorf("New returned wrong error: %v", err)
}
}
func TestTrieBasic(t *testing.T) {
edb := rawdb.NewMemoryDatabase()
db := geth_trie.NewDatabase(edb)
origtrie := geth_trie.NewEmpty(db)
origtrie.Update([]byte("foo"), packValue(842))
expected := commitTrie(t, db, origtrie)
tr := indexTrie(t, edb, expected)
got := tr.Hash()
if expected != got {
t.Errorf("got %x expected %x", got, expected)
}
checkValue(t, tr, []byte("foo"))
}
func TestTrieTiny(t *testing.T) {
// Create a realistic account trie to hash
_, accounts := makeAccounts(5)
edb := rawdb.NewMemoryDatabase()
db := geth_trie.NewDatabase(edb)
origtrie := geth_trie.NewEmpty(db)
type testCase struct {
key, account []byte
root common.Hash
}
cases := []testCase{
{
common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000001337"),
accounts[3],
common.HexToHash("8c6a85a4d9fda98feff88450299e574e5378e32391f75a055d470ac0653f1005"),
}, {
common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000001338"),
accounts[4],
common.HexToHash("ec63b967e98a5720e7f720482151963982890d82c9093c0d486b7eb8883a66b1"),
}, {
common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000001339"),
accounts[4],
common.HexToHash("0608c1d1dc3905fa22204c7a0e43644831c3b6d3def0f274be623a948197e64a"),
},
}
for i, tc := range cases {
t.Run(fmt.Sprintf("case %d", i), func(t *testing.T) {
origtrie.Update(tc.key, tc.account)
trie := indexTrie(t, edb, commitTrie(t, db, origtrie))
if exp, root := tc.root, trie.Hash(); exp != root {
t.Errorf("got %x, exp %x", root, exp)
}
checkValue(t, trie, tc.key)
})
}
}
func TestTrieMedium(t *testing.T) {
// Create a realistic account trie to hash
addresses, accounts := makeAccounts(1000)
edb := rawdb.NewMemoryDatabase()
db := geth_trie.NewDatabase(edb)
origtrie := geth_trie.NewEmpty(db)
var keys [][]byte
for i := 0; i < len(addresses); i++ {
key := crypto.Keccak256(addresses[i][:])
if i%50 == 0 {
keys = append(keys, key)
}
origtrie.Update(key, accounts[i])
}
tr := indexTrie(t, edb, commitTrie(t, db, origtrie))
root := tr.Hash()
exp := common.HexToHash("72f9d3f3fe1e1dd7b8936442e7642aef76371472d94319900790053c493f3fe6")
if exp != root {
t.Errorf("got %x, exp %x", root, exp)
}
for _, key := range keys {
checkValue(t, tr, key)
}
}
// Make deterministically random accounts
func makeAccounts(size int) (addresses [][20]byte, accounts [][]byte) {
random := rand.New(rand.NewSource(0))
addresses = make([][20]byte, size)
for i := 0; i < len(addresses); i++ {
data := make([]byte, 20)
random.Read(data)
copy(addresses[i][:], data)
}
accounts = make([][]byte, len(addresses))
for i := 0; i < len(accounts); i++ {
var (
nonce = uint64(random.Int63())
root = types.EmptyRootHash
code = crypto.Keccak256(nil)
)
// The big.Rand function is not deterministic with regards to 64 vs 32 bit systems,
// and will consume different amount of data from the rand source.
// balance = new(big.Int).Rand(random, new(big.Int).Exp(common.Big2, common.Big256, nil))
// Therefore, we instead just read via byte buffer
numBytes := random.Uint32() % 33 // [0, 32] bytes
balanceBytes := make([]byte, numBytes)
random.Read(balanceBytes)
balance := new(big.Int).SetBytes(balanceBytes)
acct := &types.StateAccount{Nonce: nonce, Balance: balance, Root: root, CodeHash: code}
data, _ := rlp.EncodeToBytes(acct)
accounts[i] = data
}
return addresses, accounts
}
func checkValue(t *testing.T, tr *trie.Trie, key []byte) {
val, err := tr.TryGet(key)
if err != nil {
t.Fatalf("error getting node: %s", err)
}
if len(val) == 0 {
t.Errorf("failed to get value for %x", key)
}
}

View File

@ -5,6 +5,7 @@ import (
"math/big"
"math/rand"
"testing"
"time"
"github.com/jmoiron/sqlx"
@ -100,13 +101,21 @@ func indexTrie(t testing.TB, edb ethdb.Database, root common.Hash) *trie.Trie {
ipfs_db := pgipfsethdb.NewDatabase(pg_db, makeCacheConfig(t))
sdb_db := state.NewDatabase(ipfs_db)
tr, err := trie.New(common.Hash{}, root, sdb_db.TrieDB(), ipld.MEthStateTrie)
tr, err := newStateTrie(root, sdb_db.TrieDB())
if err != nil {
t.Fatal(err)
}
return tr
}
func newStateTrie(root common.Hash, db *trie.Database) (*trie.Trie, error) {
tr, err := trie.New(common.Hash{}, root, db, ipld.MEthStateTrie)
if err != nil {
return nil, err
}
return tr, nil
}
// generates a random Geth LevelDB trie of n key-value pairs and corresponding value map
func randomGethTrie(n int, db *geth_trie.Database) (*geth_trie.Trie, kvMap) {
trie := geth_trie.NewEmpty(db)
@ -170,3 +179,12 @@ func TearDownDB(db *sqlx.DB) error {
}
return tx.Commit()
}
// returns a cache config with unique name (groupcache names are global)
func makeCacheConfig(t testing.TB) pgipfsethdb.CacheConfig {
return pgipfsethdb.CacheConfig{
Name: t.Name(),
Size: 3000000, // 3MB
ExpiryDuration: time.Hour,
}
}