From 784860a7f01569fceaa844a89359ccaeb4ba644d Mon Sep 17 00:00:00 2001 From: i-norden Date: Mon, 13 Mar 2023 14:06:45 -0500 Subject: [PATCH] create database abstraction to make it easier to use different drivers; add sqlx driver support to make integrating into ipld-eth-server database metrics easier --- config.go | 26 +++++ database.go | 139 +++-------------------- go.mod | 4 +- go.sum | 33 ++++++ interfaces.go | 50 +++++++++ pgx.go | 50 +++++++++ sqlx.go | 46 ++++++++ state_database.go | 138 +++++++++++++++++++++++ state_object.go | 10 +- statedb.go | 4 +- statedb_test.go | 279 +++++++++++++++++++++++++++++++++++++++++----- 11 files changed, 622 insertions(+), 157 deletions(-) create mode 100644 interfaces.go create mode 100644 pgx.go create mode 100644 sqlx.go create mode 100644 state_database.go diff --git a/config.go b/config.go index 89c147b..8c5a40a 100644 --- a/config.go +++ b/config.go @@ -2,9 +2,12 @@ package ipld_eth_statedb import ( "context" + "fmt" "time" "github.com/jackc/pgx/v4/pgxpool" + "github.com/jmoiron/sqlx" + _ "github.com/lib/pq" ) type Config struct { @@ -19,6 +22,20 @@ type Config struct { MinConns int MaxConnLifetime time.Duration MaxConnIdleTime time.Duration + MaxIdle int +} + +// DbConnectionString constructs and returns the connection string from the config (for sqlx driver) +func (c Config) DbConnectionString() string { + if len(c.Username) > 0 && len(c.Password) > 0 { + return fmt.Sprintf("postgresql://%s:%s@%s:%d/%s?sslmode=disable", + c.Username, c.Password, c.Hostname, c.Port, c.DatabaseName) + } + if len(c.Username) > 0 && len(c.Password) == 0 { + return fmt.Sprintf("postgresql://%s@%s:%d/%s?sslmode=disable", + c.Username, c.Hostname, c.Port, c.DatabaseName) + } + return fmt.Sprintf("postgresql://%s:%d/%s?sslmode=disable", c.Hostname, c.Port, c.DatabaseName) } // NewPGXPool returns a new pgx conn pool @@ -30,6 +47,15 @@ func NewPGXPool(ctx context.Context, config Config) (*pgxpool.Pool, error) { return pgxpool.ConnectConfig(ctx, pgConf) } +// NewSQLXPool returns a new sqlx conn pool +func NewSQLXPool(ctx context.Context, config Config) (*sqlx.DB, error) { + db, err := sqlx.ConnectContext(ctx, "postgres", config.DbConnectionString()) + if err != nil { + return nil, err + } + return db, nil +} + // makePGXConfig creates a pgxpool.Config from the provided Config func makePGXConfig(config Config) (*pgxpool.Config, error) { conf, err := pgxpool.ParseConfig("") diff --git a/database.go b/database.go index 3b754b8..35e6ace 100644 --- a/database.go +++ b/database.go @@ -1,135 +1,28 @@ package ipld_eth_statedb -import ( - "context" - "errors" - "fmt" - "math/big" +var _ Database = &DB{} - "github.com/VictoriaMetrics/fastcache" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/statediff/indexer/ipld" - lru "github.com/hashicorp/golang-lru" - "github.com/jackc/pgx/v4/pgxpool" - - util "github.com/cerc-io/ipld-eth-statedb/internal" -) - -const ( - // Number of codehash->size associations to keep. - codeSizeCacheSize = 100000 - - // Cache size granted for caching clean code. - codeCacheSize = 64 * 1024 * 1024 -) - -var ( - // not found error - errNotFound = errors.New("not found") -) - -// Database interface is a union of the subset of the geth state.Database interface required -// to support the vm.StateDB implementation as well as methods specific to this Postgres based implementation -type Database interface { - ContractCode(codeHash common.Hash) ([]byte, error) - ContractCodeSize(codeHash common.Hash) (int, error) - StateAccount(addressHash, blockHash common.Hash) (*types.StateAccount, error) - StorageValue(addressHash, slotHash, blockHash common.Hash) ([]byte, error) +// NewPostgresDB returns a postgres.DB using the provided driver +func NewPostgresDB(driver Driver) *DB { + return &DB{driver} } -var _ Database = &stateDatabase{} - -type stateDatabase struct { - pgdb *pgxpool.Pool - codeSizeCache *lru.Cache - codeCache *fastcache.Cache +// DB implements sql.Database using a configured driver and Postgres statement syntax +type DB struct { + Driver } -// NewStateDatabaseWithPool returns a new Database implementation using the provided postgres connection pool -func NewStateDatabaseWithPool(pgDb *pgxpool.Pool) (*stateDatabase, error) { - csc, _ := lru.New(codeSizeCacheSize) - return &stateDatabase{ - pgdb: pgDb, - codeSizeCache: csc, - codeCache: fastcache.New(codeCacheSize), - }, nil +// GetContractCodeStmt satisfies the Statements interface +func (db *DB) GetContractCodeStmt() string { + return GetContractCodePgStr } -// NewStateDatabase returns a new Database implementation using the passed parameters -func NewStateDatabase(ctx context.Context, conf Config) (*stateDatabase, error) { - pgDb, err := NewPGXPool(ctx, conf) - if err != nil { - return nil, err - } - return NewStateDatabaseWithPool(pgDb) +// GetStateAccountStmt satisfies the Statements interface +func (db *DB) GetStateAccountStmt() string { + return GetStateAccount } -// ContractCode satisfies Database, it returns the contract code for a given codehash -func (sd *stateDatabase) ContractCode(codeHash common.Hash) ([]byte, error) { - if code := sd.codeCache.Get(nil, codeHash.Bytes()); len(code) > 0 { - return code, nil - } - c, err := util.Keccak256ToCid(ipld.RawBinary, codeHash.Bytes()) - if err != nil { - return nil, fmt.Errorf("cannot derive CID from provided codehash: %s", err.Error()) - } - code := make([]byte, 0) - if err := sd.pgdb.QueryRow(context.Background(), GetContractCodePgStr, c).Scan(&code); err != nil { - return nil, err - } - if len(code) > 0 { - sd.codeCache.Set(codeHash.Bytes(), code) - sd.codeSizeCache.Add(codeHash, len(code)) - return code, nil - } - return nil, errNotFound -} - -// ContractCodeSize satisfies Database, it returns the length of the code for a provided codehash -func (sd *stateDatabase) ContractCodeSize(codeHash common.Hash) (int, error) { - if cached, ok := sd.codeSizeCache.Get(codeHash); ok { - return cached.(int), nil - } - code, err := sd.ContractCode(codeHash) - return len(code), err -} - -// StateAccount satisfies Database, it returns the types.StateAccount for a provided address and block hash -func (sd *stateDatabase) StateAccount(addressHash, blockHash common.Hash) (*types.StateAccount, error) { - res := StateAccountResult{} - err := sd.pgdb.QueryRow(context.Background(), GetStateAccount, addressHash.Hex(), blockHash.Hex()). - Scan(&res.Balance, &res.Nonce, &res.CodeHash, &res.StorageRoot, &res.Removed) - if err != nil { - return nil, err - } - if res.Removed { - // TODO: check expected behavior for deleted/non existing accounts - return nil, nil - } - bal := new(big.Int) - bal.SetString(res.Balance, 10) - return &types.StateAccount{ - Nonce: res.Nonce, - Balance: bal, - Root: common.HexToHash(res.StorageRoot), - CodeHash: common.HexToHash(res.CodeHash).Bytes(), - }, nil -} - -// StorageValue satisfies Database, it returns the RLP-encoded storage value for the provided address, slot, -// and block hash -func (sd *stateDatabase) StorageValue(addressHash, slotHash, blockHash common.Hash) ([]byte, error) { - res := StorageSlotResult{} - err := sd.pgdb.QueryRow(context.Background(), GetStorageSlot, - addressHash.Hex(), slotHash.Hex(), blockHash.Hex()). - Scan(&res.Value, &res.Removed, &res.StateLeafRemoved) - if err != nil { - return nil, err - } - if res.Removed || res.StateLeafRemoved { - // TODO: check expected behavior for deleted/non existing accounts - return nil, nil - } - return res.Value, nil +// GetStorageSlotStmt satisfies the Statements interface +func (db *DB) GetStorageSlotStmt() string { + return GetStorageSlot } diff --git a/go.mod b/go.mod index 972c01e..81479f4 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,10 @@ require ( github.com/ethereum/go-ethereum v1.10.26 github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d github.com/ipfs/go-cid v0.2.0 + github.com/jackc/pgconn v1.14.0 github.com/jackc/pgx/v4 v4.18.1 + github.com/jmoiron/sqlx v1.2.0 + github.com/lib/pq v1.10.6 github.com/multiformats/go-multihash v0.1.0 github.com/stretchr/testify v1.8.1 ) @@ -25,7 +28,6 @@ require ( github.com/ipfs/go-ipfs-util v0.0.2 // indirect github.com/ipfs/go-ipld-format v0.4.0 // indirect github.com/jackc/chunkreader/v2 v2.0.1 // indirect - github.com/jackc/pgconn v1.14.0 // indirect github.com/jackc/pgio v1.0.0 // indirect github.com/jackc/pgpassfile v1.0.0 // indirect github.com/jackc/pgproto3/v2 v2.3.2 // indirect diff --git a/go.sum b/go.sum index b2b00d9..356c82d 100644 --- a/go.sum +++ b/go.sum @@ -5,16 +5,19 @@ github.com/VictoriaMetrics/fastcache v1.6.0 h1:C/3Oi3EiBCqufydp1neRZkqcwmEiuRT9c github.com/VictoriaMetrics/fastcache v1.6.0/go.mod h1:0qHz5QP0GMX4pfmMA/zt5RgfNuXJrTP0zS7DqpHGGTw= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/btcsuite/btcd/btcec/v2 v2.2.0 h1:fzn1qaOt32TuLjFlkzYSsBC35Q3KUjT1SwPxiMSCF5k= github.com/btcsuite/btcd/btcec/v2 v2.2.0/go.mod h1:U7MHm051Al6XmscBQ0BoNydpOTsFAn707034b5nY8zU= +github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U= github.com/cerc-io/go-ethereum v1.10.26-statediff-4.2.2-alpha h1:gesMZEbNU+fcAMctITi+KO/AK80YdTq6TVB5lb4EfnU= github.com/cerc-io/go-ethereum v1.10.26-statediff-4.2.2-alpha/go.mod h1:lKBVBWksSwBDR/5D9CAxaGQzDPIS3ueWb6idy7X1Shg= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= @@ -22,20 +25,29 @@ github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7Do github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/deckarep/golang-set v1.8.0 h1:sk9/l/KqpunDwP7pSjUg0keiOOLEnOBHzykLrsPppp4= +github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0= github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1 h1:YLtO71vCjJRCBcrPMtQ9nqBsqpA1m5sE92cU+pd5Mcc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/edsrzf/mmap-go v1.0.0 h1:CEBF7HpRnUCSJgGUb5h1Gm7e3VkmVDrR8lvWVLtrOFw= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/go-kit/kit v0.8.0 h1:Wz+5lgoB0kkuqLEc6NVmwRknTKP6dTGbSqvhZtBI/j0= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= +github.com/go-logfmt/logfmt v0.5.0 h1:TrB8swr/68K7m9CcGut2g3UOihhbcbiMAYiuTXdEih4= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -54,10 +66,12 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc= github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d h1:dg1dEPuWpEqDnvIw251EVy4zlP8gWbsGj4BsUKCRpYs= github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= github.com/holiman/uint256 v1.2.0 h1:gpSYcPLWGv4sG43I2mVLiDZCNDh/EpGjSk8tmtxitHM= github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= @@ -90,6 +104,7 @@ github.com/jackc/pgio v1.0.0 h1:g12B9UwVnzGhueNavwioyEEpAmqMe1E/BN9ES+8ovkE= github.com/jackc/pgio v1.0.0/go.mod h1:oP+2QK2wFfUWgr+gxjoBH9KGBb31Eio69xUb0w5bYf8= github.com/jackc/pgmock v0.0.0-20190831213851-13a1b77aafa2/go.mod h1:fGZlG77KXmcq05nJLRkk0+p82V8B8Dw8KN2/V9c/OAE= github.com/jackc/pgmock v0.0.0-20201204152224-4fe30f7445fd/go.mod h1:hrBW0Enj2AZTNpt/7Y5rr2xe/9Mn757Wtb2xeBzPv2c= +github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65 h1:DadwsjnMwFjfWc9y5Wi/+Zz7xoE5ALHsRQlOctkOiHc= github.com/jackc/pgmock v0.0.0-20210724152146-4ad1a8207f65/go.mod h1:5R2h2EEX+qri8jOWMbJCtaPWkrrNc7OHwsp2TCqp7ak= github.com/jackc/pgpassfile v1.0.0 h1:/6Hmqy13Ss2zCq62VdNG8tM1wchn8zjSGOBJ6icpsIM= github.com/jackc/pgpassfile v1.0.0/go.mod h1:CEx0iS5ambNFdcRtxPj5JhEz+xB6uRky5eyVu/W2HEg= @@ -122,6 +137,8 @@ github.com/jackc/puddle v0.0.0-20190608224051-11cab39313c9/go.mod h1:m4B5Dj62Y0f github.com/jackc/puddle v1.1.3/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= github.com/jackc/puddle v1.3.0 h1:eHK/5clGOatcjX3oWGBO/MpxpbHzSwud5EWTSCI+MX0= github.com/jackc/puddle v1.3.0/go.mod h1:m4B5Dj62Y0fbyuIc15OsIqK0+JU8nkqQjsgx7dvjSWk= +github.com/jmoiron/sqlx v1.2.0 h1:41Ip0zITnmWNR/vHV+S4m+VoUivnWY5E4OJfLZjCJMA= +github.com/jmoiron/sqlx v1.2.0/go.mod h1:1FEQNm3xlJgrMD+FBdI9+xvCksHtbpVBBw5dYhBSsks= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= @@ -131,14 +148,18 @@ github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02 github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.2/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.6 h1:jbk+ZieJ0D7EVGJYpL9QTz7/YW6UHbmdnZWYyK5cdBs= +github.com/lib/pq v1.10.6/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= @@ -146,6 +167,8 @@ github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hd github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= +github.com/mattn/go-sqlite3 v1.9.0 h1:pDRiWfl+++eC2FEFRy6jXmQlvp4Yh3z1MJKg4UeYM/4= +github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1/go.mod h1:pD8RvIylQ358TN4wwqatJ8rNavkEINozVn9DtGI3dfQ= @@ -175,14 +198,17 @@ github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXS github.com/multiformats/go-varint v0.0.6 h1:gk85QWKxh3TazbLxED/NlDVv8+q+ReFJk7Y2W/KhfNY= github.com/multiformats/go-varint v0.0.6/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= +github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -207,6 +233,7 @@ github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdh github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= github.com/shirou/gopsutil v3.21.11+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9NzErvs504Cn4c5DxATwIqPbtswREoFCre64PpcG4= +github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= @@ -279,6 +306,7 @@ golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.6.0 h1:L4ZwwTvKW9gr0ZMS1yrHD9GZhIuVjOBBnaKH+SPQK0Q= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -342,6 +370,7 @@ golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8T golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20220517211312-f3a8303e98df h1:5Pf6pFKu98ODmgnpvkJ3kFUOQGGLIzLIkbzUHp47618= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -350,15 +379,19 @@ google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzi google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= +gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/interfaces.go b/interfaces.go new file mode 100644 index 0000000..84bba3d --- /dev/null +++ b/interfaces.go @@ -0,0 +1,50 @@ +// VulcanizeDB +// Copyright © 2023 Vulcanize + +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. + +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. + +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . + +package ipld_eth_statedb + +import ( + "context" +) + +// Database interfaces to support multiple Postgres drivers +type Database interface { + Driver + Statements +} + +// Driver interface has all the methods required by a driver implementation to support the sql indexer +type Driver interface { + QueryRow(ctx context.Context, sql string, args ...interface{}) ScannableRow + Exec(ctx context.Context, sql string, args ...interface{}) (Result, error) +} + +// ScannableRow interface to accommodate different concrete row types +type ScannableRow interface { + Scan(dest ...interface{}) error +} + +// Result interface to accommodate different concrete result types +type Result interface { + RowsAffected() (int64, error) +} + +// Statements interface to accommodate different SQL query syntax +type Statements interface { + GetContractCodeStmt() string + GetStateAccountStmt() string + GetStorageSlotStmt() string +} diff --git a/pgx.go b/pgx.go new file mode 100644 index 0000000..c780d20 --- /dev/null +++ b/pgx.go @@ -0,0 +1,50 @@ +package ipld_eth_statedb + +import ( + "context" + + "github.com/jackc/pgconn" + "github.com/jackc/pgx/v4/pgxpool" +) + +var _ Driver = &PGXDriver{} + +// PGXDriver driver, implements Driver +type PGXDriver struct { + ctx context.Context + db *pgxpool.Pool +} + +// NewPGXDriver returns a new pgx driver for Postgres +func NewPGXDriver(ctx context.Context, config Config) (*PGXDriver, error) { + db, err := NewPGXPool(ctx, config) + if err != nil { + return nil, err + } + return &PGXDriver{ctx: ctx, db: db}, nil +} + +// NewPGXDriverFromPool returns a new pgx driver for Postgres +func NewPGXDriverFromPool(ctx context.Context, db *pgxpool.Pool) (*PGXDriver, error) { + return &PGXDriver{ctx: ctx, db: db}, nil +} + +// QueryRow satisfies sql.Database +func (driver *PGXDriver) QueryRow(ctx context.Context, sql string, args ...interface{}) ScannableRow { + return driver.db.QueryRow(ctx, sql, args...) +} + +// Exec satisfies sql.Database +func (pgx *PGXDriver) Exec(ctx context.Context, sql string, args ...interface{}) (Result, error) { + res, err := pgx.db.Exec(ctx, sql, args...) + return resultWrapper{ct: res}, err +} + +type resultWrapper struct { + ct pgconn.CommandTag +} + +// RowsAffected satisfies sql.Result +func (r resultWrapper) RowsAffected() (int64, error) { + return r.ct.RowsAffected(), nil +} diff --git a/sqlx.go b/sqlx.go new file mode 100644 index 0000000..92f4c03 --- /dev/null +++ b/sqlx.go @@ -0,0 +1,46 @@ +package ipld_eth_statedb + +import ( + "context" + + "github.com/jmoiron/sqlx" +) + +var _ Driver = &SQLXDriver{} + +// SQLXDriver driver, implements Driver +type SQLXDriver struct { + ctx context.Context + db *sqlx.DB +} + +// NewSQLXDriver returns a new sqlx driver for Postgres +func NewSQLXDriver(ctx context.Context, config Config) (*SQLXDriver, error) { + db, err := NewSQLXPool(ctx, config) + if err != nil { + return nil, err + } + if config.MaxConns > 0 { + db.SetMaxOpenConns(config.MaxConns) + } + if config.MaxConnLifetime > 0 { + db.SetConnMaxLifetime(config.MaxConnLifetime) + } + db.SetMaxIdleConns(config.MaxIdle) + return &SQLXDriver{ctx: ctx, db: db}, nil +} + +// NewSQLXDriverFromPool returns a new sqlx driver for Postgres +func NewSQLXDriverFromPool(ctx context.Context, db *sqlx.DB) (*SQLXDriver, error) { + return &SQLXDriver{ctx: ctx, db: db}, nil +} + +// QueryRow satisfies sql.Database +func (driver *SQLXDriver) QueryRow(_ context.Context, sql string, args ...interface{}) ScannableRow { + return driver.db.QueryRowx(sql, args...) +} + +// Exec satisfies sql.Database +func (driver *SQLXDriver) Exec(_ context.Context, sql string, args ...interface{}) (Result, error) { + return driver.db.Exec(sql, args...) +} diff --git a/state_database.go b/state_database.go new file mode 100644 index 0000000..b7d83ea --- /dev/null +++ b/state_database.go @@ -0,0 +1,138 @@ +package ipld_eth_statedb + +import ( + "context" + "errors" + "fmt" + "math/big" + + "github.com/jmoiron/sqlx" + + "github.com/VictoriaMetrics/fastcache" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/statediff/indexer/ipld" + lru "github.com/hashicorp/golang-lru" + "github.com/jackc/pgx/v4/pgxpool" + + util "github.com/cerc-io/ipld-eth-statedb/internal" +) + +const ( + // Number of codehash->size associations to keep. + codeSizeCacheSize = 100000 + + // Cache size granted for caching clean code. + codeCacheSize = 64 * 1024 * 1024 +) + +var ( + // not found error + errNotFound = errors.New("not found") +) + +// StateDatabase interface is a union of the subset of the geth state.Database interface required +// to support the vm.StateDB implementation as well as methods specific to this Postgres based implementation +type StateDatabase interface { + ContractCode(codeHash common.Hash) ([]byte, error) + ContractCodeSize(codeHash common.Hash) (int, error) + StateAccount(addressHash, blockHash common.Hash) (*types.StateAccount, error) + StorageValue(addressHash, slotHash, blockHash common.Hash) ([]byte, error) +} + +var _ StateDatabase = &stateDatabase{} + +type stateDatabase struct { + db Database + codeSizeCache *lru.Cache + codeCache *fastcache.Cache +} + +// NewStateDatabaseWithPgxPool returns a new Database implementation using the provided postgres connection pool +func NewStateDatabaseWithPgxPool(pgDb *pgxpool.Pool) (*stateDatabase, error) { + csc, _ := lru.New(codeSizeCacheSize) + return &stateDatabase{ + db: NewPostgresDB(&PGXDriver{db: pgDb}), + codeSizeCache: csc, + codeCache: fastcache.New(codeCacheSize), + }, nil +} + +// NewStateDatabaseWithSqlxPool returns a new Database implementation using the passed parameters +func NewStateDatabaseWithSqlxPool(db *sqlx.DB) (*stateDatabase, error) { + csc, _ := lru.New(codeSizeCacheSize) + return &stateDatabase{ + db: NewPostgresDB(&SQLXDriver{db: db}), + codeSizeCache: csc, + codeCache: fastcache.New(codeCacheSize), + }, nil +} + +// ContractCode satisfies Database, it returns the contract code for a given codehash +func (sd *stateDatabase) ContractCode(codeHash common.Hash) ([]byte, error) { + if code := sd.codeCache.Get(nil, codeHash.Bytes()); len(code) > 0 { + return code, nil + } + c, err := util.Keccak256ToCid(ipld.RawBinary, codeHash.Bytes()) + if err != nil { + return nil, fmt.Errorf("cannot derive CID from provided codehash: %s", err.Error()) + } + code := make([]byte, 0) + if err := sd.db.QueryRow(context.Background(), GetContractCodePgStr, c.String()).Scan(&code); err != nil { + return nil, err + } + if len(code) > 0 { + sd.codeCache.Set(codeHash.Bytes(), code) + sd.codeSizeCache.Add(codeHash, len(code)) + return code, nil + } + return nil, errNotFound +} + +// ContractCodeSize satisfies Database, it returns the length of the code for a provided codehash +func (sd *stateDatabase) ContractCodeSize(codeHash common.Hash) (int, error) { + if cached, ok := sd.codeSizeCache.Get(codeHash); ok { + return cached.(int), nil + } + code, err := sd.ContractCode(codeHash) + return len(code), err +} + +// StateAccount satisfies Database, it returns the types.StateAccount for a provided address and block hash +func (sd *stateDatabase) StateAccount(addressHash, blockHash common.Hash) (*types.StateAccount, error) { + res := StateAccountResult{} + err := sd.db.QueryRow(context.Background(), GetStateAccount, addressHash.Hex(), blockHash.Hex()). + Scan(&res.Balance, &res.Nonce, &res.CodeHash, &res.StorageRoot, &res.Removed) + if err != nil { + return nil, err + } + if res.Removed { + // TODO: check expected behavior for deleted/non existing accounts + return nil, nil + } + bal := new(big.Int) + bal.SetString(res.Balance, 10) + return &types.StateAccount{ + Nonce: res.Nonce, + Balance: bal, + Root: common.HexToHash(res.StorageRoot), + CodeHash: common.HexToHash(res.CodeHash).Bytes(), + }, nil +} + +// StorageValue satisfies Database, it returns the RLP-encoded storage value for the provided address, slot, +// and block hash +func (sd *stateDatabase) StorageValue(addressHash, slotHash, blockHash common.Hash) ([]byte, error) { + res := StorageSlotResult{} + err := sd.db.QueryRow(context.Background(), GetStorageSlot, + addressHash.Hex(), slotHash.Hex(), blockHash.Hex()). + Scan(&res.Value, &res.Removed, &res.StateLeafRemoved) + if err != nil { + return nil, err + } + if res.Removed || res.StateLeafRemoved { + // TODO: check expected behavior for deleted/non existing accounts + return nil, nil + } + return res.Value, nil +} diff --git a/state_object.go b/state_object.go index b6613d1..9ae1e7c 100644 --- a/state_object.go +++ b/state_object.go @@ -133,7 +133,7 @@ func (s *stateObject) touch() { } // GetState retrieves a value from the account storage trie. -func (s *stateObject) GetState(db Database, key common.Hash) common.Hash { +func (s *stateObject) GetState(db StateDatabase, key common.Hash) common.Hash { // If the fake storage is set, only lookup the state here(in the debugging mode) if s.fakeStorage != nil { return s.fakeStorage[key] @@ -148,7 +148,7 @@ func (s *stateObject) GetState(db Database, key common.Hash) common.Hash { } // GetCommittedState retrieves a value from the committed account storage trie. -func (s *stateObject) GetCommittedState(db Database, key common.Hash) common.Hash { +func (s *stateObject) GetCommittedState(db StateDatabase, key common.Hash) common.Hash { // If the fake storage is set, only lookup the state here(in the debugging mode) if s.fakeStorage != nil { return s.fakeStorage[key] @@ -183,7 +183,7 @@ func (s *stateObject) GetCommittedState(db Database, key common.Hash) common.Has } // SetState updates a value in account storage. -func (s *stateObject) SetState(db Database, key, value common.Hash) { +func (s *stateObject) SetState(db StateDatabase, key, value common.Hash) { // If the fake storage is set, put the temporary state update here. if s.fakeStorage != nil { s.fakeStorage[key] = value @@ -270,7 +270,7 @@ func (s *stateObject) Address() common.Address { } // Code returns the contract code associated with this object, if any. -func (s *stateObject) Code(db Database) []byte { +func (s *stateObject) Code(db StateDatabase) []byte { if s.code != nil { return s.code } @@ -288,7 +288,7 @@ func (s *stateObject) Code(db Database) []byte { // CodeSize returns the size of the contract code associated with this object, // or zero if none. This method is an almost mirror of Code, but uses a cache // inside the database to avoid loading codes seen recently. -func (s *stateObject) CodeSize(db Database) int { +func (s *stateObject) CodeSize(db StateDatabase) int { if s.code != nil { return len(s.code) } diff --git a/statedb.go b/statedb.go index 1f97316..1d77038 100644 --- a/statedb.go +++ b/statedb.go @@ -46,7 +46,7 @@ type revision struct { // * Contracts // * Accounts type StateDB struct { - db Database + db StateDatabase hasher crypto.KeccakState // originBlockHash is the blockhash for the state we are working on top of @@ -89,7 +89,7 @@ type StateDB struct { } // New creates a new StateDB on the state for the provided blockHash -func New(blockHash common.Hash, db Database) (*StateDB, error) { +func New(blockHash common.Hash, db StateDatabase) (*StateDB, error) { sdb := &StateDB{ db: db, originBlockHash: blockHash, diff --git a/statedb_test.go b/statedb_test.go index 27c1dcd..7aa3835 100644 --- a/statedb_test.go +++ b/statedb_test.go @@ -7,7 +7,8 @@ import ( "strconv" "testing" - "github.com/jackc/pgx/v4/pgxpool" + "github.com/lib/pq" + "github.com/multiformats/go-multihash" "github.com/stretchr/testify/require" @@ -85,14 +86,18 @@ var ( RemovedNodeStorageCID = "bagmacgzayxjemamg64rtzet6pwznzrydydsqbnstzkbcoo337lmaixmfurya" ) -func TestSuite(t *testing.T) { +func TestPGXSuite(t *testing.T) { testConfig, err := getTestConfig() require.NoError(t, err) - pool, err := statedb.NewPGXPool(testCtx, testConfig) if err != nil { t.Fatal(err) } + driver, err := statedb.NewPGXDriverFromPool(context.Background(), pool) + if err != nil { + t.Fatal(err) + } + database := statedb.NewPostgresDB(driver) t.Cleanup(func() { tx, err := pool.Begin(testCtx) require.NoError(t, err) @@ -108,15 +113,15 @@ func TestSuite(t *testing.T) { } require.NoError(t, tx.Commit(testCtx)) }) - require.NoError(t, insertHeaderCID(pool, BlockHash.String(), BlockParentHash.String(), BlockNumber.Uint64())) - require.NoError(t, insertHeaderCID(pool, BlockHash2.String(), BlockHash.String(), BlockNumber2)) - require.NoError(t, insertHeaderCID(pool, BlockHash3.String(), BlockHash2.String(), BlockNumber3)) - require.NoError(t, insertHeaderCID(pool, BlockHash4.String(), BlockHash3.String(), BlockNumber4)) - require.NoError(t, insertHeaderCID(pool, NonCanonicalHash4.String(), BlockHash3.String(), BlockNumber4)) - require.NoError(t, insertHeaderCID(pool, BlockHash5.String(), BlockHash4.String(), BlockNumber5)) - require.NoError(t, insertHeaderCID(pool, NonCanonicalHash5.String(), NonCanonicalHash4.String(), BlockNumber5)) - require.NoError(t, insertHeaderCID(pool, BlockHash6.String(), BlockHash5.String(), BlockNumber6)) - require.NoError(t, insertStateCID(pool, stateModel{ + require.NoError(t, insertHeaderCID(database, BlockHash.String(), BlockParentHash.String(), BlockNumber.Uint64())) + require.NoError(t, insertHeaderCID(database, BlockHash2.String(), BlockHash.String(), BlockNumber2)) + require.NoError(t, insertHeaderCID(database, BlockHash3.String(), BlockHash2.String(), BlockNumber3)) + require.NoError(t, insertHeaderCID(database, BlockHash4.String(), BlockHash3.String(), BlockNumber4)) + require.NoError(t, insertHeaderCID(database, NonCanonicalHash4.String(), BlockHash3.String(), BlockNumber4)) + require.NoError(t, insertHeaderCID(database, BlockHash5.String(), BlockHash4.String(), BlockNumber5)) + require.NoError(t, insertHeaderCID(database, NonCanonicalHash5.String(), NonCanonicalHash4.String(), BlockNumber5)) + require.NoError(t, insertHeaderCID(database, BlockHash6.String(), BlockHash5.String(), BlockNumber6)) + require.NoError(t, insertStateCID(database, stateModel{ BlockNumber: BlockNumber.Uint64(), BlockHash: BlockHash.String(), LeafKey: AccountLeafKey.String(), @@ -128,7 +133,7 @@ func TestSuite(t *testing.T) { StorageRoot: Account.Root.String(), Removed: false, })) - require.NoError(t, insertStateCID(pool, stateModel{ + require.NoError(t, insertStateCID(database, stateModel{ BlockNumber: BlockNumber4, BlockHash: NonCanonicalHash4.String(), LeafKey: AccountLeafKey.String(), @@ -140,7 +145,7 @@ func TestSuite(t *testing.T) { StorageRoot: Account.Root.String(), Removed: false, })) - require.NoError(t, insertStateCID(pool, stateModel{ + require.NoError(t, insertStateCID(database, stateModel{ BlockNumber: BlockNumber5, BlockHash: BlockHash5.String(), LeafKey: AccountLeafKey.String(), @@ -148,7 +153,7 @@ func TestSuite(t *testing.T) { Diff: true, Removed: true, })) - require.NoError(t, insertStorageCID(pool, storageModel{ + require.NoError(t, insertStorageCID(database, storageModel{ BlockNumber: BlockNumber.Uint64(), BlockHash: BlockHash.String(), LeafKey: AccountLeafKey.String(), @@ -158,7 +163,7 @@ func TestSuite(t *testing.T) { Value: StoredValueRLP, Removed: false, })) - require.NoError(t, insertStorageCID(pool, storageModel{ + require.NoError(t, insertStorageCID(database, storageModel{ BlockNumber: BlockNumber2, BlockHash: BlockHash2.String(), LeafKey: AccountLeafKey.String(), @@ -168,7 +173,7 @@ func TestSuite(t *testing.T) { Value: []byte{}, Removed: true, })) - require.NoError(t, insertStorageCID(pool, storageModel{ + require.NoError(t, insertStorageCID(database, storageModel{ BlockNumber: BlockNumber3, BlockHash: BlockHash3.String(), LeafKey: AccountLeafKey.String(), @@ -178,7 +183,7 @@ func TestSuite(t *testing.T) { Value: StoredValueRLP2, Removed: false, })) - require.NoError(t, insertStorageCID(pool, storageModel{ + require.NoError(t, insertStorageCID(database, storageModel{ BlockNumber: BlockNumber4, BlockHash: NonCanonicalHash4.String(), LeafKey: AccountLeafKey.String(), @@ -188,9 +193,9 @@ func TestSuite(t *testing.T) { Value: NonCanonStoredValueRLP, Removed: false, })) - require.NoError(t, insertContractCode(pool)) + require.NoError(t, insertContractCode(database)) - db, err := statedb.NewStateDatabaseWithPool(pool) + db, err := statedb.NewStateDatabaseWithPgxPool(pool) require.NoError(t, err) t.Run("Database", func(t *testing.T) { @@ -303,7 +308,229 @@ func TestSuite(t *testing.T) { }) } -func insertHeaderCID(db *pgxpool.Pool, blockHash, parentHash string, blockNumber uint64) error { +func TestSQLXSuite(t *testing.T) { + testConfig, err := getTestConfig() + require.NoError(t, err) + pool, err := statedb.NewSQLXPool(testCtx, testConfig) + if err != nil { + t.Fatal(err) + } + driver, err := statedb.NewSQLXDriverFromPool(context.Background(), pool) + if err != nil { + t.Fatal(err) + } + database := statedb.NewPostgresDB(driver) + t.Cleanup(func() { + tx, err := pool.Begin() + require.NoError(t, err) + statements := []string{ + `DELETE FROM eth.header_cids`, + `DELETE FROM eth.state_cids`, + `DELETE FROM eth.storage_cids`, + `DELETE FROM ipld.blocks`, + } + for _, stm := range statements { + _, err = tx.Exec(stm) + require.NoErrorf(t, err, "Exec(`%s`)", stm) + } + require.NoError(t, tx.Commit()) + }) + require.NoError(t, insertHeaderCID(database, BlockHash.String(), BlockParentHash.String(), BlockNumber.Uint64())) + require.NoError(t, insertHeaderCID(database, BlockHash2.String(), BlockHash.String(), BlockNumber2)) + require.NoError(t, insertHeaderCID(database, BlockHash3.String(), BlockHash2.String(), BlockNumber3)) + require.NoError(t, insertHeaderCID(database, BlockHash4.String(), BlockHash3.String(), BlockNumber4)) + require.NoError(t, insertHeaderCID(database, NonCanonicalHash4.String(), BlockHash3.String(), BlockNumber4)) + require.NoError(t, insertHeaderCID(database, BlockHash5.String(), BlockHash4.String(), BlockNumber5)) + require.NoError(t, insertHeaderCID(database, NonCanonicalHash5.String(), NonCanonicalHash4.String(), BlockNumber5)) + require.NoError(t, insertHeaderCID(database, BlockHash6.String(), BlockHash5.String(), BlockNumber6)) + require.NoError(t, insertStateCID(database, stateModel{ + BlockNumber: BlockNumber.Uint64(), + BlockHash: BlockHash.String(), + LeafKey: AccountLeafKey.String(), + CID: AccountCID.String(), + Diff: true, + Balance: Account.Balance.Uint64(), + Nonce: Account.Nonce, + CodeHash: AccountCodeHash.String(), + StorageRoot: Account.Root.String(), + Removed: false, + })) + require.NoError(t, insertStateCID(database, stateModel{ + BlockNumber: BlockNumber4, + BlockHash: NonCanonicalHash4.String(), + LeafKey: AccountLeafKey.String(), + CID: AccountCID.String(), + Diff: true, + Balance: big.NewInt(123).Uint64(), + Nonce: Account.Nonce, + CodeHash: AccountCodeHash.String(), + StorageRoot: Account.Root.String(), + Removed: false, + })) + require.NoError(t, insertStateCID(database, stateModel{ + BlockNumber: BlockNumber5, + BlockHash: BlockHash5.String(), + LeafKey: AccountLeafKey.String(), + CID: RemovedNodeStateCID, + Diff: true, + Removed: true, + })) + require.NoError(t, insertStorageCID(database, storageModel{ + BlockNumber: BlockNumber.Uint64(), + BlockHash: BlockHash.String(), + LeafKey: AccountLeafKey.String(), + StorageLeafKey: StorageLeafKey.String(), + StorageCID: StorageCID.String(), + Diff: true, + Value: StoredValueRLP, + Removed: false, + })) + require.NoError(t, insertStorageCID(database, storageModel{ + BlockNumber: BlockNumber2, + BlockHash: BlockHash2.String(), + LeafKey: AccountLeafKey.String(), + StorageLeafKey: StorageLeafKey.String(), + StorageCID: RemovedNodeStorageCID, + Diff: true, + Value: []byte{}, + Removed: true, + })) + require.NoError(t, insertStorageCID(database, storageModel{ + BlockNumber: BlockNumber3, + BlockHash: BlockHash3.String(), + LeafKey: AccountLeafKey.String(), + StorageLeafKey: StorageLeafKey.String(), + StorageCID: StorageCID.String(), + Diff: true, + Value: StoredValueRLP2, + Removed: false, + })) + require.NoError(t, insertStorageCID(database, storageModel{ + BlockNumber: BlockNumber4, + BlockHash: NonCanonicalHash4.String(), + LeafKey: AccountLeafKey.String(), + StorageLeafKey: StorageLeafKey.String(), + StorageCID: StorageCID.String(), + Diff: true, + Value: NonCanonStoredValueRLP, + Removed: false, + })) + require.NoError(t, insertContractCode(database)) + + db, err := statedb.NewStateDatabaseWithSqlxPool(pool) + require.NoError(t, err) + + t.Run("Database", func(t *testing.T) { + size, err := db.ContractCodeSize(AccountCodeHash) + require.NoError(t, err) + require.Equal(t, len(AccountCode), size) + + code, err := db.ContractCode(AccountCodeHash) + require.NoError(t, err) + require.Equal(t, AccountCode, code) + + acct, err := db.StateAccount(AccountLeafKey, BlockHash) + require.NoError(t, err) + require.Equal(t, &Account, acct) + + acct2, err := db.StateAccount(AccountLeafKey, BlockHash2) + require.NoError(t, err) + require.Equal(t, &Account, acct2) + + acct3, err := db.StateAccount(AccountLeafKey, BlockHash3) + require.NoError(t, err) + require.Equal(t, &Account, acct3) + + // check that we don't get the non-canonical account + acct4, err := db.StateAccount(AccountLeafKey, BlockHash4) + require.NoError(t, err) + require.Equal(t, &Account, acct4) + + acct5, err := db.StateAccount(AccountLeafKey, BlockHash5) + require.NoError(t, err) + require.Nil(t, acct5) + + acct6, err := db.StateAccount(AccountLeafKey, BlockHash6) + require.NoError(t, err) + require.Nil(t, acct6) + + val, err := db.StorageValue(AccountLeafKey, StorageLeafKey, BlockHash) + require.NoError(t, err) + require.Equal(t, StoredValueRLP, val) + + val2, err := db.StorageValue(AccountLeafKey, StorageLeafKey, BlockHash2) + require.NoError(t, err) + require.Nil(t, val2) + + val3, err := db.StorageValue(AccountLeafKey, StorageLeafKey, BlockHash3) + require.NoError(t, err) + require.Equal(t, StoredValueRLP2, val3) + + // this checks that we don't get the non-canonical result + val4, err := db.StorageValue(AccountLeafKey, StorageLeafKey, BlockHash4) + require.NoError(t, err) + require.Equal(t, StoredValueRLP2, val4) + + // this checks that when the entire account was deleted, we return nil result for storage slot + val5, err := db.StorageValue(AccountLeafKey, StorageLeafKey, BlockHash5) + require.NoError(t, err) + require.Nil(t, val5) + + val6, err := db.StorageValue(AccountLeafKey, StorageLeafKey, BlockHash6) + require.NoError(t, err) + require.Nil(t, val6) + }) + + t.Run("StateDB", func(t *testing.T) { + sdb, err := statedb.New(BlockHash, db) + require.NoError(t, err) + + checkAccountUnchanged := func() { + require.Equal(t, Account.Balance, sdb.GetBalance(AccountAddress)) + require.Equal(t, Account.Nonce, sdb.GetNonce(AccountAddress)) + require.Equal(t, StoredValue, sdb.GetState(AccountAddress, StorageLeafKey)) + require.Equal(t, AccountCodeHash, sdb.GetCodeHash(AccountAddress)) + require.Equal(t, AccountCode, sdb.GetCode(AccountAddress)) + require.Equal(t, len(AccountCode), sdb.GetCodeSize(AccountAddress)) + } + + require.True(t, sdb.Exist(AccountAddress)) + checkAccountUnchanged() + + id := sdb.Snapshot() + + newStorage := crypto.Keccak256Hash([]byte{5, 4, 3, 2, 1}) + newCode := []byte{1, 3, 3, 7} + + sdb.SetBalance(AccountAddress, big.NewInt(300)) + sdb.AddBalance(AccountAddress, big.NewInt(200)) + sdb.SubBalance(AccountAddress, big.NewInt(100)) + sdb.SetNonce(AccountAddress, 42) + sdb.SetState(AccountAddress, StorageLeafKey, newStorage) + sdb.SetCode(AccountAddress, newCode) + + require.Equal(t, big.NewInt(400), sdb.GetBalance(AccountAddress)) + require.Equal(t, uint64(42), sdb.GetNonce(AccountAddress)) + require.Equal(t, newStorage, sdb.GetState(AccountAddress, StorageLeafKey)) + require.Equal(t, newCode, sdb.GetCode(AccountAddress)) + + sdb.AddSlotToAccessList(AccountAddress, StorageLeafKey) + require.True(t, sdb.AddressInAccessList(AccountAddress)) + hasAddr, hasSlot := sdb.SlotInAccessList(AccountAddress, StorageLeafKey) + require.True(t, hasAddr) + require.True(t, hasSlot) + + sdb.RevertToSnapshot(id) + + checkAccountUnchanged() + require.False(t, sdb.AddressInAccessList(AccountAddress)) + hasAddr, hasSlot = sdb.SlotInAccessList(AccountAddress, StorageLeafKey) + require.False(t, hasAddr) + require.False(t, hasSlot) + }) +} + +func insertHeaderCID(db statedb.Database, blockHash, parentHash string, blockNumber uint64) error { cid, err := util.Keccak256ToCid(ipld.MEthHeader, common.HexToHash(blockHash).Bytes()) if err != nil { return err @@ -329,7 +556,7 @@ func insertHeaderCID(db *pgxpool.Pool, blockHash, parentHash string, blockNumber blockHash, parentHash, cid.String(), - 0, []string{}, 0, + 0, pq.StringArray([]string{}), 0, Header.Root.String(), Header.TxHash.String(), Header.ReceiptHash.String(), @@ -354,7 +581,7 @@ type stateModel struct { Removed bool } -func insertStateCID(db *pgxpool.Pool, cidModel stateModel) error { +func insertStateCID(db statedb.Database, cidModel stateModel) error { sql := `INSERT INTO eth.state_cids ( block_number, header_id, @@ -393,7 +620,7 @@ type storageModel struct { Removed bool } -func insertStorageCID(db *pgxpool.Pool, cidModel storageModel) error { +func insertStorageCID(db statedb.Database, cidModel storageModel) error { sql := `INSERT INTO eth.storage_cids ( block_number, header_id, @@ -417,9 +644,9 @@ func insertStorageCID(db *pgxpool.Pool, cidModel storageModel) error { return err } -func insertContractCode(db *pgxpool.Pool) error { +func insertContractCode(db statedb.Database) error { sql := `INSERT INTO ipld.blocks (block_number, key, data) VALUES ($1, $2, $3)` - _, err := db.Exec(testCtx, sql, BlockNumber.Uint64(), AccountCodeCID, AccountCode) + _, err := db.Exec(testCtx, sql, BlockNumber.Uint64(), AccountCodeCID.String(), AccountCode) return err }