Copy package from geth fork

This commit is contained in:
2023-07-11 20:10:38 +08:00
parent 88a2405805
commit bb00c46735
111 changed files with 18496 additions and 1 deletions
+140
View File
@@ -0,0 +1,140 @@
// VulcanizeDB
// Copyright © 2021 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package postgres
import (
"fmt"
"os"
"strconv"
"strings"
"time"
"github.com/ethereum/go-ethereum/statediff/indexer/shared"
)
// DriverType to explicitly type the kind of sql driver we are using
type DriverType string
const (
PGX DriverType = "PGX"
SQLX DriverType = "SQLX"
Unknown DriverType = "Unknown"
)
// Env variables
const (
DATABASE_NAME = "DATABASE_NAME"
DATABASE_HOSTNAME = "DATABASE_HOSTNAME"
DATABASE_PORT = "DATABASE_PORT"
DATABASE_USER = "DATABASE_USER"
DATABASE_PASSWORD = "DATABASE_PASSWORD"
)
// ResolveDriverType resolves a DriverType from a provided string
func ResolveDriverType(str string) (DriverType, error) {
switch strings.ToLower(str) {
case "pgx", "pgxpool":
return PGX, nil
case "sqlx":
return SQLX, nil
default:
return Unknown, fmt.Errorf("unrecognized driver type string: %s", str)
}
}
// TestConfig specifies default parameters for connecting to a testing DB
var TestConfig = Config{
Hostname: "localhost",
Port: 8077,
DatabaseName: "cerc_testing",
Username: "vdbm",
Password: "password",
Driver: SQLX,
}
// Config holds params for a Postgres db
type Config struct {
// conn string params
Hostname string
Port int
DatabaseName string
Username string
Password string
// conn settings
MaxConns int
MaxIdle int
MinConns int
MaxConnIdleTime time.Duration
MaxConnLifetime time.Duration
ConnTimeout time.Duration
LogStatements bool
// node info params
ID string
ClientName string
// driver type
Driver DriverType
// toggle on/off upserts
Upsert bool
// toggle on/off CopyFrom
CopyFrom bool
}
// Type satisfies interfaces.Config
func (c Config) Type() shared.DBType {
return shared.POSTGRES
}
// DbConnectionString constructs and returns the connection string from the config
func (c Config) DbConnectionString() string {
if len(c.Username) > 0 && len(c.Password) > 0 {
return fmt.Sprintf("postgresql://%s:%s@%s:%d/%s?sslmode=disable",
c.Username, c.Password, c.Hostname, c.Port, c.DatabaseName)
}
if len(c.Username) > 0 && len(c.Password) == 0 {
return fmt.Sprintf("postgresql://%s@%s:%d/%s?sslmode=disable",
c.Username, c.Hostname, c.Port, c.DatabaseName)
}
return fmt.Sprintf("postgresql://%s:%d/%s?sslmode=disable", c.Hostname, c.Port, c.DatabaseName)
}
func (c Config) WithEnv() (Config, error) {
if val := os.Getenv(DATABASE_NAME); val != "" {
c.DatabaseName = val
}
if val := os.Getenv(DATABASE_HOSTNAME); val != "" {
c.Hostname = val
}
if val := os.Getenv(DATABASE_PORT); val != "" {
port, err := strconv.Atoi(val)
if err != nil {
return c, err
}
c.Port = port
}
if val := os.Getenv(DATABASE_USER); val != "" {
c.Username = val
}
if val := os.Getenv(DATABASE_PASSWORD); val != "" {
c.Password = val
}
return c, nil
}
+126
View File
@@ -0,0 +1,126 @@
// VulcanizeDB
// Copyright © 2021 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package postgres
import (
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
"github.com/ethereum/go-ethereum/statediff/indexer/shared/schema"
)
var _ sql.Database = &DB{}
const (
createNodeStm = `INSERT INTO nodes (genesis_block, network_id, node_id, client_name, chain_id) VALUES ($1, $2, $3, $4, $5)
ON CONFLICT (node_id) DO NOTHING`
)
// NewPostgresDB returns a postgres.DB using the provided driver
func NewPostgresDB(driver sql.Driver, upsert bool) *DB {
return &DB{upsert, driver}
}
// DB implements sql.Database using a configured driver and Postgres statement syntax
type DB struct {
upsert bool
sql.Driver
}
// InsertHeaderStm satisfies the sql.Statements interface
// Stm == Statement
func (db *DB) InsertHeaderStm() string {
return schema.TableHeader.ToInsertStatement(db.upsert)
}
// InsertUncleStm satisfies the sql.Statements interface
func (db *DB) InsertUncleStm() string {
return schema.TableUncle.ToInsertStatement(db.upsert)
}
// InsertTxStm satisfies the sql.Statements interface
func (db *DB) InsertTxStm() string {
return schema.TableTransaction.ToInsertStatement(db.upsert)
}
// InsertRctStm satisfies the sql.Statements interface
func (db *DB) InsertRctStm() string {
return schema.TableReceipt.ToInsertStatement(db.upsert)
}
// InsertLogStm satisfies the sql.Statements interface
func (db *DB) InsertLogStm() string {
return schema.TableLog.ToInsertStatement(db.upsert)
}
// InsertStateStm satisfies the sql.Statements interface
func (db *DB) InsertStateStm() string {
return schema.TableStateNode.ToInsertStatement(db.upsert)
}
// InsertStorageStm satisfies the sql.Statements interface
func (db *DB) InsertStorageStm() string {
return schema.TableStorageNode.ToInsertStatement(db.upsert)
}
// InsertIPLDStm satisfies the sql.Statements interface
func (db *DB) InsertIPLDStm() string {
return schema.TableIPLDBlock.ToInsertStatement(db.upsert)
}
// InsertIPLDsStm satisfies the sql.Statements interface
func (db *DB) InsertIPLDsStm() string {
return `INSERT INTO ipld.blocks (block_number, key, data) VALUES (unnest($1::BIGINT[]), unnest($2::TEXT[]), unnest($3::BYTEA[])) ON CONFLICT DO NOTHING`
}
func (db *DB) LogTableName() []string {
return []string{"eth", "log_cids"}
}
func (db *DB) LogColumnNames() []string {
return []string{"block_number", "header_id", "cid", "rct_id", "address", "index", "topic0", "topic1", "topic2", "topic3"}
}
func (db *DB) RctTableName() []string {
return []string{"eth", "receipt_cids"}
}
func (db *DB) RctColumnNames() []string {
return []string{"block_number", "header_id", "tx_id", "cid", "contract", "post_state", "post_status"}
}
func (db *DB) StateTableName() []string {
return []string{"eth", "state_cids"}
}
func (db *DB) StateColumnNames() []string {
return []string{"block_number", "header_id", "state_leaf_key", "cid", "diff", "balance", "nonce", "code_hash", "storage_root", "removed"}
}
func (db *DB) StorageTableName() []string {
return []string{"eth", "storage_cids"}
}
func (db *DB) StorageColumnNames() []string {
return []string{"block_number", "header_id", "state_leaf_key", "storage_leaf_key", "cid", "diff", "val", "removed"}
}
func (db *DB) TxTableName() []string {
return []string{"eth", "transaction_cids"}
}
func (db *DB) TxColumnNames() []string {
return []string{"block_number", "header_id", "tx_hash", "cid", "dst", "src", "index", "tx_type", "value"}
}
+38
View File
@@ -0,0 +1,38 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package postgres
import (
"fmt"
)
const (
DbConnectionFailedMsg = "db connection failed"
SettingNodeFailedMsg = "unable to set db node"
)
func ErrDBConnectionFailed(connectErr error) error {
return formatError(DbConnectionFailedMsg, connectErr)
}
func ErrUnableToSetNode(setErr error) error {
return formatError(SettingNodeFailedMsg, setErr)
}
func formatError(msg string, err error) error {
return fmt.Errorf("%s: %w", msg, err)
}
@@ -0,0 +1,61 @@
// Copyright © 2023 Cerc
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package postgres
import (
"context"
"github.com/ethereum/go-ethereum/log"
"github.com/jackc/pgx/v4"
)
type LogAdapter struct {
l log.Logger
}
func NewLogAdapter(l log.Logger) *LogAdapter {
return &LogAdapter{l: l}
}
func (l *LogAdapter) Log(ctx context.Context, level pgx.LogLevel, msg string, data map[string]interface{}) {
var logger log.Logger
if data != nil {
var args = make([]interface{}, 0)
for key, value := range data {
if value != nil {
args = append(args, key, value)
}
}
logger = l.l.New(args...)
} else {
logger = l.l
}
switch level {
case pgx.LogLevelTrace:
logger.Trace(msg)
case pgx.LogLevelDebug:
logger.Debug(msg)
case pgx.LogLevelInfo:
logger.Info(msg)
case pgx.LogLevelWarn:
logger.Warn(msg)
case pgx.LogLevelError:
logger.Error(msg)
default:
logger.New("INVALID_PGX_LOG_LEVEL", level).Error(msg)
}
}
+256
View File
@@ -0,0 +1,256 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package postgres
import (
"context"
"time"
"github.com/ethereum/go-ethereum/log"
"github.com/georgysavva/scany/pgxscan"
"github.com/jackc/pgconn"
"github.com/jackc/pgx/v4"
"github.com/jackc/pgx/v4/pgxpool"
"github.com/ethereum/go-ethereum/statediff/indexer/database/metrics"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
"github.com/ethereum/go-ethereum/statediff/indexer/node"
)
// PGXDriver driver, implements sql.Driver
type PGXDriver struct {
ctx context.Context
pool *pgxpool.Pool
nodeInfo node.Info
nodeID string
config Config
}
// ConnectPGX initializes and returns a PGX connection pool
func ConnectPGX(ctx context.Context, config Config) (*pgxpool.Pool, error) {
pgConf, err := MakeConfig(config)
if err != nil {
return nil, err
}
return pgxpool.ConnectConfig(ctx, pgConf)
}
// NewPGXDriver returns a new pgx driver
// it initializes the connection pool and creates the node info table
func NewPGXDriver(ctx context.Context, config Config, node node.Info) (*PGXDriver, error) {
dbPool, err := ConnectPGX(ctx, config)
if err != nil {
return nil, ErrDBConnectionFailed(err)
}
pg := &PGXDriver{ctx: ctx, pool: dbPool, nodeInfo: node, config: config}
nodeErr := pg.createNode()
if nodeErr != nil {
return &PGXDriver{}, ErrUnableToSetNode(nodeErr)
}
return pg, nil
}
// MakeConfig creates a pgxpool.Config from the provided Config
func MakeConfig(config Config) (*pgxpool.Config, error) {
conf, err := pgxpool.ParseConfig("")
if err != nil {
return nil, err
}
//conf.ConnConfig.BuildStatementCache = nil
conf.ConnConfig.Config.Host = config.Hostname
conf.ConnConfig.Config.Port = uint16(config.Port)
conf.ConnConfig.Config.Database = config.DatabaseName
conf.ConnConfig.Config.User = config.Username
conf.ConnConfig.Config.Password = config.Password
if config.ConnTimeout != 0 {
conf.ConnConfig.Config.ConnectTimeout = config.ConnTimeout
}
if config.MaxConns != 0 {
conf.MaxConns = int32(config.MaxConns)
}
if config.MinConns != 0 {
conf.MinConns = int32(config.MinConns)
}
if config.MaxConnLifetime != 0 {
conf.MaxConnLifetime = config.MaxConnLifetime
}
if config.MaxConnIdleTime != 0 {
conf.MaxConnIdleTime = config.MaxConnIdleTime
}
if config.LogStatements {
conf.ConnConfig.Logger = NewLogAdapter(log.New())
}
return conf, nil
}
func (pgx *PGXDriver) createNode() error {
_, err := pgx.pool.Exec(
pgx.ctx,
createNodeStm,
pgx.nodeInfo.GenesisBlock, pgx.nodeInfo.NetworkID,
pgx.nodeInfo.ID, pgx.nodeInfo.ClientName,
pgx.nodeInfo.ChainID)
if err != nil {
return ErrUnableToSetNode(err)
}
pgx.nodeID = pgx.nodeInfo.ID
return nil
}
// QueryRow satisfies sql.Database
func (pgx *PGXDriver) QueryRow(ctx context.Context, sql string, args ...interface{}) sql.ScannableRow {
return pgx.pool.QueryRow(ctx, sql, args...)
}
// Exec satisfies sql.Database
func (pgx *PGXDriver) Exec(ctx context.Context, sql string, args ...interface{}) (sql.Result, error) {
res, err := pgx.pool.Exec(ctx, sql, args...)
return resultWrapper{ct: res}, err
}
// Select satisfies sql.Database
func (pgx *PGXDriver) Select(ctx context.Context, dest interface{}, query string, args ...interface{}) error {
return pgxscan.Select(ctx, pgx.pool, dest, query, args...)
}
// Get satisfies sql.Database
func (pgx *PGXDriver) Get(ctx context.Context, dest interface{}, query string, args ...interface{}) error {
return pgxscan.Get(ctx, pgx.pool, dest, query, args...)
}
// Begin satisfies sql.Database
func (pgx *PGXDriver) Begin(ctx context.Context) (sql.Tx, error) {
tx, err := pgx.pool.Begin(ctx)
if err != nil {
return nil, err
}
return pgxTxWrapper{tx: tx}, nil
}
func (pgx *PGXDriver) Stats() metrics.DbStats {
stats := pgx.pool.Stat()
return pgxStatsWrapper{stats: stats}
}
// NodeID satisfies sql.Database
func (pgx *PGXDriver) NodeID() string {
return pgx.nodeID
}
// Close satisfies sql.Database/io.Closer
func (pgx *PGXDriver) Close() error {
pgx.pool.Close()
return nil
}
// Context satisfies sql.Database
func (pgx *PGXDriver) Context() context.Context {
return pgx.ctx
}
// HasCopy satisfies sql.Database
func (pgx *PGXDriver) UseCopyFrom() bool {
return pgx.config.CopyFrom
}
type resultWrapper struct {
ct pgconn.CommandTag
}
// RowsAffected satisfies sql.Result
func (r resultWrapper) RowsAffected() (int64, error) {
return r.ct.RowsAffected(), nil
}
type pgxStatsWrapper struct {
stats *pgxpool.Stat
}
// MaxOpen satisfies metrics.DbStats
func (s pgxStatsWrapper) MaxOpen() int64 {
return int64(s.stats.MaxConns())
}
// Open satisfies metrics.DbStats
func (s pgxStatsWrapper) Open() int64 {
return int64(s.stats.TotalConns())
}
// InUse satisfies metrics.DbStats
func (s pgxStatsWrapper) InUse() int64 {
return int64(s.stats.AcquiredConns())
}
// Idle satisfies metrics.DbStats
func (s pgxStatsWrapper) Idle() int64 {
return int64(s.stats.IdleConns())
}
// WaitCount satisfies metrics.DbStats
func (s pgxStatsWrapper) WaitCount() int64 {
return s.stats.EmptyAcquireCount()
}
// WaitDuration satisfies metrics.DbStats
func (s pgxStatsWrapper) WaitDuration() time.Duration {
return s.stats.AcquireDuration()
}
// MaxIdleClosed satisfies metrics.DbStats
func (s pgxStatsWrapper) MaxIdleClosed() int64 {
// this stat isn't supported by pgxpool, but we don't want to panic
return 0
}
// MaxLifetimeClosed satisfies metrics.DbStats
func (s pgxStatsWrapper) MaxLifetimeClosed() int64 {
return s.stats.CanceledAcquireCount()
}
type pgxTxWrapper struct {
tx pgx.Tx
}
// QueryRow satisfies sql.Tx
func (t pgxTxWrapper) QueryRow(ctx context.Context, sql string, args ...interface{}) sql.ScannableRow {
return t.tx.QueryRow(ctx, sql, args...)
}
// Exec satisfies sql.Tx
func (t pgxTxWrapper) Exec(ctx context.Context, sql string, args ...interface{}) (sql.Result, error) {
res, err := t.tx.Exec(ctx, sql, args...)
return resultWrapper{ct: res}, err
}
// Commit satisfies sql.Tx
func (t pgxTxWrapper) Commit(ctx context.Context) error {
return t.tx.Commit(ctx)
}
// Rollback satisfies sql.Tx
func (t pgxTxWrapper) Rollback(ctx context.Context) error {
return t.tx.Rollback(ctx)
}
func (t pgxTxWrapper) CopyFrom(ctx context.Context, tableName []string, columnNames []string, rows [][]interface{}) (int64, error) {
return t.tx.CopyFrom(ctx, tableName, columnNames, pgx.CopyFromRows(rows))
}
+121
View File
@@ -0,0 +1,121 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package postgres_test
import (
"context"
"fmt"
"math/big"
"strings"
"testing"
"github.com/jackc/pgx/v4/pgxpool"
"github.com/stretchr/testify/require"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
"github.com/ethereum/go-ethereum/statediff/indexer/node"
)
var (
pgConfig, _ = postgres.MakeConfig(postgres.TestConfig)
ctx = context.Background()
)
func expectContainsSubstring(t *testing.T, full string, sub string) {
if !strings.Contains(full, sub) {
t.Fatalf("Expected \"%v\" to contain substring \"%v\"\n", full, sub)
}
}
func TestPostgresPGX(t *testing.T) {
t.Run("connects to the sql", func(t *testing.T) {
dbPool, err := pgxpool.ConnectConfig(context.Background(), pgConfig)
if err != nil {
t.Fatalf("failed to connect to db with connection string: %s err: %v", pgConfig.ConnString(), err)
}
if dbPool == nil {
t.Fatal("DB pool is nil")
}
dbPool.Close()
})
t.Run("serializes big.Int to db", func(t *testing.T) {
// postgres driver doesn't support go big.Int type
// various casts in golang uint64, int64, overflow for
// transaction value (in wei) even though
// postgres numeric can handle an arbitrary
// sized int, so use string representation of big.Int
// and cast on insert
dbPool, err := pgxpool.ConnectConfig(context.Background(), pgConfig)
if err != nil {
t.Fatalf("failed to connect to db with connection string: %s err: %v", pgConfig.ConnString(), err)
}
defer dbPool.Close()
bi := new(big.Int)
bi.SetString("34940183920000000000", 10)
require.Equal(t, "34940183920000000000", bi.String())
defer dbPool.Exec(ctx, `DROP TABLE IF EXISTS example`)
_, err = dbPool.Exec(ctx, "CREATE TABLE example ( id INTEGER, data NUMERIC )")
if err != nil {
t.Fatal(err)
}
sqlStatement := `
INSERT INTO example (id, data)
VALUES (1, cast($1 AS NUMERIC))`
_, err = dbPool.Exec(ctx, sqlStatement, bi.String())
if err != nil {
t.Fatal(err)
}
var data string
err = dbPool.QueryRow(ctx, `SELECT cast(data AS TEXT) FROM example WHERE id = 1`).Scan(&data)
if err != nil {
t.Fatal(err)
}
require.Equal(t, data, bi.String())
actual := new(big.Int)
actual.SetString(data, 10)
require.Equal(t, bi, actual)
})
t.Run("throws error when can't connect to the database", func(t *testing.T) {
goodInfo := node.Info{GenesisBlock: "GENESIS", NetworkID: "1", ID: "x123", ClientName: "geth"}
_, err := postgres.NewPGXDriver(ctx, postgres.Config{}, goodInfo)
if err == nil {
t.Fatal("Expected an error")
}
expectContainsSubstring(t, err.Error(), postgres.DbConnectionFailedMsg)
})
t.Run("throws error when can't create node", func(t *testing.T) {
badHash := fmt.Sprintf("x %s", strings.Repeat("1", 100))
badInfo := node.Info{GenesisBlock: badHash, NetworkID: "1", ID: "x123", ClientName: "geth"}
_, err := postgres.NewPGXDriver(ctx, postgres.TestConfig, badInfo)
if err == nil {
t.Fatal("Expected an error")
}
expectContainsSubstring(t, err.Error(), postgres.SettingNodeFailedMsg)
})
}
@@ -0,0 +1,33 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package postgres_test
import (
"fmt"
"os"
"github.com/ethereum/go-ethereum/log"
)
func init() {
if os.Getenv("MODE") != "statediff" {
fmt.Println("Skipping statediff test")
os.Exit(0)
}
log.Root().SetHandler(log.DiscardHandler())
}
+210
View File
@@ -0,0 +1,210 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package postgres
import (
"context"
coresql "database/sql"
"errors"
"time"
"github.com/jmoiron/sqlx"
"github.com/ethereum/go-ethereum/statediff/indexer/database/metrics"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
"github.com/ethereum/go-ethereum/statediff/indexer/node"
)
// SQLXDriver driver, implements sql.Driver
type SQLXDriver struct {
ctx context.Context
db *sqlx.DB
nodeInfo node.Info
nodeID string
}
// ConnectSQLX initializes and returns a SQLX connection pool for postgres
func ConnectSQLX(ctx context.Context, config Config) (*sqlx.DB, error) {
db, err := sqlx.ConnectContext(ctx, "postgres", config.DbConnectionString())
if err != nil {
return nil, ErrDBConnectionFailed(err)
}
if config.MaxConns > 0 {
db.SetMaxOpenConns(config.MaxConns)
}
if config.MaxConnLifetime > 0 {
db.SetConnMaxLifetime(config.MaxConnLifetime)
}
db.SetMaxIdleConns(config.MaxIdle)
return db, nil
}
// NewSQLXDriver returns a new sqlx driver for Postgres
// it initializes the connection pool and creates the node info table
func NewSQLXDriver(ctx context.Context, config Config, node node.Info) (*SQLXDriver, error) {
db, err := ConnectSQLX(ctx, config)
if err != nil {
return nil, err
}
driver := &SQLXDriver{ctx: ctx, db: db, nodeInfo: node}
if err := driver.createNode(); err != nil {
return nil, err
}
return driver, nil
}
func (driver *SQLXDriver) createNode() error {
_, err := driver.db.Exec(
createNodeStm,
driver.nodeInfo.GenesisBlock,
driver.nodeInfo.NetworkID,
driver.nodeInfo.ID,
driver.nodeInfo.ClientName,
driver.nodeInfo.ChainID)
if err != nil {
return ErrUnableToSetNode(err)
}
driver.nodeID = driver.nodeInfo.ID
return nil
}
// QueryRow satisfies sql.Database
func (driver *SQLXDriver) QueryRow(_ context.Context, sql string, args ...interface{}) sql.ScannableRow {
return driver.db.QueryRowx(sql, args...)
}
// Exec satisfies sql.Database
func (driver *SQLXDriver) Exec(_ context.Context, sql string, args ...interface{}) (sql.Result, error) {
return driver.db.Exec(sql, args...)
}
// Select satisfies sql.Database
func (driver *SQLXDriver) Select(_ context.Context, dest interface{}, query string, args ...interface{}) error {
return driver.db.Select(dest, query, args...)
}
// Get satisfies sql.Database
func (driver *SQLXDriver) Get(_ context.Context, dest interface{}, query string, args ...interface{}) error {
return driver.db.Get(dest, query, args...)
}
// Begin satisfies sql.Database
func (driver *SQLXDriver) Begin(_ context.Context) (sql.Tx, error) {
tx, err := driver.db.Beginx()
if err != nil {
return nil, err
}
return sqlxTxWrapper{tx: tx}, nil
}
func (driver *SQLXDriver) Stats() metrics.DbStats {
stats := driver.db.Stats()
return sqlxStatsWrapper{stats: stats}
}
// NodeID satisfies sql.Database
func (driver *SQLXDriver) NodeID() string {
return driver.nodeID
}
// Close satisfies sql.Database/io.Closer
func (driver *SQLXDriver) Close() error {
return driver.db.Close()
}
// Context satisfies sql.Database
func (driver *SQLXDriver) Context() context.Context {
return driver.ctx
}
// HasCopy satisfies sql.Database
func (driver *SQLXDriver) UseCopyFrom() bool {
// sqlx does not currently support COPY.
return false
}
type sqlxStatsWrapper struct {
stats coresql.DBStats
}
// MaxOpen satisfies metrics.DbStats
func (s sqlxStatsWrapper) MaxOpen() int64 {
return int64(s.stats.MaxOpenConnections)
}
// Open satisfies metrics.DbStats
func (s sqlxStatsWrapper) Open() int64 {
return int64(s.stats.OpenConnections)
}
// InUse satisfies metrics.DbStats
func (s sqlxStatsWrapper) InUse() int64 {
return int64(s.stats.InUse)
}
// Idle satisfies metrics.DbStats
func (s sqlxStatsWrapper) Idle() int64 {
return int64(s.stats.Idle)
}
// WaitCount satisfies metrics.DbStats
func (s sqlxStatsWrapper) WaitCount() int64 {
return s.stats.WaitCount
}
// WaitDuration satisfies metrics.DbStats
func (s sqlxStatsWrapper) WaitDuration() time.Duration {
return s.stats.WaitDuration
}
// MaxIdleClosed satisfies metrics.DbStats
func (s sqlxStatsWrapper) MaxIdleClosed() int64 {
return s.stats.MaxIdleClosed
}
// MaxLifetimeClosed satisfies metrics.DbStats
func (s sqlxStatsWrapper) MaxLifetimeClosed() int64 {
return s.stats.MaxLifetimeClosed
}
type sqlxTxWrapper struct {
tx *sqlx.Tx
}
// QueryRow satisfies sql.Tx
func (t sqlxTxWrapper) QueryRow(ctx context.Context, sql string, args ...interface{}) sql.ScannableRow {
return t.tx.QueryRowx(sql, args...)
}
// Exec satisfies sql.Tx
func (t sqlxTxWrapper) Exec(ctx context.Context, sql string, args ...interface{}) (sql.Result, error) {
return t.tx.Exec(sql, args...)
}
// Commit satisfies sql.Tx
func (t sqlxTxWrapper) Commit(ctx context.Context) error {
return t.tx.Commit()
}
// Rollback satisfies sql.Tx
func (t sqlxTxWrapper) Rollback(ctx context.Context) error {
return t.tx.Rollback()
}
func (t sqlxTxWrapper) CopyFrom(ctx context.Context, tableName []string, columnNames []string, rows [][]interface{}) (int64, error) {
return 0, errors.New("Unsupported Operation")
}
+119
View File
@@ -0,0 +1,119 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package postgres_test
import (
"fmt"
"math/big"
"strings"
"testing"
"github.com/jmoiron/sqlx"
_ "github.com/lib/pq"
"github.com/stretchr/testify/require"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
"github.com/ethereum/go-ethereum/statediff/indexer/node"
)
func TestPostgresSQLX(t *testing.T) {
var sqlxdb *sqlx.DB
t.Run("connects to the database", func(t *testing.T) {
var err error
connStr := postgres.TestConfig.DbConnectionString()
sqlxdb, err = sqlx.Connect("postgres", connStr)
if err != nil {
t.Fatalf("failed to connect to db with connection string: %s err: %v", connStr, err)
}
if sqlxdb == nil {
t.Fatal("DB is nil")
}
err = sqlxdb.Close()
if err != nil {
t.Fatal(err)
}
})
t.Run("serializes big.Int to db", func(t *testing.T) {
// postgres driver doesn't support go big.Int type
// various casts in golang uint64, int64, overflow for
// transaction value (in wei) even though
// postgres numeric can handle an arbitrary
// sized int, so use string representation of big.Int
// and cast on insert
connStr := postgres.TestConfig.DbConnectionString()
db, err := sqlx.Connect("postgres", connStr)
if err != nil {
t.Fatal(err)
}
defer db.Close()
bi := new(big.Int)
bi.SetString("34940183920000000000", 10)
require.Equal(t, "34940183920000000000", bi.String())
defer db.Exec(`DROP TABLE IF EXISTS example`)
_, err = db.Exec("CREATE TABLE example ( id INTEGER, data NUMERIC )")
if err != nil {
t.Fatal(err)
}
sqlStatement := `
INSERT INTO example (id, data)
VALUES (1, cast($1 AS NUMERIC))`
_, err = db.Exec(sqlStatement, bi.String())
if err != nil {
t.Fatal(err)
}
var data string
err = db.QueryRow(`SELECT data FROM example WHERE id = 1`).Scan(&data)
if err != nil {
t.Fatal(err)
}
require.Equal(t, data, bi.String())
actual := new(big.Int)
actual.SetString(data, 10)
require.Equal(t, bi, actual)
})
t.Run("throws error when can't connect to the database", func(t *testing.T) {
goodInfo := node.Info{GenesisBlock: "GENESIS", NetworkID: "1", ID: "x123", ClientName: "geth"}
_, err := postgres.NewSQLXDriver(ctx, postgres.Config{}, goodInfo)
if err == nil {
t.Fatal("Expected an error")
}
expectContainsSubstring(t, err.Error(), postgres.DbConnectionFailedMsg)
})
t.Run("throws error when can't create node", func(t *testing.T) {
badHash := fmt.Sprintf("x %s", strings.Repeat("1", 100))
badInfo := node.Info{GenesisBlock: badHash, NetworkID: "1", ID: "x123", ClientName: "geth"}
_, err := postgres.NewSQLXDriver(ctx, postgres.TestConfig, badInfo)
if err == nil {
t.Fatal("Expected an error")
}
expectContainsSubstring(t, err.Error(), postgres.SettingNodeFailedMsg)
})
}
@@ -0,0 +1,44 @@
// VulcanizeDB
// Copyright © 2021 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package postgres
import (
"context"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
"github.com/ethereum/go-ethereum/statediff/indexer/node"
)
// SetupSQLXDB is used to setup a sqlx db for tests
func SetupSQLXDB() (sql.Database, error) {
conf := TestConfig
conf.MaxIdle = 0
driver, err := NewSQLXDriver(context.Background(), conf, node.Info{})
if err != nil {
return nil, err
}
return NewPostgresDB(driver, false), nil
}
// SetupPGXDB is used to setup a pgx db for tests
func SetupPGXDB(config Config) (sql.Database, error) {
driver, err := NewPGXDriver(context.Background(), config, node.Info{})
if err != nil {
return nil, err
}
return NewPostgresDB(driver, false), nil
}