cerc-io/plugeth-statediff
11
0
Fork 0
Code Issues 3 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

Initial plugin implementation

Browse Source 
* refactor packages, flags, subscriptions
* DRY refactor builder tests
* use mockgen to generate mocks
* update README
* MODE=statediff no longer needed for unit tests
* simplify func names, clean up metrics
* move write params to service field
* sql indexer: confirm quit after ipld cache reset
  prevents negative waitgroup panic
* don't let TotalDifficulty become nil
* use forked plugeth, plugeth-utils for now
This commit is contained in:
roysc
2023-07-14 12:56:36 +08:00
parent bb00c46735
commit ebc2eb37e7
89 changed files with 3742 additions and 3160 deletions
Download Patch File Download Diff File Expand all files Collapse all files
indexer/database/sql/batch_tx.go
+7 -6
View File
@@ -23,9 +23,9 @@ import (
"github.com/lib/pq"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
"github.com/ethereum/go-ethereum/statediff/indexer/models"
"github.com/cerc-io/plugeth-statediff/indexer/ipld"
"github.com/cerc-io/plugeth-statediff/indexer/models"
"github.com/cerc-io/plugeth-statediff/utils/log"
)
const startingCacheCapacity = 1024 * 24
@@ -36,7 +36,7 @@ type BatchTx struct {
ctx context.Context
dbtx Tx
stm string
quit chan struct{}
quit chan (chan<- struct{})
iplds chan models.IPLDModel
ipldCache models.IPLDBatch
removedCacheFlag *uint32
@@ -81,8 +81,9 @@ func (tx *BatchTx) cache() {
tx.ipldCache.Keys = append(tx.ipldCache.Keys, i.Key)
tx.ipldCache.Values = append(tx.ipldCache.Values, i.Data)
tx.cacheWg.Done()
case <-tx.quit:
case confirm := <-tx.quit:
tx.ipldCache = models.IPLDBatch{}
confirm <- struct{}{}
return
}
}
@@ -121,6 +122,6 @@ func (tx *BatchTx) cacheRemoved(key string, value []byte) {
// rollback sql transaction and log any error
func rollback(ctx context.Context, tx Tx) {
if err := tx.Rollback(ctx); err != nil {
log.Error(err.Error())
log.Error("error during rollback", "error", err)
}
}
indexer/database/sql/indexer.go
+29 -27
View File
@@ -20,27 +20,26 @@
package sql
import (
"bytes"
"context"
"fmt"
"math/big"
"time"
"github.com/multiformats/go-multihash"
"github.com/ethereum/go-ethereum/common"
core "github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/metrics"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/rlp"
metrics2 "github.com/ethereum/go-ethereum/statediff/indexer/database/metrics"
"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
"github.com/ethereum/go-ethereum/statediff/indexer/ipld"
"github.com/ethereum/go-ethereum/statediff/indexer/models"
"github.com/ethereum/go-ethereum/statediff/indexer/shared"
sdtypes "github.com/ethereum/go-ethereum/statediff/types"
"github.com/multiformats/go-multihash"
metrics2 "github.com/cerc-io/plugeth-statediff/indexer/database/metrics"
"github.com/cerc-io/plugeth-statediff/indexer/interfaces"
"github.com/cerc-io/plugeth-statediff/indexer/ipld"
"github.com/cerc-io/plugeth-statediff/indexer/models"
"github.com/cerc-io/plugeth-statediff/indexer/shared"
sdtypes "github.com/cerc-io/plugeth-statediff/types"
"github.com/cerc-io/plugeth-statediff/utils/log"
)
var _ interfaces.StateDiffIndexer = &StateDiffIndexer{}
@@ -130,7 +129,7 @@ func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receip
BlockNumber: block.Number().String(),
stm: sdi.dbWriter.db.InsertIPLDsStm(),
iplds: make(chan models.IPLDModel),
quit: make(chan struct{}),
quit: make(chan (chan<- struct{})),
ipldCache: models.IPLDBatch{
BlockNumbers: make([]string, 0, startingCacheCapacity),
Keys: make([]string, 0, startingCacheCapacity),
@@ -140,7 +139,10 @@ func (sdi *StateDiffIndexer) PushBlock(block *types.Block, receipts types.Receip
// handle transaction commit or rollback for any return case
submit: func(self *BatchTx, err error) error {
defer func() {
confirm := make(chan struct{})
self.quit <- confirm
close(self.quit)
<-confirm
close(self.iplds)
}()
if p := recover(); p != nil {
@@ -249,15 +251,15 @@ func (sdi *StateDiffIndexer) processHeader(tx *BatchTx, header *types.Header, he
}
// processUncles publishes and indexes uncle IPLDs in Postgres
func (sdi *StateDiffIndexer) processUncles(tx *BatchTx, headerID string, blockNumber *big.Int, unclesHash common.Hash, uncles []*types.Header) error {
func (sdi *StateDiffIndexer) processUncles(tx *BatchTx, headerID string, blockNumber *big.Int, unclesHash core.Hash, uncles []*types.Header) error {
// publish and index uncles
uncleEncoding, err := rlp.EncodeToBytes(uncles)
if err != nil {
return err
}
preparedHash := crypto.Keccak256Hash(uncleEncoding)
if !bytes.Equal(preparedHash.Bytes(), unclesHash.Bytes()) {
return fmt.Errorf("derived uncles hash (%s) does not match the hash in the header (%s)", preparedHash.Hex(), unclesHash.Hex())
if preparedHash != unclesHash {
return fmt.Errorf("derived uncles hash (%s) does not match the hash in the header (%s)", preparedHash.String(), unclesHash.String())
}
unclesCID, err := ipld.RawdataToCid(ipld.MEthHeaderList, uncleEncoding, multihash.KECCAK_256)
if err != nil {
@@ -350,7 +352,7 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(tx *BatchTx, args processArgs
if len(receipt.PostState) == 0 {
rctModel.PostStatus = receipt.Status
} else {
rctModel.PostState = common.BytesToHash(receipt.PostState).String()
rctModel.PostState = core.BytesToHash(receipt.PostState).String()
}
if err := sdi.dbWriter.upsertReceiptCID(tx.dbtx, rctModel); err != nil {
@@ -363,7 +365,7 @@ func (sdi *StateDiffIndexer) processReceiptsAndTxs(tx *BatchTx, args processArgs
tx.cacheIPLD(args.logNodes[i][idx])
topicSet := make([]string, 4)
for ti, topic := range l.Topics {
topicSet[ti] = topic.Hex()
topicSet[ti] = topic.String()
}
logDataSet[idx] = &models.LogsModel{
@@ -401,7 +403,7 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt
stateModel = models.StateNodeModel{
BlockNumber: tx.BlockNumber,
HeaderID: headerID,
StateKey: common.BytesToHash(stateNode.AccountWrapper.LeafKey).String(),
StateKey: core.BytesToHash(stateNode.AccountWrapper.LeafKey).String(),
CID: shared.RemovedNodeStateCID,
Removed: true,
}
@@ -409,12 +411,12 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt
stateModel = models.StateNodeModel{
BlockNumber: tx.BlockNumber,
HeaderID: headerID,
StateKey: common.BytesToHash(stateNode.AccountWrapper.LeafKey).String(),
StateKey: core.BytesToHash(stateNode.AccountWrapper.LeafKey).String(),
CID: stateNode.AccountWrapper.CID,
Removed: false,
Balance: stateNode.AccountWrapper.Account.Balance.String(),
Nonce: stateNode.AccountWrapper.Account.Nonce,
CodeHash: common.BytesToHash(stateNode.AccountWrapper.Account.CodeHash).String(),
CodeHash: core.BytesToHash(stateNode.AccountWrapper.Account.CodeHash).String(),
StorageRoot: stateNode.AccountWrapper.Account.Root.String(),
}
}
@@ -431,8 +433,8 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt
storageModel := models.StorageNodeModel{
BlockNumber: tx.BlockNumber,
HeaderID: headerID,
StateKey: common.BytesToHash(stateNode.AccountWrapper.LeafKey).String(),
StorageKey: common.BytesToHash(storageNode.LeafKey).String(),
StateKey: core.BytesToHash(stateNode.AccountWrapper.LeafKey).String(),
StorageKey: core.BytesToHash(storageNode.LeafKey).String(),
CID: shared.RemovedNodeStorageCID,
Removed: true,
Value: []byte{},
@@ -445,8 +447,8 @@ func (sdi *StateDiffIndexer) PushStateNode(batch interfaces.Batch, stateNode sdt
storageModel := models.StorageNodeModel{
BlockNumber: tx.BlockNumber,
HeaderID: headerID,
StateKey: common.BytesToHash(stateNode.AccountWrapper.LeafKey).String(),
StorageKey: common.BytesToHash(storageNode.LeafKey).String(),
StateKey: core.BytesToHash(stateNode.AccountWrapper.LeafKey).String(),
StorageKey: core.BytesToHash(storageNode.LeafKey).String(),
CID: storageNode.CID,
Removed: false,
Value: storageNode.Value,
@@ -477,7 +479,7 @@ func (sdi *StateDiffIndexer) Close() error {
// Update the known gaps table with the gap information.
// LoadWatchedAddresses reads watched addresses from the database
func (sdi *StateDiffIndexer) LoadWatchedAddresses() ([]common.Address, error) {
func (sdi *StateDiffIndexer) LoadWatchedAddresses() ([]core.Address, error) {
addressStrings := make([]string, 0)
pgStr := "SELECT address FROM eth_meta.watched_addresses"
err := sdi.dbWriter.db.Select(sdi.ctx, &addressStrings, pgStr)
@@ -485,9 +487,9 @@ func (sdi *StateDiffIndexer) LoadWatchedAddresses() ([]common.Address, error) {
return nil, fmt.Errorf("error loading watched addresses: %v", err)
}
watchedAddresses := []common.Address{}
watchedAddresses := []core.Address{}
for _, addressString := range addressStrings {
watchedAddresses = append(watchedAddresses, common.HexToAddress(addressString))
watchedAddresses = append(watchedAddresses, core.HexToAddress(addressString))
}
return watchedAddresses, nil
indexer/database/sql/indexer_shared_test.go
+3 -3
View File
@@ -5,9 +5,9 @@ import (
"github.com/stretchr/testify/require"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
"github.com/ethereum/go-ethereum/statediff/indexer/test_helpers"
"github.com/cerc-io/plugeth-statediff/indexer/database/sql"
"github.com/cerc-io/plugeth-statediff/indexer/interfaces"
"github.com/cerc-io/plugeth-statediff/indexer/test_helpers"
)
var (
indexer/database/sql/interfaces.go
+1 -1
View File
@@ -20,7 +20,7 @@ import (
"context"
"io"
"github.com/ethereum/go-ethereum/statediff/indexer/database/metrics"
"github.com/cerc-io/plugeth-statediff/indexer/database/metrics"
)
// Database interfaces required by the sql indexer
indexer/database/sql/lazy_tx.go
+2 -2
View File
@@ -4,7 +4,7 @@ import (
"context"
"reflect"
"github.com/ethereum/go-ethereum/log"
"github.com/cerc-io/plugeth-statediff/utils/log"
)
// Changing this to 1 would make sure only sequential COPYs were combined.
@@ -86,7 +86,7 @@ func (tx *DelayedTx) Commit(ctx context.Context) error {
case *copyFrom:
_, err := base.CopyFrom(ctx, item.tableName, item.columnNames, item.rows)
if err != nil {
log.Error("COPY error", "table", item.tableName, "err", err)
log.Error("COPY error", "table", item.tableName, "error", err)
return err
}
case cachedStmt:
indexer/database/sql/mainnet_tests/indexer_test.go
+5 -14
View File
@@ -18,20 +18,18 @@ package mainnet_tests
import (
"context"
"fmt"
"math/big"
"os"
"testing"
"github.com/stretchr/testify/require"
"github.com/cerc-io/plugeth-statediff/indexer/database/sql"
"github.com/cerc-io/plugeth-statediff/indexer/database/sql/postgres"
"github.com/cerc-io/plugeth-statediff/indexer/interfaces"
"github.com/cerc-io/plugeth-statediff/indexer/test"
"github.com/cerc-io/plugeth-statediff/indexer/test_helpers"
"github.com/ethereum/go-ethereum/core/types"
"github.com/ethereum/go-ethereum/params"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
"github.com/ethereum/go-ethereum/statediff/indexer/test"
"github.com/ethereum/go-ethereum/statediff/indexer/test_helpers"
)
var (
@@ -41,13 +39,6 @@ var (
chainConf = params.MainnetChainConfig
)
func init() {
if os.Getenv("MODE") != "statediff" {
fmt.Println("Skipping statediff test")
os.Exit(0)
}
}
func TestMainnetIndexer(t *testing.T) {
conf := test_helpers.GetTestConfig()
indexer/database/sql/pgx_indexer_legacy_test.go
+3 -3
View File
@@ -22,9 +22,9 @@ import (
"github.com/stretchr/testify/require"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
"github.com/ethereum/go-ethereum/statediff/indexer/test"
"github.com/cerc-io/plugeth-statediff/indexer/database/sql"
"github.com/cerc-io/plugeth-statediff/indexer/database/sql/postgres"
"github.com/cerc-io/plugeth-statediff/indexer/test"
)
func setupLegacyPGXIndexer(t *testing.T) {
indexer/database/sql/pgx_indexer_test.go
+19 -19
View File
@@ -23,10 +23,10 @@ import (
"github.com/stretchr/testify/require"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
"github.com/ethereum/go-ethereum/statediff/indexer/mocks"
"github.com/ethereum/go-ethereum/statediff/indexer/test"
"github.com/cerc-io/plugeth-statediff/indexer/database/sql"
"github.com/cerc-io/plugeth-statediff/indexer/database/sql/postgres"
"github.com/cerc-io/plugeth-statediff/indexer/mocks"
"github.com/cerc-io/plugeth-statediff/indexer/test"
)
func setupPGXIndexer(t *testing.T, config postgres.Config) {
@@ -59,7 +59,7 @@ func TestPGXIndexer(t *testing.T) {
defer tearDown(t)
defer checkTxClosure(t, 1, 0, 1)
test.TestPublishAndIndexHeaderIPLDs(t, db)
test.DoTestPublishAndIndexHeaderIPLDs(t, db)
})
t.Run("Publish and index transaction IPLDs in a single tx", func(t *testing.T) {
@@ -67,7 +67,7 @@ func TestPGXIndexer(t *testing.T) {
defer tearDown(t)
defer checkTxClosure(t, 1, 0, 1)
test.TestPublishAndIndexTransactionIPLDs(t, db)
test.DoTestPublishAndIndexTransactionIPLDs(t, db)
})
t.Run("Publish and index log IPLDs for multiple receipt of a specific block", func(t *testing.T) {
@@ -75,7 +75,7 @@ func TestPGXIndexer(t *testing.T) {
defer tearDown(t)
defer checkTxClosure(t, 1, 0, 1)
test.TestPublishAndIndexLogIPLDs(t, db)
test.DoTestPublishAndIndexLogIPLDs(t, db)
})
t.Run("Publish and index receipt IPLDs in a single tx", func(t *testing.T) {
@@ -83,7 +83,7 @@ func TestPGXIndexer(t *testing.T) {
defer tearDown(t)
defer checkTxClosure(t, 1, 0, 1)
test.TestPublishAndIndexReceiptIPLDs(t, db)
test.DoTestPublishAndIndexReceiptIPLDs(t, db)
})
t.Run("Publish and index state IPLDs in a single tx", func(t *testing.T) {
@@ -91,7 +91,7 @@ func TestPGXIndexer(t *testing.T) {
defer tearDown(t)
defer checkTxClosure(t, 1, 0, 1)
test.TestPublishAndIndexStateIPLDs(t, db)
test.DoTestPublishAndIndexStateIPLDs(t, db)
})
t.Run("Publish and index storage IPLDs in a single tx", func(t *testing.T) {
@@ -99,7 +99,7 @@ func TestPGXIndexer(t *testing.T) {
defer tearDown(t)
defer checkTxClosure(t, 1, 0, 1)
test.TestPublishAndIndexStorageIPLDs(t, db)
test.DoTestPublishAndIndexStorageIPLDs(t, db)
})
t.Run("Publish and index with CopyFrom enabled.", func(t *testing.T) {
@@ -110,10 +110,10 @@ func TestPGXIndexer(t *testing.T) {
defer tearDown(t)
defer checkTxClosure(t, 1, 0, 1)
test.TestPublishAndIndexStateIPLDs(t, db)
test.TestPublishAndIndexStorageIPLDs(t, db)
test.TestPublishAndIndexReceiptIPLDs(t, db)
test.TestPublishAndIndexLogIPLDs(t, db)
test.DoTestPublishAndIndexStateIPLDs(t, db)
test.DoTestPublishAndIndexStorageIPLDs(t, db)
test.DoTestPublishAndIndexReceiptIPLDs(t, db)
test.DoTestPublishAndIndexLogIPLDs(t, db)
})
}
@@ -132,7 +132,7 @@ func TestPGXIndexerNonCanonical(t *testing.T) {
defer tearDown(t)
defer checkTxClosure(t, 1, 0, 1)
test.TestPublishAndIndexTransactionsNonCanonical(t, db)
test.DoTestPublishAndIndexTransactionsNonCanonical(t, db)
})
t.Run("Publish and index receipts", func(t *testing.T) {
@@ -140,7 +140,7 @@ func TestPGXIndexerNonCanonical(t *testing.T) {
defer tearDown(t)
defer checkTxClosure(t, 1, 0, 1)
test.TestPublishAndIndexReceiptsNonCanonical(t, db)
test.DoTestPublishAndIndexReceiptsNonCanonical(t, db)
})
t.Run("Publish and index logs", func(t *testing.T) {
@@ -148,7 +148,7 @@ func TestPGXIndexerNonCanonical(t *testing.T) {
defer tearDown(t)
defer checkTxClosure(t, 1, 0, 1)
test.TestPublishAndIndexLogsNonCanonical(t, db)
test.DoTestPublishAndIndexLogsNonCanonical(t, db)
})
t.Run("Publish and index state nodes", func(t *testing.T) {
@@ -156,7 +156,7 @@ func TestPGXIndexerNonCanonical(t *testing.T) {
defer tearDown(t)
defer checkTxClosure(t, 1, 0, 1)
test.TestPublishAndIndexStateNonCanonical(t, db)
test.DoTestPublishAndIndexStateNonCanonical(t, db)
})
t.Run("Publish and index storage nodes", func(t *testing.T) {
@@ -164,7 +164,7 @@ func TestPGXIndexerNonCanonical(t *testing.T) {
defer tearDown(t)
defer checkTxClosure(t, 1, 0, 1)
test.TestPublishAndIndexStorageNonCanonical(t, db)
test.DoTestPublishAndIndexStorageNonCanonical(t, db)
})
}
indexer/database/sql/postgres/config.go
+53 -41
View File
@@ -23,49 +23,9 @@ import (
"strings"
"time"
"github.com/ethereum/go-ethereum/statediff/indexer/shared"
"github.com/cerc-io/plugeth-statediff/indexer/shared"
)
// DriverType to explicitly type the kind of sql driver we are using
type DriverType string
const (
PGX DriverType = "PGX"
SQLX DriverType = "SQLX"
Unknown DriverType = "Unknown"
)
// Env variables
const (
DATABASE_NAME = "DATABASE_NAME"
DATABASE_HOSTNAME = "DATABASE_HOSTNAME"
DATABASE_PORT = "DATABASE_PORT"
DATABASE_USER = "DATABASE_USER"
DATABASE_PASSWORD = "DATABASE_PASSWORD"
)
// ResolveDriverType resolves a DriverType from a provided string
func ResolveDriverType(str string) (DriverType, error) {
switch strings.ToLower(str) {
case "pgx", "pgxpool":
return PGX, nil
case "sqlx":
return SQLX, nil
default:
return Unknown, fmt.Errorf("unrecognized driver type string: %s", str)
}
}
// TestConfig specifies default parameters for connecting to a testing DB
var TestConfig = Config{
Hostname: "localhost",
Port: 8077,
DatabaseName: "cerc_testing",
Username: "vdbm",
Password: "password",
Driver: SQLX,
}
// Config holds params for a Postgres db
type Config struct {
// conn string params
@@ -98,6 +58,34 @@ type Config struct {
CopyFrom bool
}
// DriverType to explicitly type the kind of sql driver we are using
type DriverType string
const (
PGX DriverType = "PGX"
SQLX DriverType = "SQLX"
Invalid DriverType = "Invalid"
)
// Env variables
const (
DATABASE_NAME = "DATABASE_NAME"
DATABASE_HOSTNAME = "DATABASE_HOSTNAME"
DATABASE_PORT = "DATABASE_PORT"
DATABASE_USER = "DATABASE_USER"
DATABASE_PASSWORD = "DATABASE_PASSWORD"
)
// TestConfig specifies default parameters for connecting to a testing DB
var TestConfig = Config{
Hostname: "localhost",
Port: 8077,
DatabaseName: "cerc_testing",
Username: "vdbm",
Password: "password",
Driver: SQLX,
}
// Type satisfies interfaces.Config
func (c Config) Type() shared.DBType {
return shared.POSTGRES
@@ -116,6 +104,7 @@ func (c Config) DbConnectionString() string {
return fmt.Sprintf("postgresql://%s:%d/%s?sslmode=disable", c.Hostname, c.Port, c.DatabaseName)
}
// WithEnv overrides the config with env variables, returning a new instance
func (c Config) WithEnv() (Config, error) {
if val := os.Getenv(DATABASE_NAME); val != "" {
c.DatabaseName = val
@@ -138,3 +127,26 @@ func (c Config) WithEnv() (Config, error) {
}
return c, nil
}
// ResolveDriverType resolves a DriverType from a provided string
func ResolveDriverType(str string) (DriverType, error) {
switch strings.ToLower(str) {
case "pgx", "pgxpool":
return PGX, nil
case "sqlx":
return SQLX, nil
default:
return Invalid, fmt.Errorf("unrecognized driver type string: %s", str)
}
}
// Set satisfies flag.Value
func (d *DriverType) Set(v string) (err error) {
*d, err = ResolveDriverType(v)
return
}
// String satisfies flag.Value
func (d *DriverType) String() string {
return strings.ToLower(string(*d))
}
indexer/database/sql/postgres/database.go
+2 -2
View File
@@ -17,8 +17,8 @@
package postgres
import (
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
"github.com/ethereum/go-ethereum/statediff/indexer/shared/schema"
"github.com/cerc-io/plugeth-statediff/indexer/database/sql"
"github.com/cerc-io/plugeth-statediff/indexer/shared/schema"
)
var _ sql.Database = &DB{}
indexer/database/sql/postgres/log_adapter.go
+13 -17
View File
@@ -18,8 +18,9 @@ package postgres
import (
"context"
"github.com/ethereum/go-ethereum/log"
"github.com/jackc/pgx/v4"
"github.com/cerc-io/plugeth-statediff/utils/log"
)
type LogAdapter struct {
@@ -31,31 +32,26 @@ func NewLogAdapter(l log.Logger) *LogAdapter {
}
func (l *LogAdapter) Log(ctx context.Context, level pgx.LogLevel, msg string, data map[string]interface{}) {
var logger log.Logger
if data != nil {
var args = make([]interface{}, 0)
for key, value := range data {
if value != nil {
args = append(args, key, value)
}
args := make([]interface{}, 0)
for key, value := range data {
if value != nil {
args = append(args, key, value)
}
logger = l.l.New(args...)
} else {
logger = l.l
}
logger := l.l
switch level {
case pgx.LogLevelTrace:
logger.Trace(msg)
logger.Trace(msg, args...)
case pgx.LogLevelDebug:
logger.Debug(msg)
logger.Debug(msg, args...)
case pgx.LogLevelInfo:
logger.Info(msg)
logger.Info(msg, args...)
case pgx.LogLevelWarn:
logger.Warn(msg)
logger.Warn(msg, args...)
case pgx.LogLevelError:
logger.Error(msg)
logger.Error(msg, args...)
default:
logger.New("INVALID_PGX_LOG_LEVEL", level).Error(msg)
logger.Error(msg, "INVALID_PGX_LOG_LEVEL", level)
}
}
indexer/database/sql/postgres/pgx.go
+9 -7
View File
@@ -20,16 +20,16 @@ import (
"context"
"time"
"github.com/ethereum/go-ethereum/log"
"github.com/cerc-io/plugeth-statediff/utils/log"
"github.com/georgysavva/scany/pgxscan"
"github.com/jackc/pgconn"
"github.com/jackc/pgx/v4"
"github.com/jackc/pgx/v4/pgxpool"
"github.com/ethereum/go-ethereum/statediff/indexer/database/metrics"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
"github.com/ethereum/go-ethereum/statediff/indexer/node"
"github.com/cerc-io/plugeth-statediff/indexer/database/metrics"
"github.com/cerc-io/plugeth-statediff/indexer/database/sql"
"github.com/cerc-io/plugeth-statediff/indexer/node"
)
// PGXDriver driver, implements sql.Driver
@@ -96,7 +96,7 @@ func MakeConfig(config Config) (*pgxpool.Config, error) {
}
if config.LogStatements {
conf.ConnConfig.Logger = NewLogAdapter(log.New())
conf.ConnConfig.Logger = NewLogAdapter(log.DefaultLogger)
}
return conf, nil
@@ -106,8 +106,10 @@ func (pgx *PGXDriver) createNode() error {
_, err := pgx.pool.Exec(
pgx.ctx,
createNodeStm,
pgx.nodeInfo.GenesisBlock, pgx.nodeInfo.NetworkID,
pgx.nodeInfo.ID, pgx.nodeInfo.ClientName,
pgx.nodeInfo.GenesisBlock,
pgx.nodeInfo.NetworkID,
pgx.nodeInfo.ID,
pgx.nodeInfo.ClientName,
pgx.nodeInfo.ChainID)
if err != nil {
return ErrUnableToSetNode(err)
indexer/database/sql/postgres/pgx_test.go
+2 -2
View File
@@ -26,8 +26,8 @@ import (
"github.com/jackc/pgx/v4/pgxpool"
"github.com/stretchr/testify/require"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
"github.com/ethereum/go-ethereum/statediff/indexer/node"
"github.com/cerc-io/plugeth-statediff/indexer/database/sql/postgres"
"github.com/cerc-io/plugeth-statediff/indexer/node"
)
var (
indexer/database/sql/postgres/postgres_suite_test.go
-33
View File
@@ -1,33 +0,0 @@
// VulcanizeDB
// Copyright © 2019 Vulcanize
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Affero General Public License for more details.
// You should have received a copy of the GNU Affero General Public License
// along with this program. If not, see <http://www.gnu.org/licenses/>.
package postgres_test
import (
"fmt"
"os"
"github.com/ethereum/go-ethereum/log"
)
func init() {
if os.Getenv("MODE") != "statediff" {
fmt.Println("Skipping statediff test")
os.Exit(0)
}
log.Root().SetHandler(log.DiscardHandler())
}
indexer/database/sql/postgres/sqlx.go
+3 -3
View File
@@ -24,9 +24,9 @@ import (
"github.com/jmoiron/sqlx"
"github.com/ethereum/go-ethereum/statediff/indexer/database/metrics"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
"github.com/ethereum/go-ethereum/statediff/indexer/node"
"github.com/cerc-io/plugeth-statediff/indexer/database/metrics"
"github.com/cerc-io/plugeth-statediff/indexer/database/sql"
"github.com/cerc-io/plugeth-statediff/indexer/node"
)
// SQLXDriver driver, implements sql.Driver
indexer/database/sql/postgres/sqlx_test.go
+2 -2
View File
@@ -26,8 +26,8 @@ import (
_ "github.com/lib/pq"
"github.com/stretchr/testify/require"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
"github.com/ethereum/go-ethereum/statediff/indexer/node"
"github.com/cerc-io/plugeth-statediff/indexer/database/sql/postgres"
"github.com/cerc-io/plugeth-statediff/indexer/node"
)
func TestPostgresSQLX(t *testing.T) {
indexer/database/sql/postgres/test_helpers.go
+2 -2
View File
@@ -19,8 +19,8 @@ package postgres
import (
"context"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
"github.com/ethereum/go-ethereum/statediff/indexer/node"
"github.com/cerc-io/plugeth-statediff/indexer/database/sql"
"github.com/cerc-io/plugeth-statediff/indexer/node"
)
// SetupSQLXDB is used to setup a sqlx db for tests
indexer/database/sql/sqlx_indexer_legacy_test.go
+3 -3
View File
@@ -22,9 +22,9 @@ import (
"github.com/stretchr/testify/require"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
"github.com/ethereum/go-ethereum/statediff/indexer/test"
"github.com/cerc-io/plugeth-statediff/indexer/database/sql"
"github.com/cerc-io/plugeth-statediff/indexer/database/sql/postgres"
"github.com/cerc-io/plugeth-statediff/indexer/test"
)
func setupLegacySQLXIndexer(t *testing.T) {
indexer/database/sql/sqlx_indexer_test.go
+15 -15
View File
@@ -23,10 +23,10 @@ import (
"github.com/stretchr/testify/require"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
"github.com/ethereum/go-ethereum/statediff/indexer/mocks"
"github.com/ethereum/go-ethereum/statediff/indexer/test"
"github.com/cerc-io/plugeth-statediff/indexer/database/sql"
"github.com/cerc-io/plugeth-statediff/indexer/database/sql/postgres"
"github.com/cerc-io/plugeth-statediff/indexer/mocks"
"github.com/cerc-io/plugeth-statediff/indexer/test"
)
func setupSQLXIndexer(t *testing.T) {
@@ -55,7 +55,7 @@ func TestSQLXIndexer(t *testing.T) {
defer tearDown(t)
defer checkTxClosure(t, 0, 0, 0)
test.TestPublishAndIndexHeaderIPLDs(t, db)
test.DoTestPublishAndIndexHeaderIPLDs(t, db)
})
t.Run("Publish and index transaction IPLDs in a single tx", func(t *testing.T) {
@@ -63,7 +63,7 @@ func TestSQLXIndexer(t *testing.T) {
defer tearDown(t)
defer checkTxClosure(t, 0, 0, 0)
test.TestPublishAndIndexTransactionIPLDs(t, db)
test.DoTestPublishAndIndexTransactionIPLDs(t, db)
})
t.Run("Publish and index log IPLDs for multiple receipt of a specific block", func(t *testing.T) {
@@ -71,7 +71,7 @@ func TestSQLXIndexer(t *testing.T) {
defer tearDown(t)
defer checkTxClosure(t, 0, 0, 0)
test.TestPublishAndIndexLogIPLDs(t, db)
test.DoTestPublishAndIndexLogIPLDs(t, db)
})
t.Run("Publish and index receipt IPLDs in a single tx", func(t *testing.T) {
@@ -79,7 +79,7 @@ func TestSQLXIndexer(t *testing.T) {
defer tearDown(t)
defer checkTxClosure(t, 0, 0, 0)
test.TestPublishAndIndexReceiptIPLDs(t, db)
test.DoTestPublishAndIndexReceiptIPLDs(t, db)
})
t.Run("Publish and index state IPLDs in a single tx", func(t *testing.T) {
@@ -87,7 +87,7 @@ func TestSQLXIndexer(t *testing.T) {
defer tearDown(t)
defer checkTxClosure(t, 0, 0, 0)
test.TestPublishAndIndexStateIPLDs(t, db)
test.DoTestPublishAndIndexStateIPLDs(t, db)
})
t.Run("Publish and index storage IPLDs in a single tx", func(t *testing.T) {
@@ -95,7 +95,7 @@ func TestSQLXIndexer(t *testing.T) {
defer tearDown(t)
defer checkTxClosure(t, 0, 0, 0)
test.TestPublishAndIndexStorageIPLDs(t, db)
test.DoTestPublishAndIndexStorageIPLDs(t, db)
})
}
@@ -114,7 +114,7 @@ func TestSQLXIndexerNonCanonical(t *testing.T) {
defer tearDown(t)
defer checkTxClosure(t, 0, 0, 0)
test.TestPublishAndIndexTransactionsNonCanonical(t, db)
test.DoTestPublishAndIndexTransactionsNonCanonical(t, db)
})
t.Run("Publish and index receipts", func(t *testing.T) {
@@ -122,7 +122,7 @@ func TestSQLXIndexerNonCanonical(t *testing.T) {
defer tearDown(t)
defer checkTxClosure(t, 0, 0, 0)
test.TestPublishAndIndexReceiptsNonCanonical(t, db)
test.DoTestPublishAndIndexReceiptsNonCanonical(t, db)
})
t.Run("Publish and index logs", func(t *testing.T) {
@@ -130,7 +130,7 @@ func TestSQLXIndexerNonCanonical(t *testing.T) {
defer tearDown(t)
defer checkTxClosure(t, 0, 0, 0)
test.TestPublishAndIndexLogsNonCanonical(t, db)
test.DoTestPublishAndIndexLogsNonCanonical(t, db)
})
t.Run("Publish and index state nodes", func(t *testing.T) {
@@ -138,7 +138,7 @@ func TestSQLXIndexerNonCanonical(t *testing.T) {
defer tearDown(t)
defer checkTxClosure(t, 0, 0, 0)
test.TestPublishAndIndexStateNonCanonical(t, db)
test.DoTestPublishAndIndexStateNonCanonical(t, db)
})
t.Run("Publish and index storage nodes", func(t *testing.T) {
@@ -146,7 +146,7 @@ func TestSQLXIndexerNonCanonical(t *testing.T) {
defer tearDown(t)
defer checkTxClosure(t, 0, 0, 0)
test.TestPublishAndIndexStorageNonCanonical(t, db)
test.DoTestPublishAndIndexStorageNonCanonical(t, db)
})
}
indexer/database/sql/writer.go
+2 -2
View File
@@ -25,8 +25,8 @@ import (
"github.com/lib/pq"
"github.com/shopspring/decimal"
"github.com/ethereum/go-ethereum/statediff/indexer/database/metrics"
"github.com/ethereum/go-ethereum/statediff/indexer/models"
"github.com/cerc-io/plugeth-statediff/indexer/database/metrics"
"github.com/cerc-io/plugeth-statediff/indexer/models"
)
// Writer handles processing and writing of indexed IPLD objects to Postgres