Initial plugin implementation
* refactor packages, flags, subscriptions * DRY refactor builder tests * use mockgen to generate mocks * update README * MODE=statediff no longer needed for unit tests * simplify func names, clean up metrics * move write params to service field * sql indexer: confirm quit after ipld cache reset prevents negative waitgroup panic * don't let TotalDifficulty become nil * use forked plugeth, plugeth-utils for now
This commit is contained in:
+182
@@ -0,0 +1,182 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"os"
|
||||
|
||||
"github.com/cerc-io/plugeth-statediff"
|
||||
"github.com/cerc-io/plugeth-statediff/indexer/database/dump"
|
||||
"github.com/cerc-io/plugeth-statediff/indexer/database/file"
|
||||
"github.com/cerc-io/plugeth-statediff/indexer/database/sql/postgres"
|
||||
"github.com/cerc-io/plugeth-statediff/indexer/interfaces"
|
||||
"github.com/cerc-io/plugeth-statediff/indexer/shared"
|
||||
"github.com/cerc-io/plugeth-statediff/utils"
|
||||
)
|
||||
|
||||
var (
|
||||
Flags = *flag.NewFlagSet("statediff", flag.PanicOnError)
|
||||
|
||||
enableStatediff bool
|
||||
config = statediff.Config{
|
||||
Context: context.Background(),
|
||||
}
|
||||
dbType = shared.POSTGRES
|
||||
dbDumpDst = dump.STDOUT
|
||||
dbConfig = postgres.Config{Driver: postgres.PGX}
|
||||
fileConfig = file.Config{Mode: file.CSV}
|
||||
)
|
||||
|
||||
func init() {
|
||||
Flags.BoolVar(&enableStatediff,
|
||||
"statediff", false,
|
||||
"Enables the processing of state diffs between each block",
|
||||
)
|
||||
Flags.BoolVar(&config.EnableWriteLoop,
|
||||
"statediff.writing", false,
|
||||
"Activates progressive writing of state diffs to database as new blocks are synced",
|
||||
)
|
||||
Flags.StringVar(&config.ID,
|
||||
"statediff.db.nodeid", "",
|
||||
"Node ID to use when writing state diffs to database",
|
||||
)
|
||||
Flags.StringVar(&config.ClientName,
|
||||
"statediff.db.clientname", "go-ethereum",
|
||||
"Client name to use when writing state diffs to database",
|
||||
)
|
||||
Flags.UintVar(&config.NumWorkers,
|
||||
"statediff.workers", 1,
|
||||
"Number of concurrent workers to use during statediff processing (default 1)",
|
||||
)
|
||||
Flags.BoolVar(&config.WaitForSync,
|
||||
"statediff.waitforsync", false,
|
||||
"Should the statediff service wait for geth to catch up to the head of the chain?",
|
||||
)
|
||||
|
||||
Flags.Var(&dbType,
|
||||
"statediff.db.type",
|
||||
"Statediff database type (current options: postgres, file, dump)",
|
||||
)
|
||||
Flags.StringVar(&dbDumpDst,
|
||||
"statediff.dump.dst", "stdout",
|
||||
"Statediff database dump destination (default is stdout)",
|
||||
)
|
||||
|
||||
Flags.Var(&dbConfig.Driver,
|
||||
"statediff.db.driver",
|
||||
"Statediff database driver type",
|
||||
)
|
||||
Flags.StringVar(&dbConfig.Hostname,
|
||||
"statediff.db.host", "localhost",
|
||||
"Statediff database hostname/ip",
|
||||
)
|
||||
Flags.IntVar(&dbConfig.Port,
|
||||
"statediff.db.port", 5432,
|
||||
"Statediff database port",
|
||||
)
|
||||
Flags.StringVar(&dbConfig.DatabaseName,
|
||||
"statediff.db.name", "",
|
||||
"Statediff database name",
|
||||
)
|
||||
Flags.StringVar(&dbConfig.Password,
|
||||
"statediff.db.password", "",
|
||||
"Statediff database password",
|
||||
)
|
||||
Flags.StringVar(&dbConfig.Username,
|
||||
"statediff.db.user", "postgres",
|
||||
"Statediff database username",
|
||||
)
|
||||
Flags.DurationVar(&dbConfig.MaxConnLifetime,
|
||||
"statediff.db.maxconnlifetime", 0,
|
||||
"Statediff database maximum connection lifetime (in seconds)",
|
||||
)
|
||||
Flags.DurationVar(&dbConfig.MaxConnIdleTime,
|
||||
"statediff.db.maxconnidletime", 0,
|
||||
"Statediff database maximum connection idle time (in seconds)",
|
||||
)
|
||||
Flags.IntVar(&dbConfig.MaxConns,
|
||||
"statediff.db.maxconns", 0,
|
||||
"Statediff database maximum connections",
|
||||
)
|
||||
Flags.IntVar(&dbConfig.MinConns,
|
||||
"statediff.db.minconns", 0,
|
||||
"Statediff database minimum connections",
|
||||
)
|
||||
Flags.IntVar(&dbConfig.MaxIdle,
|
||||
"statediff.db.maxidleconns", 0,
|
||||
"Statediff database maximum idle connections",
|
||||
)
|
||||
Flags.DurationVar(&dbConfig.ConnTimeout,
|
||||
"statediff.db.conntimeout", 0,
|
||||
"Statediff database connection timeout (in seconds)",
|
||||
)
|
||||
Flags.BoolVar(&dbConfig.Upsert,
|
||||
"statediff.db.upsert", false,
|
||||
"Should the statediff service overwrite data existing in the database?",
|
||||
)
|
||||
Flags.BoolVar(&dbConfig.CopyFrom,
|
||||
"statediff.db.copyfrom", false,
|
||||
"Should the statediff service use COPY FROM for multiple inserts? (Note: pgx only)",
|
||||
)
|
||||
Flags.BoolVar(&dbConfig.LogStatements,
|
||||
"statediff.db.logstatements", false,
|
||||
"Should the statediff service log all database statements? (Note: pgx only)",
|
||||
)
|
||||
|
||||
Flags.Var(&fileConfig.Mode,
|
||||
"statediff.file.mode",
|
||||
"Statediff file writing mode (current options: csv, sql)",
|
||||
)
|
||||
Flags.StringVar(&fileConfig.OutputDir,
|
||||
"statediff.file.csvdir", "",
|
||||
"Full path of output directory to write statediff data out to when operating in csv file mode",
|
||||
)
|
||||
Flags.StringVar(&fileConfig.FilePath,
|
||||
"statediff.file.path", "",
|
||||
"Full path (including filename) to write statediff data out to when operating in sql file mode",
|
||||
)
|
||||
Flags.StringVar(&fileConfig.WatchedAddressesFilePath,
|
||||
"statediff.file.wapath", "",
|
||||
"Full path (including filename) to write statediff watched addresses out to when operating in file mode",
|
||||
)
|
||||
}
|
||||
|
||||
func GetConfig() statediff.Config {
|
||||
initConfig()
|
||||
return config
|
||||
}
|
||||
|
||||
func initConfig() {
|
||||
if !enableStatediff {
|
||||
config = statediff.Config{}
|
||||
return
|
||||
}
|
||||
|
||||
if config.ID == "" {
|
||||
utils.Fatalf("Must specify node ID for statediff DB output")
|
||||
}
|
||||
|
||||
var indexerConfig interfaces.Config
|
||||
switch dbType {
|
||||
case shared.FILE:
|
||||
indexerConfig = fileConfig
|
||||
case shared.POSTGRES:
|
||||
dbConfig.ID = config.ID
|
||||
dbConfig.ClientName = config.ClientName
|
||||
indexerConfig = dbConfig
|
||||
case shared.DUMP:
|
||||
switch dbDumpDst {
|
||||
case dump.STDERR:
|
||||
indexerConfig = dump.Config{Dump: os.Stdout}
|
||||
case dump.STDOUT:
|
||||
indexerConfig = dump.Config{Dump: os.Stderr}
|
||||
case dump.DISCARD:
|
||||
indexerConfig = dump.Config{Dump: dump.Discard}
|
||||
default:
|
||||
utils.Fatalf("unrecognized dump destination: %s", dbDumpDst)
|
||||
}
|
||||
default:
|
||||
utils.Fatalf("unrecognized database type: %s", dbType)
|
||||
}
|
||||
config.IndexerConfig = indexerConfig
|
||||
}
|
||||
+107
@@ -0,0 +1,107 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
|
||||
geth_flags "github.com/ethereum/go-ethereum/cmd/utils"
|
||||
"github.com/openrelayxyz/plugeth-utils/core"
|
||||
"github.com/openrelayxyz/plugeth-utils/restricted"
|
||||
|
||||
statediff "github.com/cerc-io/plugeth-statediff"
|
||||
"github.com/cerc-io/plugeth-statediff/adapt"
|
||||
ind "github.com/cerc-io/plugeth-statediff/indexer"
|
||||
"github.com/cerc-io/plugeth-statediff/indexer/interfaces"
|
||||
"github.com/cerc-io/plugeth-statediff/indexer/node"
|
||||
"github.com/cerc-io/plugeth-statediff/utils/log"
|
||||
)
|
||||
|
||||
var (
|
||||
pluginLoader core.PluginLoader
|
||||
gethContext core.Context
|
||||
service *statediff.Service
|
||||
blockchain statediff.BlockChain
|
||||
)
|
||||
|
||||
func Initialize(ctx core.Context, pl core.PluginLoader, logger core.Logger) {
|
||||
log.SetDefaultLogger(logger)
|
||||
|
||||
// lvl, err := strconv.ParseInt(ctx.String("verbosity"), 10, 8)
|
||||
// if err != nil {
|
||||
// log.Error("cannot parse verbosity", "error", err)
|
||||
// }
|
||||
// log.TestLogger.SetLevel(int(lvl))
|
||||
// log.SetDefaultLogger(log.TestLogger)
|
||||
|
||||
pluginLoader = pl
|
||||
gethContext = ctx
|
||||
|
||||
log.Debug("Initialized statediff plugin")
|
||||
}
|
||||
|
||||
func InitializeNode(stack core.Node, b core.Backend) {
|
||||
backend := b.(restricted.Backend)
|
||||
|
||||
networkid, err := strconv.ParseUint(gethContext.String(geth_flags.NetworkIdFlag.Name), 10, 64)
|
||||
if err != nil {
|
||||
log.Error("cannot parse network ID", "error", err)
|
||||
return
|
||||
}
|
||||
serviceConfig := GetConfig()
|
||||
blockchain = statediff.NewPluginBlockChain(backend)
|
||||
|
||||
var indexer interfaces.StateDiffIndexer
|
||||
if serviceConfig.IndexerConfig != nil {
|
||||
info := node.Info{
|
||||
GenesisBlock: blockchain.GetBlockByNumber(0).Hash().String(),
|
||||
NetworkID: strconv.FormatUint(networkid, 10),
|
||||
ChainID: backend.ChainConfig().ChainID.Uint64(),
|
||||
ID: serviceConfig.ID,
|
||||
ClientName: serviceConfig.ClientName,
|
||||
}
|
||||
var err error
|
||||
_, indexer, err = ind.NewStateDiffIndexer(serviceConfig.Context,
|
||||
adapt.ChainConfig(backend.ChainConfig()), info, serviceConfig.IndexerConfig)
|
||||
if err != nil {
|
||||
log.Error("failed to construct indexer", "error", err)
|
||||
}
|
||||
}
|
||||
service, err := statediff.NewService(serviceConfig, blockchain, backend, indexer)
|
||||
if err != nil {
|
||||
log.Error("failed to construct service", "error", err)
|
||||
}
|
||||
if err = service.Start(); err != nil {
|
||||
log.Error("failed to start service", "error", err)
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
func GetAPIs(stack core.Node, backend core.Backend) []core.API {
|
||||
return []core.API{
|
||||
{
|
||||
Namespace: statediff.APIName,
|
||||
Version: statediff.APIVersion,
|
||||
Service: statediff.NewPublicAPI(service),
|
||||
Public: true,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// StateUpdate gives us updates about state changes made in each block.
|
||||
// We extract contract code here, since it's not exposed by plugeth's state interfaces.
|
||||
func StateUpdate(
|
||||
blockRoot core.Hash,
|
||||
parentRoot core.Hash,
|
||||
destructs map[core.Hash]struct{},
|
||||
accounts map[core.Hash][]byte,
|
||||
storage map[core.Hash]map[core.Hash][]byte,
|
||||
codeUpdates map[core.Hash][]byte) {
|
||||
if blockchain == nil {
|
||||
log.Warn("StateUpdate called before InitializeNode", "root", blockRoot)
|
||||
return
|
||||
}
|
||||
|
||||
// for hash, code := range codeUpdates {
|
||||
// log.Debug("UPDATING CODE", "hash", hash)
|
||||
// codeStore.Set(hash, code)
|
||||
// }
|
||||
}
|
||||
Reference in New Issue
Block a user