diff --git a/cmd/geth/config.go b/cmd/geth/config.go index 98e9a25d7..cc947f369 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -21,6 +21,14 @@ import ( "context" "errors" "fmt" + "math/big" + "os" + "reflect" + "time" + "unicode" + + "github.com/urfave/cli/v2" + "github.com/ethereum/go-ethereum/accounts/external" "github.com/ethereum/go-ethereum/accounts/keystore" "github.com/ethereum/go-ethereum/accounts/scwallet" @@ -41,12 +49,6 @@ import ( "github.com/ethereum/go-ethereum/statediff/indexer/interfaces" "github.com/ethereum/go-ethereum/statediff/indexer/shared" "github.com/naoina/toml" - "github.com/urfave/cli/v2" - "math/big" - "os" - "reflect" - "time" - "unicode" ) var ( @@ -156,7 +158,7 @@ func makeConfigNode(ctx *cli.Context) (*node.Node, gethConfig) { cfg.Ethstats.URL = ctx.String(utils.EthStatsURLFlag.Name) } applyMetricConfig(ctx, &cfg) - if ctx.GlobalBool(utils.StateDiffFlag.Name) { + if ctx.Bool(utils.StateDiffFlag.Name) { cfg.Eth.Diffing = true } @@ -192,25 +194,25 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) { } } - if ctx.GlobalBool(utils.StateDiffFlag.Name) { + if ctx.Bool(utils.StateDiffFlag.Name) { var indexerConfig interfaces.Config var clientName, nodeID string - if ctx.GlobalIsSet(utils.StateDiffWritingFlag.Name) { - clientName = ctx.GlobalString(utils.StateDiffDBClientNameFlag.Name) - if ctx.GlobalIsSet(utils.StateDiffDBNodeIDFlag.Name) { - nodeID = ctx.GlobalString(utils.StateDiffDBNodeIDFlag.Name) + if ctx.IsSet(utils.StateDiffWritingFlag.Name) { + clientName = ctx.String(utils.StateDiffDBClientNameFlag.Name) + if ctx.IsSet(utils.StateDiffDBNodeIDFlag.Name) { + nodeID = ctx.String(utils.StateDiffDBNodeIDFlag.Name) } else { utils.Fatalf("Must specify node ID for statediff DB output") } - dbTypeStr := ctx.GlobalString(utils.StateDiffDBTypeFlag.Name) + dbTypeStr := ctx.String(utils.StateDiffDBTypeFlag.Name) dbType, err := shared.ResolveDBType(dbTypeStr) if err != nil { utils.Fatalf("%v", err) } switch dbType { case shared.FILE: - fileModeStr := ctx.GlobalString(utils.StateDiffFileMode.Name) + fileModeStr := ctx.String(utils.StateDiffFileMode.Name) fileMode, err := file.ResolveFileMode(fileModeStr) if err != nil { utils.Fatalf("%v", err) @@ -218,47 +220,47 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) { indexerConfig = file.Config{ Mode: fileMode, - OutputDir: ctx.GlobalString(utils.StateDiffFileCsvDir.Name), - FilePath: ctx.GlobalString(utils.StateDiffFilePath.Name), - WatchedAddressesFilePath: ctx.GlobalString(utils.StateDiffWatchedAddressesFilePath.Name), + OutputDir: ctx.String(utils.StateDiffFileCsvDir.Name), + FilePath: ctx.String(utils.StateDiffFilePath.Name), + WatchedAddressesFilePath: ctx.String(utils.StateDiffWatchedAddressesFilePath.Name), } case shared.POSTGRES: - driverTypeStr := ctx.GlobalString(utils.StateDiffDBDriverTypeFlag.Name) + driverTypeStr := ctx.String(utils.StateDiffDBDriverTypeFlag.Name) driverType, err := postgres.ResolveDriverType(driverTypeStr) if err != nil { utils.Fatalf("%v", err) } pgConfig := postgres.Config{ - Hostname: ctx.GlobalString(utils.StateDiffDBHostFlag.Name), - Port: ctx.GlobalInt(utils.StateDiffDBPortFlag.Name), - DatabaseName: ctx.GlobalString(utils.StateDiffDBNameFlag.Name), - Username: ctx.GlobalString(utils.StateDiffDBUserFlag.Name), - Password: ctx.GlobalString(utils.StateDiffDBPasswordFlag.Name), + Hostname: ctx.String(utils.StateDiffDBHostFlag.Name), + Port: ctx.Int(utils.StateDiffDBPortFlag.Name), + DatabaseName: ctx.String(utils.StateDiffDBNameFlag.Name), + Username: ctx.String(utils.StateDiffDBUserFlag.Name), + Password: ctx.String(utils.StateDiffDBPasswordFlag.Name), ID: nodeID, ClientName: clientName, Driver: driverType, } - if ctx.GlobalIsSet(utils.StateDiffDBMinConns.Name) { - pgConfig.MinConns = ctx.GlobalInt(utils.StateDiffDBMinConns.Name) + if ctx.IsSet(utils.StateDiffDBMinConns.Name) { + pgConfig.MinConns = ctx.Int(utils.StateDiffDBMinConns.Name) } - if ctx.GlobalIsSet(utils.StateDiffDBMaxConns.Name) { - pgConfig.MaxConns = ctx.GlobalInt(utils.StateDiffDBMaxConns.Name) + if ctx.IsSet(utils.StateDiffDBMaxConns.Name) { + pgConfig.MaxConns = ctx.Int(utils.StateDiffDBMaxConns.Name) } - if ctx.GlobalIsSet(utils.StateDiffDBMaxIdleConns.Name) { - pgConfig.MaxIdle = ctx.GlobalInt(utils.StateDiffDBMaxIdleConns.Name) + if ctx.IsSet(utils.StateDiffDBMaxIdleConns.Name) { + pgConfig.MaxIdle = ctx.Int(utils.StateDiffDBMaxIdleConns.Name) } - if ctx.GlobalIsSet(utils.StateDiffDBMaxConnLifetime.Name) { - pgConfig.MaxConnLifetime = time.Duration(ctx.GlobalDuration(utils.StateDiffDBMaxConnLifetime.Name).Seconds()) + if ctx.IsSet(utils.StateDiffDBMaxConnLifetime.Name) { + pgConfig.MaxConnLifetime = time.Duration(ctx.Duration(utils.StateDiffDBMaxConnLifetime.Name).Seconds()) } - if ctx.GlobalIsSet(utils.StateDiffDBMaxConnIdleTime.Name) { - pgConfig.MaxConnIdleTime = time.Duration(ctx.GlobalDuration(utils.StateDiffDBMaxConnIdleTime.Name).Seconds()) + if ctx.IsSet(utils.StateDiffDBMaxConnIdleTime.Name) { + pgConfig.MaxConnIdleTime = time.Duration(ctx.Duration(utils.StateDiffDBMaxConnIdleTime.Name).Seconds()) } - if ctx.GlobalIsSet(utils.StateDiffDBConnTimeout.Name) { - pgConfig.ConnTimeout = time.Duration(ctx.GlobalDuration(utils.StateDiffDBConnTimeout.Name).Seconds()) + if ctx.IsSet(utils.StateDiffDBConnTimeout.Name) { + pgConfig.ConnTimeout = time.Duration(ctx.Duration(utils.StateDiffDBConnTimeout.Name).Seconds()) } indexerConfig = pgConfig case shared.DUMP: - dumpTypeStr := ctx.GlobalString(utils.StateDiffDBDumpDst.Name) + dumpTypeStr := ctx.String(utils.StateDiffDBDumpDst.Name) dumpType, err := dumpdb.ResolveDumpType(dumpTypeStr) if err != nil { utils.Fatalf("%v", err) @@ -279,13 +281,13 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) { } p := statediff.Config{ IndexerConfig: indexerConfig, - KnownGapsFilePath: ctx.GlobalString(utils.StateDiffKnownGapsFilePath.Name), + KnownGapsFilePath: ctx.String(utils.StateDiffKnownGapsFilePath.Name), ID: nodeID, ClientName: clientName, Context: context.Background(), - EnableWriteLoop: ctx.GlobalBool(utils.StateDiffWritingFlag.Name), - NumWorkers: ctx.GlobalUint(utils.StateDiffWorkersFlag.Name), - WaitForSync: ctx.GlobalBool(utils.StateDiffWaitForSync.Name), + EnableWriteLoop: ctx.Bool(utils.StateDiffWritingFlag.Name), + NumWorkers: ctx.Uint(utils.StateDiffWorkersFlag.Name), + WaitForSync: ctx.Bool(utils.StateDiffWaitForSync.Name), } utils.RegisterStateDiffService(stack, eth, &cfg.Eth, p, backend) } diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 50e70f175..a41ccf3d7 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -973,113 +973,114 @@ var ( Category: flags.MetricsCategory, } - StateDiffFlag = cli.BoolFlag{ - Name: "statediff", - Usage: "Enables the processing of state diffs between each block", + StateDiffFlag = &cli.BoolFlag{ + Name: "statediff", + Usage: "Enables the processing of state diffs between each block", + Category: flags.MiscCategory, } - StateDiffDBTypeFlag = cli.StringFlag{ + StateDiffDBTypeFlag = &cli.StringFlag{ Name: "statediff.db.type", Usage: "Statediff database type (current options: postgres, file, dump)", Value: "postgres", } - StateDiffDBDriverTypeFlag = cli.StringFlag{ + StateDiffDBDriverTypeFlag = &cli.StringFlag{ Name: "statediff.db.driver", Usage: "Statediff database driver type", Value: "pgx", } - StateDiffDBDumpDst = cli.StringFlag{ + StateDiffDBDumpDst = &cli.StringFlag{ Name: "statediff.dump.dst", Usage: "Statediff database dump destination (default is stdout)", Value: "stdout", } - StateDiffDBHostFlag = cli.StringFlag{ + StateDiffDBHostFlag = &cli.StringFlag{ Name: "statediff.db.host", Usage: "Statediff database hostname/ip", Value: "localhost", } - StateDiffDBPortFlag = cli.IntFlag{ + StateDiffDBPortFlag = &cli.IntFlag{ Name: "statediff.db.port", Usage: "Statediff database port", Value: 5432, } - StateDiffDBNameFlag = cli.StringFlag{ + StateDiffDBNameFlag = &cli.StringFlag{ Name: "statediff.db.name", Usage: "Statediff database name", } - StateDiffDBPasswordFlag = cli.StringFlag{ + StateDiffDBPasswordFlag = &cli.StringFlag{ Name: "statediff.db.password", Usage: "Statediff database password", } - StateDiffDBUserFlag = cli.StringFlag{ + StateDiffDBUserFlag = &cli.StringFlag{ Name: "statediff.db.user", Usage: "Statediff database username", Value: "postgres", } - StateDiffDBMaxConnLifetime = cli.DurationFlag{ + StateDiffDBMaxConnLifetime = &cli.DurationFlag{ Name: "statediff.db.maxconnlifetime", Usage: "Statediff database maximum connection lifetime (in seconds)", } - StateDiffDBMaxConnIdleTime = cli.DurationFlag{ + StateDiffDBMaxConnIdleTime = &cli.DurationFlag{ Name: "statediff.db.maxconnidletime", Usage: "Statediff database maximum connection idle time (in seconds)", } - StateDiffDBMaxConns = cli.IntFlag{ + StateDiffDBMaxConns = &cli.IntFlag{ Name: "statediff.db.maxconns", Usage: "Statediff database maximum connections", } - StateDiffDBMinConns = cli.IntFlag{ + StateDiffDBMinConns = &cli.IntFlag{ Name: "statediff.db.minconns", Usage: "Statediff database minimum connections", } - StateDiffDBMaxIdleConns = cli.IntFlag{ + StateDiffDBMaxIdleConns = &cli.IntFlag{ Name: "statediff.db.maxidleconns", Usage: "Statediff database maximum idle connections", } - StateDiffDBConnTimeout = cli.DurationFlag{ + StateDiffDBConnTimeout = &cli.DurationFlag{ Name: "statediff.db.conntimeout", Usage: "Statediff database connection timeout (in seconds)", } - StateDiffDBNodeIDFlag = cli.StringFlag{ + StateDiffDBNodeIDFlag = &cli.StringFlag{ Name: "statediff.db.nodeid", Usage: "Node ID to use when writing state diffs to database", } - StateDiffFileMode = cli.StringFlag{ + StateDiffFileMode = &cli.StringFlag{ Name: "statediff.file.mode", Usage: "Statediff file writing mode (current options: csv, sql)", Value: "csv", } - StateDiffFileCsvDir = cli.StringFlag{ + StateDiffFileCsvDir = &cli.StringFlag{ Name: "statediff.file.csvdir", Usage: "Full path of output directory to write statediff data out to when operating in csv file mode", } - StateDiffFilePath = cli.StringFlag{ + StateDiffFilePath = &cli.StringFlag{ Name: "statediff.file.path", Usage: "Full path (including filename) to write statediff data out to when operating in sql file mode", } - StateDiffKnownGapsFilePath = cli.StringFlag{ + StateDiffKnownGapsFilePath = &cli.StringFlag{ Name: "statediff.knowngapsfile.path", Usage: "Full path (including filename) to write knownGaps statements when the DB is unavailable.", Value: "./known_gaps.sql", } - StateDiffWatchedAddressesFilePath = cli.StringFlag{ + StateDiffWatchedAddressesFilePath = &cli.StringFlag{ Name: "statediff.file.wapath", Usage: "Full path (including filename) to write statediff watched addresses out to when operating in file mode", } - StateDiffDBClientNameFlag = cli.StringFlag{ + StateDiffDBClientNameFlag = &cli.StringFlag{ Name: "statediff.db.clientname", Usage: "Client name to use when writing state diffs to database", Value: "go-ethereum", } - StateDiffWritingFlag = cli.BoolFlag{ + StateDiffWritingFlag = &cli.BoolFlag{ Name: "statediff.writing", Usage: "Activates progressive writing of state diffs to database as new block are synced", } - StateDiffWorkersFlag = cli.UintFlag{ + StateDiffWorkersFlag = &cli.UintFlag{ Name: "statediff.workers", Usage: "Number of concurrent workers to use during statediff processing (default 1)", Value: 1, } - StateDiffWaitForSync = cli.BoolFlag{ + StateDiffWaitForSync = &cli.BoolFlag{ Name: "statediff.waitforsync", Usage: "Should the statediff service wait for geth to catch up to the head of the chain?", } @@ -1345,7 +1346,7 @@ func setWS(ctx *cli.Context, cfg *node.Config) { cfg.WSPathPrefix = ctx.String(WSPathPrefixFlag.Name) } - if ctx.GlobalBool(StateDiffFlag.Name) { + if ctx.Bool(StateDiffFlag.Name) { cfg.WSModules = append(cfg.WSModules, "statediff") } } diff --git a/statediff/indexer/ipld/eth_account_test.go b/statediff/indexer/ipld/eth_account_test.go index dbfabc9fb..f7c5341a6 100644 --- a/statediff/indexer/ipld/eth_account_test.go +++ b/statediff/indexer/ipld/eth_account_test.go @@ -128,7 +128,6 @@ func TestAccountSnapshotResolve(t *testing.T) { t.Fatalf("error should be nil %v", gc) } } - } func TestAccountSnapshotTree(t *testing.T) { diff --git a/statediff/indexer/ipld/eth_header_test.go b/statediff/indexer/ipld/eth_header_test.go index ebbab2129..fa4806fbf 100644 --- a/statediff/indexer/ipld/eth_header_test.go +++ b/statediff/indexer/ipld/eth_header_test.go @@ -290,7 +290,6 @@ func TestEthBlockResolveNonLinkFieldsExtraPathElements(t *testing.T) { if err.Error() != "unexpected path elements past "+field { t.Fatalf("Wrong error\r\nexpected %s\r\ngot %s", "unexpected path elements past "+field, err.Error()) } - } } diff --git a/statediff/indexer/ipld/eth_tx_test.go b/statediff/indexer/ipld/eth_tx_test.go index 5afa42234..8b459621e 100644 --- a/statediff/indexer/ipld/eth_tx_test.go +++ b/statediff/indexer/ipld/eth_tx_test.go @@ -187,7 +187,6 @@ func TestEthTxResolve(t *testing.T) { t.Fatalf("error should be nil %v", gc) } } - } func TestEthTxTree(t *testing.T) { diff --git a/statediff/indexer/ipld/eth_tx_trie_test.go b/statediff/indexer/ipld/eth_tx_trie_test.go index 42637ceb4..b067d0ea4 100644 --- a/statediff/indexer/ipld/eth_tx_trie_test.go +++ b/statediff/indexer/ipld/eth_tx_trie_test.go @@ -224,7 +224,6 @@ func TestTxTrieResolveBranch(t *testing.T) { if err != nil { t.Fatal("Error should be nil") } - } } diff --git a/statediff/known_gaps.go b/statediff/known_gaps.go index df6860c83..922eb6100 100644 --- a/statediff/known_gaps.go +++ b/statediff/known_gaps.go @@ -65,9 +65,7 @@ type KnownGapsState struct { } // Create a new KnownGapsState struct, currently unused. -func NewKnownGapsState(checkForGaps bool, processingKey int64, expectedDifference *big.Int, - errorState bool, writeFilePath string, db sql.Database, statediffMetrics statediffMetricsHandles) *KnownGapsState { - +func NewKnownGapsState(checkForGaps bool, processingKey int64, expectedDifference *big.Int, errorState bool, writeFilePath string, db sql.Database, statediffMetrics statediffMetricsHandles) *KnownGapsState { return &KnownGapsState{ checkForGaps: checkForGaps, processingKey: processingKey, @@ -77,7 +75,6 @@ func NewKnownGapsState(checkForGaps bool, processingKey int64, expectedDifferenc db: db, statediffMetrics: statediffMetrics, } - } func minMax(array []*big.Int) (*big.Int, *big.Int) { @@ -142,7 +139,6 @@ func (kg *KnownGapsState) captureErrorBlocks(knownErrorBlocks []*big.Int) { log.Warn("The following Gaps were found", "knownErrorBlocks", knownErrorBlocks) log.Warn("Updating known Gaps table", "startErrorBlock", startErrorBlock, "endErrorBlock", endErrorBlock, "processingKey", kg.processingKey) kg.pushKnownGaps(startErrorBlock, endErrorBlock, false, kg.processingKey) - } // Users provide the latestBlockInDb and the latestBlockOnChain @@ -156,7 +152,6 @@ func isGap(latestBlockInDb *big.Int, latestBlockOnChain *big.Int, expectedDiffer return true } return false - } // This function will check for Gaps and update the DB if gaps are found. diff --git a/statediff/known_gaps_test.go b/statediff/known_gaps_test.go index 11463b7ad..258bb887a 100644 --- a/statediff/known_gaps_test.go +++ b/statediff/known_gaps_test.go @@ -28,7 +28,6 @@ type gapValues struct { // Test for failures when they are expected, when we go from smaller block to larger block // We should no longer see the smaller block in DB func TestKnownGaps(t *testing.T) { - tests := []gapValues{ // Known Gaps {knownErrorBlocksStart: 115, knownErrorBlocksEnd: 120, expectedDif: 1, processingKey: 1}, @@ -77,7 +76,6 @@ func testWriteToDb(t *testing.T, tests []gapValues, wipeDbBeforeStart bool) { validateUpsert(t, service, tc.knownErrorBlocksStart, tc.knownErrorBlocksEnd) } tearDown(t, db) - } // test writing blocks to file and then inserting them to DB @@ -167,7 +165,6 @@ func testFindAndUpdateGaps(t *testing.T, wipeDbBeforeStart bool) { startBlock.Add(latestBlockInDb, expectedDifference) endBlock.Sub(latestBlockOnChain, expectedDifference) validateUpsert(t, service, startBlock.Int64(), endBlock.Int64()) - } // test capturing missed blocks diff --git a/statediff/service.go b/statediff/service.go index 30d227f17..b73247647 100644 --- a/statediff/service.go +++ b/statediff/service.go @@ -647,7 +647,7 @@ func (sds *Service) Unsubscribe(id rpc.ID) error { // This function will check the status of geth syncing. // It will return false if geth has finished syncing. // It will return a true Geth is still syncing. -func (sds *Service) GetSyncStatus(pubEthAPI *ethapi.PublicEthereumAPI) (bool, error) { +func (sds *Service) GetSyncStatus(pubEthAPI *ethapi.EthereumAPI) (bool, error) { syncStatus, err := pubEthAPI.Syncing() if err != nil { return true, err @@ -667,7 +667,7 @@ func (sds *Service) WaitingForSync() error { // Has the geth node synced to head? Synced := false - pubEthAPI := ethapi.NewPublicEthereumAPI(sds.BackendAPI) + pubEthAPI := ethapi.NewEthereumAPI(sds.BackendAPI) for !Synced { syncStatus, err := sds.GetSyncStatus(pubEthAPI) if err != nil { diff --git a/statediff/service_test.go b/statediff/service_test.go index ecf8bdadb..1df068608 100644 --- a/statediff/service_test.go +++ b/statediff/service_test.go @@ -396,7 +396,7 @@ func testGetSyncStatus(t *testing.T) { // Update the backend current block value t.Log("Updating Current Block to: ", table.currentBlock) backend.CurrBlock = table.currentBlock - pubEthAPI := ethapi.NewPublicEthereumAPI(service.BackendAPI) + pubEthAPI := ethapi.NewEthereumAPI(service.BackendAPI) syncStatus, err := service.GetSyncStatus(pubEthAPI) if err != nil { diff --git a/statediff/trie_helpers/helpers.go b/statediff/trie_helpers/helpers.go index ce3365f2c..087cfe419 100644 --- a/statediff/trie_helpers/helpers.go +++ b/statediff/trie_helpers/helpers.go @@ -120,5 +120,4 @@ func FindIntersection(a, b []string) []string { } } } - }