From aa984711bd95f404debbc739385d01f58c33def0 Mon Sep 17 00:00:00 2001 From: i-norden Date: Wed, 17 Nov 2021 12:22:40 -0600 Subject: [PATCH] cli integration --- cmd/geth/config.go | 9 +++++++++ cmd/geth/main.go | 2 ++ cmd/geth/usage.go | 2 ++ cmd/utils/flags.go | 10 +++++++++- statediff/indexer/constructor.go | 3 +-- statediff/indexer/database/file/config.go | 2 +- statediff/indexer/database/file/writer.go | 4 +--- .../indexer/database/sql/pgx_indexer_legacy_test.go | 2 +- statediff/indexer/database/sql/sqlx_indexer_test.go | 2 +- statediff/indexer/models/models.go | 4 ++-- 10 files changed, 29 insertions(+), 11 deletions(-) diff --git a/cmd/geth/config.go b/cmd/geth/config.go index 9a8b169be..bb93509e0 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -45,6 +45,7 @@ import ( "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/statediff" dumpdb "github.com/ethereum/go-ethereum/statediff/indexer/database/dump" + "github.com/ethereum/go-ethereum/statediff/indexer/database/file" "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" "github.com/ethereum/go-ethereum/statediff/indexer/interfaces" "github.com/ethereum/go-ethereum/statediff/indexer/shared" @@ -204,6 +205,14 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) { utils.Fatalf("%v", err) } switch dbType { + case shared.FILE: + if !ctx.GlobalIsSet(utils.StateDiffFileNodeRowIDFlag.Name) { + utils.Fatalf("In statediff file writing mode a node row ID must be provided") + } + indexerConfig = file.Config{ + NodeID: int64(ctx.GlobalInt(utils.StateDiffFileNodeRowIDFlag.Name)), + FilePath: ctx.GlobalString(utils.StateDiffFilePath.Name), + } case shared.POSTGRES: driverTypeStr := ctx.GlobalString(utils.StateDiffDBDriverTypeFlag.Name) driverType, err := postgres.ResolveDriverType(driverTypeStr) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index c92810d11..e027eddf9 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -167,6 +167,8 @@ var ( utils.StateDiffDBClientNameFlag, utils.StateDiffWritingFlag, utils.StateDiffWorkersFlag, + utils.StateDiffFileNodeRowIDFlag, + utils.StateDiffFilePath, configFileFlag, utils.CatalystFlag, } diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index 68e2a3f4c..7f768c11a 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -243,6 +243,8 @@ var AppHelpFlagGroups = []flags.FlagGroup{ utils.StateDiffDBClientNameFlag, utils.StateDiffWritingFlag, utils.StateDiffWorkersFlag, + utils.StateDiffFileNodeRowIDFlag, + utils.StateDiffFilePath, }, }, { diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 08f9088f5..26f904aaa 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -788,7 +788,7 @@ var ( } StateDiffDBTypeFlag = cli.StringFlag{ Name: "statediff.db.type", - Usage: "Statediff database type", + Usage: "Statediff database type (current options: postgres, file, dump)", Value: "postgres", } StateDiffDBDriverTypeFlag = cli.StringFlag{ @@ -852,6 +852,14 @@ var ( Name: "statediff.db.nodeid", Usage: "Node ID to use when writing state diffs to database", } + StateDiffFileNodeRowIDFlag = cli.IntFlag{ + Name: "statediff.file.nodeid", + Usage: "Node row ID to use as FK when writing state diffs to database", + } + StateDiffFilePath = cli.StringFlag{ + Name: "statediff.file.path", + Usage: "Full path (including filename) to write statediff data out to when operating in file mode", + } StateDiffDBClientNameFlag = cli.StringFlag{ Name: "statediff.db.clientname", Usage: "Client name to use when writing state diffs to database", diff --git a/statediff/indexer/constructor.go b/statediff/indexer/constructor.go index a4cfa271d..60e6dadb7 100644 --- a/statediff/indexer/constructor.go +++ b/statediff/indexer/constructor.go @@ -20,10 +20,9 @@ import ( "context" "fmt" - "github.com/ethereum/go-ethereum/statediff/indexer/database/file" - "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/statediff/indexer/database/dump" + "github.com/ethereum/go-ethereum/statediff/indexer/database/file" "github.com/ethereum/go-ethereum/statediff/indexer/database/sql" "github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres" "github.com/ethereum/go-ethereum/statediff/indexer/interfaces" diff --git a/statediff/indexer/database/file/config.go b/statediff/indexer/database/file/config.go index 312ad7009..fd9e488c9 100644 --- a/statediff/indexer/database/file/config.go +++ b/statediff/indexer/database/file/config.go @@ -22,7 +22,7 @@ import ( // Config holds params for writing sql statements out to a file type Config struct { - NodeID int64 // this is the nodeID used as FK in public.blocks + NodeID int64 // this is the nodeID used as FK in eth.header_cids FilePath string } diff --git a/statediff/indexer/database/file/writer.go b/statediff/indexer/database/file/writer.go index 2a836c8b3..d95fba328 100644 --- a/statediff/indexer/database/file/writer.go +++ b/statediff/indexer/database/file/writer.go @@ -63,9 +63,7 @@ func NewSQLWriter(file *os.File) *SQLWriter { func (sqw *SQLWriter) Loop() { sqw.collationIndex = 0 go func() { - defer func() { - close(sqw.doneChan) - }() + defer close(sqw.doneChan) var l int for { select { diff --git a/statediff/indexer/database/sql/pgx_indexer_legacy_test.go b/statediff/indexer/database/sql/pgx_indexer_legacy_test.go index d08336e63..0373ff5ee 100644 --- a/statediff/indexer/database/sql/pgx_indexer_legacy_test.go +++ b/statediff/indexer/database/sql/pgx_indexer_legacy_test.go @@ -53,7 +53,7 @@ func setupLegacyPGX(t *testing.T) { } }() for _, node := range legacyData.StateDiffs { - err = ind.PushStateNode(tx, node) + err = ind.PushStateNode(tx, node, legacyData.MockBlock.Hash().String()) require.NoError(t, err) } diff --git a/statediff/indexer/database/sql/sqlx_indexer_test.go b/statediff/indexer/database/sql/sqlx_indexer_test.go index 0fa4e8c1a..1861d65d4 100644 --- a/statediff/indexer/database/sql/sqlx_indexer_test.go +++ b/statediff/indexer/database/sql/sqlx_indexer_test.go @@ -159,7 +159,7 @@ func setupSQLX(t *testing.T) { } }() for _, node := range mocks.StateDiffs { - err = ind.PushStateNode(tx, node) + err = ind.PushStateNode(tx, node, mockBlock.Hash().String()) if err != nil { t.Fatal(err) } diff --git a/statediff/indexer/models/models.go b/statediff/indexer/models/models.go index 7d2fc43b6..60d83d96e 100644 --- a/statediff/indexer/models/models.go +++ b/statediff/indexer/models/models.go @@ -100,7 +100,7 @@ type StateNodeModel struct { // StorageNodeModel is the db model for eth.storage_cids type StorageNodeModel struct { - HeaderID string `db:"header_id""` + HeaderID string `db:"header_id"` StatePath []byte `db:"state_path"` Path []byte `db:"storage_path"` StorageKey string `db:"storage_leaf_key"` @@ -112,7 +112,7 @@ type StorageNodeModel struct { // StorageNodeWithStateKeyModel is a db model for eth.storage_cids + eth.state_cids.state_key type StorageNodeWithStateKeyModel struct { - HeaderID string `db:"header_id""` + HeaderID string `db:"header_id"` StatePath []byte `db:"state_path"` Path []byte `db:"storage_path"` StateKey string `db:"state_leaf_key"`