cli integration

cmd/geth/config.go

@@ -45,6 +45,7 @@ import (
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/statediff"
 	dumpdb "github.com/ethereum/go-ethereum/statediff/indexer/database/dump"
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/file"
 	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
 	"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"
 	"github.com/ethereum/go-ethereum/statediff/indexer/shared"
@@ -204,6 +205,14 @@ func makeFullNode(ctx *cli.Context) (*node.Node, ethapi.Backend) {
 				utils.Fatalf("%v", err)
 			}
 			switch dbType {
+			case shared.FILE:
+				if !ctx.GlobalIsSet(utils.StateDiffFileNodeRowIDFlag.Name) {
+					utils.Fatalf("In statediff file writing mode a node row ID must be provided")
+				}
+				indexerConfig = file.Config{
+					NodeID:   int64(ctx.GlobalInt(utils.StateDiffFileNodeRowIDFlag.Name)),
+					FilePath: ctx.GlobalString(utils.StateDiffFilePath.Name),
+				}
 			case shared.POSTGRES:
 				driverTypeStr := ctx.GlobalString(utils.StateDiffDBDriverTypeFlag.Name)
 				driverType, err := postgres.ResolveDriverType(driverTypeStr)

cmd/geth/main.go

@@ -167,6 +167,8 @@ var (
 		utils.StateDiffDBClientNameFlag,
 		utils.StateDiffWritingFlag,
 		utils.StateDiffWorkersFlag,
+		utils.StateDiffFileNodeRowIDFlag,
+		utils.StateDiffFilePath,
 		configFileFlag,
 		utils.CatalystFlag,
 	}

cmd/geth/usage.go

@@ -243,6 +243,8 @@ var AppHelpFlagGroups = []flags.FlagGroup{
 			utils.StateDiffDBClientNameFlag,
 			utils.StateDiffWritingFlag,
 			utils.StateDiffWorkersFlag,
+			utils.StateDiffFileNodeRowIDFlag,
+			utils.StateDiffFilePath,
 		},
 	},
 	{

cmd/utils/flags.go

@@ -788,7 +788,7 @@ var (
 	}
 	StateDiffDBTypeFlag = cli.StringFlag{
 		Name:  "statediff.db.type",
-		Usage: "Statediff database type",
+		Usage: "Statediff database type (current options: postgres, file, dump)",
 		Value: "postgres",
 	}
 	StateDiffDBDriverTypeFlag = cli.StringFlag{
@@ -852,6 +852,14 @@ var (
 		Name:  "statediff.db.nodeid",
 		Usage: "Node ID to use when writing state diffs to database",
 	}
+	StateDiffFileNodeRowIDFlag = cli.IntFlag{
+		Name:  "statediff.file.nodeid",
+		Usage: "Node row ID to use as FK when writing state diffs to database",
+	}
+	StateDiffFilePath = cli.StringFlag{
+		Name:  "statediff.file.path",
+		Usage: "Full path (including filename) to write statediff data out to when operating in file mode",
+	}
 	StateDiffDBClientNameFlag = cli.StringFlag{
 		Name:  "statediff.db.clientname",
 		Usage: "Client name to use when writing state diffs to database",

statediff/indexer/constructor.go

@@ -20,10 +20,9 @@ import (
 	"context"
 	"fmt"
 
-	"github.com/ethereum/go-ethereum/statediff/indexer/database/file"
-
 	"github.com/ethereum/go-ethereum/params"
 	"github.com/ethereum/go-ethereum/statediff/indexer/database/dump"
+	"github.com/ethereum/go-ethereum/statediff/indexer/database/file"
 	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql"
 	"github.com/ethereum/go-ethereum/statediff/indexer/database/sql/postgres"
 	"github.com/ethereum/go-ethereum/statediff/indexer/interfaces"

statediff/indexer/database/file/config.go

@@ -22,7 +22,7 @@ import (
 
 // Config holds params for writing sql statements out to a file
 type Config struct {
-	NodeID   int64 // this is the nodeID used as FK in public.blocks
+	NodeID   int64 // this is the nodeID used as FK in eth.header_cids
 	FilePath string
 }
 

statediff/indexer/database/file/writer.go

@@ -63,9 +63,7 @@ func NewSQLWriter(file *os.File) *SQLWriter {
 func (sqw *SQLWriter) Loop() {
 	sqw.collationIndex = 0
 	go func() {
-		defer func() {
+		defer close(sqw.doneChan)
-			close(sqw.doneChan)
-		}()
 		var l int
 		for {
 			select {

statediff/indexer/database/sql/pgx_indexer_legacy_test.go

@@ -53,7 +53,7 @@ func setupLegacyPGX(t *testing.T) {
 		}
 	}()
 	for _, node := range legacyData.StateDiffs {
-		err = ind.PushStateNode(tx, node)
+		err = ind.PushStateNode(tx, node, legacyData.MockBlock.Hash().String())
 		require.NoError(t, err)
 	}
 

statediff/indexer/database/sql/sqlx_indexer_test.go

@@ -159,7 +159,7 @@ func setupSQLX(t *testing.T) {
 		}
 	}()
 	for _, node := range mocks.StateDiffs {
-		err = ind.PushStateNode(tx, node)
+		err = ind.PushStateNode(tx, node, mockBlock.Hash().String())
 		if err != nil {
 			t.Fatal(err)
 		}

statediff/indexer/models/models.go

@@ -100,7 +100,7 @@ type StateNodeModel struct {
 
 // StorageNodeModel is the db model for eth.storage_cids
 type StorageNodeModel struct {
-	HeaderID   string `db:"header_id""`
+	HeaderID   string `db:"header_id"`
 	StatePath  []byte `db:"state_path"`
 	Path       []byte `db:"storage_path"`
 	StorageKey string `db:"storage_leaf_key"`
@@ -112,7 +112,7 @@ type StorageNodeModel struct {
 
 // StorageNodeWithStateKeyModel is a db model for eth.storage_cids + eth.state_cids.state_key
 type StorageNodeWithStateKeyModel struct {
-	HeaderID   string `db:"header_id""`
+	HeaderID   string `db:"header_id"`
 	StatePath  []byte `db:"state_path"`
 	Path       []byte `db:"storage_path"`
 	StateKey   string `db:"state_leaf_key"`