Move back to lotus-shed and add subcategory for indexes

cli/evm.go

@@ -3,14 +3,11 @@ package cli
 import (
 	"bytes"
 	"context"
-	"database/sql"
 	"encoding/base64"
 	"encoding/hex"
 	"fmt"
 	"os"
-	"path/filepath"
 
-	"github.com/mitchellh/go-homedir"
 	"github.com/urfave/cli/v2"
 	cbg "github.com/whyrusleeping/cbor-gen"
 	"golang.org/x/xerrors"
@@ -21,7 +18,6 @@ import (
 	"github.com/filecoin-project/go-state-types/big"
 	builtintypes "github.com/filecoin-project/go-state-types/builtin"
 	"github.com/filecoin-project/go-state-types/builtin/v10/eam"
-	"github.com/filecoin-project/go-state-types/crypto"
 
 	"github.com/filecoin-project/lotus/api/v0api"
 	"github.com/filecoin-project/lotus/chain/actors"
@@ -40,7 +36,6 @@ var EvmCmd = &cli.Command{
 		EvmCallSimulateCmd,
 		EvmGetContractAddress,
 		EvmGetBytecode,
-		EvmBackfillTxHashCmd,
 	},
 }
 
@@ -540,142 +535,3 @@ var EvmGetBytecode = &cli.Command{
 		return nil
 	},
 }
-
-var EvmBackfillTxHashCmd = &cli.Command{
-	Name:  "backfill-txhash",
-	Usage: "Backfills the txhash.db for a number of epochs starting from a specified height",
-	Flags: []cli.Flag{
-		&cli.UintFlag{
-			Name:  "from",
-			Value: 0,
-			Usage: "the tipset height to start backfilling from (0 is head of chain)",
-		},
-		&cli.IntFlag{
-			Name:  "epochs",
-			Value: 2000,
-			Usage: "the number of epochs to backfill",
-		},
-		&cli.StringFlag{
-			Name:  "repo",
-			Value: "~/.lotus",
-			Usage: "path to the repo",
-		},
-	},
-	Action: func(cctx *cli.Context) error {
-		return backfillTxHash(cctx)
-	},
-}
-
-func backfillTxHash(cctx *cli.Context) error {
-	api, closer, err := GetFullNodeAPI(cctx)
-	if err != nil {
-		return err
-	}
-	defer closer()
-
-	ctx := ReqContext(cctx)
-
-	curTs, err := api.ChainHead(ctx)
-	if err != nil {
-		return err
-	}
-
-	startHeight := int64(cctx.Int("from"))
-	if startHeight == 0 {
-		startHeight = int64(curTs.Height()) - 1
-	}
-
-	epochs := cctx.Int("epochs")
-
-	basePath, err := homedir.Expand(cctx.String("repo"))
-	if err != nil {
-		return err
-	}
-
-	dbPath := filepath.Join(basePath, "sqlite", "txhash.db")
-	db, err := sql.Open("sqlite3", dbPath)
-	if err != nil {
-		return err
-	}
-
-	defer func() {
-		err := db.Close()
-		if err != nil {
-			fmt.Printf("ERROR: closing db: %s", err)
-		}
-	}()
-
-	insertStmt, err := db.Prepare("INSERT OR IGNORE INTO eth_tx_hashes(hash, cid) VALUES(?, ?)")
-	if err != nil {
-		return err
-	}
-
-	var totalRowsAffected int64 = 0
-	for i := 0; i < epochs; i++ {
-		epoch := abi.ChainEpoch(startHeight - int64(i))
-
-		select {
-		case <-cctx.Done():
-			fmt.Println("request cancelled")
-			return nil
-		default:
-		}
-
-		curTsk := curTs.Parents()
-		execTs, err := api.ChainGetTipSet(ctx, curTsk)
-		if err != nil {
-			return fmt.Errorf("failed to call ChainGetTipSet for %s: %w", curTsk, err)
-		}
-
-		if i%100 == 0 {
-			log.Infof("%d/%d processing epoch:%d", i, epochs, epoch)
-		}
-
-		for _, blockheader := range execTs.Blocks() {
-			blkMsgs, err := api.ChainGetBlockMessages(ctx, blockheader.Cid())
-			if err != nil {
-				log.Infof("Could not get block messages at epoch: %d, stopping walking up the chain", epoch)
-				epochs = i
-				break
-			}
-
-			for _, smsg := range blkMsgs.SecpkMessages {
-				if smsg.Signature.Type != crypto.SigTypeDelegated {
-					continue
-				}
-
-				tx, err := ethtypes.EthTxFromSignedEthMessage(smsg)
-				if err != nil {
-					return fmt.Errorf("failed to convert from signed message: %w at epoch: %d", err, epoch)
-				}
-
-				tx.Hash, err = tx.TxHash()
-				if err != nil {
-					return fmt.Errorf("failed to calculate hash for ethTx: %w at epoch: %d", err, epoch)
-				}
-
-				res, err := insertStmt.Exec(tx.Hash.String(), smsg.Cid().String())
-				if err != nil {
-					return fmt.Errorf("error inserting tx mapping to db: %s at epoch: %d", err, epoch)
-				}
-
-				rowsAffected, err := res.RowsAffected()
-				if err != nil {
-					return fmt.Errorf("error getting rows affected: %s at epoch: %d", err, epoch)
-				}
-
-				if rowsAffected > 0 {
-					log.Debugf("Inserted txhash %s, cid: %s at epoch: %d", tx.Hash.String(), smsg.Cid().String(), epoch)
-				}
-
-				totalRowsAffected += rowsAffected
-			}
-		}
-
-		curTs = execTs
-	}
-
-	log.Infof("Done, inserted %d missing txhashes", totalRowsAffected)
-
-	return nil
-}

cmd/lotus-shed/indexes.go

@@ -4,29 +4,39 @@ import (
 	"database/sql"
 	"fmt"
 	"path"
+	"path/filepath"
+	"strings"
 
-	_ "github.com/mattn/go-sqlite3"
 	"github.com/mitchellh/go-homedir"
 	"github.com/urfave/cli/v2"
 	"golang.org/x/xerrors"
 
 	"github.com/filecoin-project/go-state-types/abi"
+	"github.com/filecoin-project/go-state-types/crypto"
 
+	"github.com/filecoin-project/lotus/chain/types/ethtypes"
 	lcli "github.com/filecoin-project/lotus/cli"
 )
 
-var msgindexCmd = &cli.Command{
+func withCategory(cat string, cmd *cli.Command) *cli.Command {
-	Name:  "msgindex",
+	cmd.Category = strings.ToUpper(cat)
-	Usage: "Tools for managing the message index",
+	return cmd
+}
+
+var indexesCmd = &cli.Command{
+	Name:            "indexes",
+	Usage:           "Commands related to managing sqlite indexes",
+	HideHelpCommand: true,
 	Subcommands: []*cli.Command{
-		msgindexBackfillCmd,
+		withCategory("msgindex", backfillMsgIndexCmd),
-		msgindexPruneCmd,
+		withCategory("msgindex", pruneMsgIndexCmd),
+		withCategory("txhash", backfillTxHashCmd),
 	},
 }
 
-var msgindexBackfillCmd = &cli.Command{
+var backfillMsgIndexCmd = &cli.Command{
-	Name:  "backfill",
+	Name:  "backfill-msgindex",
-	Usage: "Backfill the message index for a number of epochs starting from a specified height",
+	Usage: "Backfill the msgindex.db for a number of epochs starting from a specified height",
 	Flags: []cli.Flag{
 		&cli.IntFlag{
 			Name:  "from",
@@ -38,11 +48,6 @@ var msgindexBackfillCmd = &cli.Command{
 			Value: 1800,
 			Usage: "number of epochs to backfill; defaults to 1800 (2 finalities)",
 		},
-		&cli.StringFlag{
-			Name:  "repo",
-			Value: "~/.lotus",
-			Usage: "path to the repo",
-		},
 	},
 	Action: func(cctx *cli.Context) error {
 		api, closer, err := lcli.GetFullNodeAPI(cctx)
@@ -131,19 +136,14 @@ var msgindexBackfillCmd = &cli.Command{
 	},
 }
 
-var msgindexPruneCmd = &cli.Command{
+var pruneMsgIndexCmd = &cli.Command{
-	Name:  "prune",
+	Name:  "prune-msgindex",
-	Usage: "Prune the message index for messages included before a given epoch",
+	Usage: "Prune the msgindex.db for messages included before a given epoch",
 	Flags: []cli.Flag{
 		&cli.IntFlag{
 			Name:  "from",
 			Usage: "height to start the prune; if negative it indicates epochs from current head",
 		},
-		&cli.StringFlag{
-			Name:  "repo",
-			Value: "~/.lotus",
-			Usage: "path to the repo",
-		},
 	},
 	Action: func(cctx *cli.Context) error {
 		api, closer, err := lcli.GetFullNodeAPI(cctx)
@@ -205,3 +205,133 @@ var msgindexPruneCmd = &cli.Command{
 		return nil
 	},
 }
+
+var backfillTxHashCmd = &cli.Command{
+	Name:  "backfill-txhash",
+	Usage: "Backfills the txhash.db for a number of epochs starting from a specified height",
+	Flags: []cli.Flag{
+		&cli.UintFlag{
+			Name:  "from",
+			Value: 0,
+			Usage: "the tipset height to start backfilling from (0 is head of chain)",
+		},
+		&cli.IntFlag{
+			Name:  "epochs",
+			Value: 2000,
+			Usage: "the number of epochs to backfill",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		api, closer, err := lcli.GetFullNodeAPI(cctx)
+		if err != nil {
+			return err
+		}
+		defer closer()
+
+		ctx := lcli.ReqContext(cctx)
+
+		curTs, err := api.ChainHead(ctx)
+		if err != nil {
+			return err
+		}
+
+		startHeight := int64(cctx.Int("from"))
+		if startHeight == 0 {
+			startHeight = int64(curTs.Height()) - 1
+		}
+
+		epochs := cctx.Int("epochs")
+
+		basePath, err := homedir.Expand(cctx.String("repo"))
+		if err != nil {
+			return err
+		}
+
+		dbPath := filepath.Join(basePath, "sqlite", "txhash.db")
+		db, err := sql.Open("sqlite3", dbPath)
+		if err != nil {
+			return err
+		}
+
+		defer func() {
+			err := db.Close()
+			if err != nil {
+				fmt.Printf("ERROR: closing db: %s", err)
+			}
+		}()
+
+		insertStmt, err := db.Prepare("INSERT OR IGNORE INTO eth_tx_hashes(hash, cid) VALUES(?, ?)")
+		if err != nil {
+			return err
+		}
+
+		var totalRowsAffected int64 = 0
+		for i := 0; i < epochs; i++ {
+			epoch := abi.ChainEpoch(startHeight - int64(i))
+
+			select {
+			case <-cctx.Done():
+				fmt.Println("request cancelled")
+				return nil
+			default:
+			}
+
+			curTsk := curTs.Parents()
+			execTs, err := api.ChainGetTipSet(ctx, curTsk)
+			if err != nil {
+				return fmt.Errorf("failed to call ChainGetTipSet for %s: %w", curTsk, err)
+			}
+
+			if i%100 == 0 {
+				log.Infof("%d/%d processing epoch:%d", i, epochs, epoch)
+			}
+
+			for _, blockheader := range execTs.Blocks() {
+				blkMsgs, err := api.ChainGetBlockMessages(ctx, blockheader.Cid())
+				if err != nil {
+					log.Infof("Could not get block messages at epoch: %d, stopping walking up the chain", epoch)
+					epochs = i
+					break
+				}
+
+				for _, smsg := range blkMsgs.SecpkMessages {
+					if smsg.Signature.Type != crypto.SigTypeDelegated {
+						continue
+					}
+
+					tx, err := ethtypes.EthTxFromSignedEthMessage(smsg)
+					if err != nil {
+						return fmt.Errorf("failed to convert from signed message: %w at epoch: %d", err, epoch)
+					}
+
+					tx.Hash, err = tx.TxHash()
+					if err != nil {
+						return fmt.Errorf("failed to calculate hash for ethTx: %w at epoch: %d", err, epoch)
+					}
+
+					res, err := insertStmt.Exec(tx.Hash.String(), smsg.Cid().String())
+					if err != nil {
+						return fmt.Errorf("error inserting tx mapping to db: %s at epoch: %d", err, epoch)
+					}
+
+					rowsAffected, err := res.RowsAffected()
+					if err != nil {
+						return fmt.Errorf("error getting rows affected: %s at epoch: %d", err, epoch)
+					}
+
+					if rowsAffected > 0 {
+						log.Debugf("Inserted txhash %s, cid: %s at epoch: %d", tx.Hash.String(), smsg.Cid().String(), epoch)
+					}
+
+					totalRowsAffected += rowsAffected
+				}
+			}
+
+			curTs = execTs
+		}
+
+		log.Infof("Done, inserted %d missing txhashes", totalRowsAffected)
+
+		return nil
+	},
+}

cmd/lotus-shed/main.go

@@ -86,7 +86,7 @@ func main() {
 		invariantsCmd,
 		gasTraceCmd,
 		replayOfflineCmd,
-		msgindexCmd,
+		indexesCmd,
 		FevmAnalyticsCmd,
 		mismatchesCmd,
 	}

documentation/en/cli-lotus.md

@@ -2648,7 +2648,6 @@ COMMANDS:
      call              Simulate an eth contract call
      contract-address  Generate contract address from smart contract code
      bytecode          Write the bytecode of a smart contract to a file
-     backfill-txhash   
      help, h           Shows a list of commands or help for one command
 
 OPTIONS:
@@ -2736,24 +2735,6 @@ OPTIONS:
    
 ```
 
-### lotus evm backfill-txhash
-```
-NAME:
-   lotus evm backfill-txhash
-
-USAGE:
-   lotus evm backfill-txhash [command options] [arguments...]
-
-DESCRIPTION:
-   Backfills the txhash.db for a number of epochs starting from a specified height
-
-OPTIONS:
-   --epochs value  the number of epochs to backfill (default: 2000)
-   --from value    the tipset height to start backfilling from (0 is head of chain) (default: 0)
-   --repo value    path to the repo (default: "~/.lotus")
-   
-```
-
 ## lotus net
 ```
 NAME: