Move back to lotus-shed and add subcategory for indexes
This commit is contained in:
Fridrik Asmundsson
parent
e414acb1f5
commit
26b4866841
144
cli/evm.go
144
cli/evm.go
@ -3,14 +3,11 @@ package cli
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"database/sql"
|
||||
"encoding/base64"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/mitchellh/go-homedir"
|
||||
"github.com/urfave/cli/v2"
|
||||
cbg "github.com/whyrusleeping/cbor-gen"
|
||||
"golang.org/x/xerrors"
|
||||
@ -21,7 +18,6 @@ import (
|
||||
"github.com/filecoin-project/go-state-types/big"
|
||||
builtintypes "github.com/filecoin-project/go-state-types/builtin"
|
||||
"github.com/filecoin-project/go-state-types/builtin/v10/eam"
|
||||
"github.com/filecoin-project/go-state-types/crypto"
|
||||
|
||||
"github.com/filecoin-project/lotus/api/v0api"
|
||||
"github.com/filecoin-project/lotus/chain/actors"
|
||||
@ -40,7 +36,6 @@ var EvmCmd = &cli.Command{
|
||||
EvmCallSimulateCmd,
|
||||
EvmGetContractAddress,
|
||||
EvmGetBytecode,
|
||||
EvmBackfillTxHashCmd,
|
||||
},
|
||||
}
|
||||
|
||||
@ -540,142 +535,3 @@ var EvmGetBytecode = &cli.Command{
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
var EvmBackfillTxHashCmd = &cli.Command{
|
||||
Name: "backfill-txhash",
|
||||
Usage: "Backfills the txhash.db for a number of epochs starting from a specified height",
|
||||
Flags: []cli.Flag{
|
||||
&cli.UintFlag{
|
||||
Name: "from",
|
||||
Value: 0,
|
||||
Usage: "the tipset height to start backfilling from (0 is head of chain)",
|
||||
},
|
||||
&cli.IntFlag{
|
||||
Name: "epochs",
|
||||
Value: 2000,
|
||||
Usage: "the number of epochs to backfill",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "repo",
|
||||
Value: "~/.lotus",
|
||||
Usage: "path to the repo",
|
||||
},
|
||||
},
|
||||
Action: func(cctx *cli.Context) error {
|
||||
return backfillTxHash(cctx)
|
||||
},
|
||||
}
|
||||
|
||||
func backfillTxHash(cctx *cli.Context) error {
|
||||
api, closer, err := GetFullNodeAPI(cctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer closer()
|
||||
|
||||
ctx := ReqContext(cctx)
|
||||
|
||||
curTs, err := api.ChainHead(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
startHeight := int64(cctx.Int("from"))
|
||||
if startHeight == 0 {
|
||||
startHeight = int64(curTs.Height()) - 1
|
||||
}
|
||||
|
||||
epochs := cctx.Int("epochs")
|
||||
|
||||
basePath, err := homedir.Expand(cctx.String("repo"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
dbPath := filepath.Join(basePath, "sqlite", "txhash.db")
|
||||
db, err := sql.Open("sqlite3", dbPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
err := db.Close()
|
||||
if err != nil {
|
||||
fmt.Printf("ERROR: closing db: %s", err)
|
||||
}
|
||||
}()
|
||||
|
||||
insertStmt, err := db.Prepare("INSERT OR IGNORE INTO eth_tx_hashes(hash, cid) VALUES(?, ?)")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var totalRowsAffected int64 = 0
|
||||
for i := 0; i < epochs; i++ {
|
||||
epoch := abi.ChainEpoch(startHeight - int64(i))
|
||||
|
||||
select {
|
||||
case <-cctx.Done():
|
||||
fmt.Println("request cancelled")
|
||||
return nil
|
||||
default:
|
||||
}
|
||||
|
||||
curTsk := curTs.Parents()
|
||||
execTs, err := api.ChainGetTipSet(ctx, curTsk)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to call ChainGetTipSet for %s: %w", curTsk, err)
|
||||
}
|
||||
|
||||
if i%100 == 0 {
|
||||
log.Infof("%d/%d processing epoch:%d", i, epochs, epoch)
|
||||
}
|
||||
|
||||
for _, blockheader := range execTs.Blocks() {
|
||||
blkMsgs, err := api.ChainGetBlockMessages(ctx, blockheader.Cid())
|
||||
if err != nil {
|
||||
log.Infof("Could not get block messages at epoch: %d, stopping walking up the chain", epoch)
|
||||
epochs = i
|
||||
break
|
||||
}
|
||||
|
||||
for _, smsg := range blkMsgs.SecpkMessages {
|
||||
if smsg.Signature.Type != crypto.SigTypeDelegated {
|
||||
continue
|
||||
}
|
||||
|
||||
tx, err := ethtypes.EthTxFromSignedEthMessage(smsg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to convert from signed message: %w at epoch: %d", err, epoch)
|
||||
}
|
||||
|
||||
tx.Hash, err = tx.TxHash()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to calculate hash for ethTx: %w at epoch: %d", err, epoch)
|
||||
}
|
||||
|
||||
res, err := insertStmt.Exec(tx.Hash.String(), smsg.Cid().String())
|
||||
if err != nil {
|
||||
return fmt.Errorf("error inserting tx mapping to db: %s at epoch: %d", err, epoch)
|
||||
}
|
||||
|
||||
rowsAffected, err := res.RowsAffected()
|
||||
if err != nil {
|
||||
return fmt.Errorf("error getting rows affected: %s at epoch: %d", err, epoch)
|
||||
}
|
||||
|
||||
if rowsAffected > 0 {
|
||||
log.Debugf("Inserted txhash %s, cid: %s at epoch: %d", tx.Hash.String(), smsg.Cid().String(), epoch)
|
||||
}
|
||||
|
||||
totalRowsAffected += rowsAffected
|
||||
}
|
||||
}
|
||||
|
||||
curTs = execTs
|
||||
}
|
||||
|
||||
log.Infof("Done, inserted %d missing txhashes", totalRowsAffected)
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -4,29 +4,39 @@ import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
_ "github.com/mattn/go-sqlite3"
|
||||
"github.com/mitchellh/go-homedir"
|
||||
"github.com/urfave/cli/v2"
|
||||
"golang.org/x/xerrors"
|
||||
|
||||
"github.com/filecoin-project/go-state-types/abi"
|
||||
"github.com/filecoin-project/go-state-types/crypto"
|
||||
|
||||
"github.com/filecoin-project/lotus/chain/types/ethtypes"
|
||||
lcli "github.com/filecoin-project/lotus/cli"
|
||||
)
|
||||
|
||||
var msgindexCmd = &cli.Command{
|
||||
Name: "msgindex",
|
||||
Usage: "Tools for managing the message index",
|
||||
func withCategory(cat string, cmd *cli.Command) *cli.Command {
|
||||
cmd.Category = strings.ToUpper(cat)
|
||||
return cmd
|
||||
}
|
||||
|
||||
var indexesCmd = &cli.Command{
|
||||
Name: "indexes",
|
||||
Usage: "Commands related to managing sqlite indexes",
|
||||
HideHelpCommand: true,
|
||||
Subcommands: []*cli.Command{
|
||||
msgindexBackfillCmd,
|
||||
msgindexPruneCmd,
|
||||
withCategory("msgindex", backfillMsgIndexCmd),
|
||||
withCategory("msgindex", pruneMsgIndexCmd),
|
||||
withCategory("txhash", backfillTxHashCmd),
|
||||
},
|
||||
}
|
||||
|
||||
var msgindexBackfillCmd = &cli.Command{
|
||||
Name: "backfill",
|
||||
Usage: "Backfill the message index for a number of epochs starting from a specified height",
|
||||
var backfillMsgIndexCmd = &cli.Command{
|
||||
Name: "backfill-msgindex",
|
||||
Usage: "Backfill the msgindex.db for a number of epochs starting from a specified height",
|
||||
Flags: []cli.Flag{
|
||||
&cli.IntFlag{
|
||||
Name: "from",
|
||||
@ -38,11 +48,6 @@ var msgindexBackfillCmd = &cli.Command{
|
||||
Value: 1800,
|
||||
Usage: "number of epochs to backfill; defaults to 1800 (2 finalities)",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "repo",
|
||||
Value: "~/.lotus",
|
||||
Usage: "path to the repo",
|
||||
},
|
||||
},
|
||||
Action: func(cctx *cli.Context) error {
|
||||
api, closer, err := lcli.GetFullNodeAPI(cctx)
|
||||
@ -131,19 +136,14 @@ var msgindexBackfillCmd = &cli.Command{
|
||||
},
|
||||
}
|
||||
|
||||
var msgindexPruneCmd = &cli.Command{
|
||||
Name: "prune",
|
||||
Usage: "Prune the message index for messages included before a given epoch",
|
||||
var pruneMsgIndexCmd = &cli.Command{
|
||||
Name: "prune-msgindex",
|
||||
Usage: "Prune the msgindex.db for messages included before a given epoch",
|
||||
Flags: []cli.Flag{
|
||||
&cli.IntFlag{
|
||||
Name: "from",
|
||||
Usage: "height to start the prune; if negative it indicates epochs from current head",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "repo",
|
||||
Value: "~/.lotus",
|
||||
Usage: "path to the repo",
|
||||
},
|
||||
},
|
||||
Action: func(cctx *cli.Context) error {
|
||||
api, closer, err := lcli.GetFullNodeAPI(cctx)
|
||||
@ -205,3 +205,133 @@ var msgindexPruneCmd = &cli.Command{
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
var backfillTxHashCmd = &cli.Command{
|
||||
Name: "backfill-txhash",
|
||||
Usage: "Backfills the txhash.db for a number of epochs starting from a specified height",
|
||||
Flags: []cli.Flag{
|
||||
&cli.UintFlag{
|
||||
Name: "from",
|
||||
Value: 0,
|
||||
Usage: "the tipset height to start backfilling from (0 is head of chain)",
|
||||
},
|
||||
&cli.IntFlag{
|
||||
Name: "epochs",
|
||||
Value: 2000,
|
||||
Usage: "the number of epochs to backfill",
|
||||
},
|
||||
},
|
||||
Action: func(cctx *cli.Context) error {
|
||||
api, closer, err := lcli.GetFullNodeAPI(cctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer closer()
|
||||
|
||||
ctx := lcli.ReqContext(cctx)
|
||||
|
||||
curTs, err := api.ChainHead(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
startHeight := int64(cctx.Int("from"))
|
||||
if startHeight == 0 {
|
||||
startHeight = int64(curTs.Height()) - 1
|
||||
}
|
||||
|
||||
epochs := cctx.Int("epochs")
|
||||
|
||||
basePath, err := homedir.Expand(cctx.String("repo"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
dbPath := filepath.Join(basePath, "sqlite", "txhash.db")
|
||||
db, err := sql.Open("sqlite3", dbPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
err := db.Close()
|
||||
if err != nil {
|
||||
fmt.Printf("ERROR: closing db: %s", err)
|
||||
}
|
||||
}()
|
||||
|
||||
insertStmt, err := db.Prepare("INSERT OR IGNORE INTO eth_tx_hashes(hash, cid) VALUES(?, ?)")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var totalRowsAffected int64 = 0
|
||||
for i := 0; i < epochs; i++ {
|
||||
epoch := abi.ChainEpoch(startHeight - int64(i))
|
||||
|
||||
select {
|
||||
case <-cctx.Done():
|
||||
fmt.Println("request cancelled")
|
||||
return nil
|
||||
default:
|
||||
}
|
||||
|
||||
curTsk := curTs.Parents()
|
||||
execTs, err := api.ChainGetTipSet(ctx, curTsk)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to call ChainGetTipSet for %s: %w", curTsk, err)
|
||||
}
|
||||
|
||||
if i%100 == 0 {
|
||||
log.Infof("%d/%d processing epoch:%d", i, epochs, epoch)
|
||||
}
|
||||
|
||||
for _, blockheader := range execTs.Blocks() {
|
||||
blkMsgs, err := api.ChainGetBlockMessages(ctx, blockheader.Cid())
|
||||
if err != nil {
|
||||
log.Infof("Could not get block messages at epoch: %d, stopping walking up the chain", epoch)
|
||||
epochs = i
|
||||
break
|
||||
}
|
||||
|
||||
for _, smsg := range blkMsgs.SecpkMessages {
|
||||
if smsg.Signature.Type != crypto.SigTypeDelegated {
|
||||
continue
|
||||
}
|
||||
|
||||
tx, err := ethtypes.EthTxFromSignedEthMessage(smsg)
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to convert from signed message: %w at epoch: %d", err, epoch)
|
||||
}
|
||||
|
||||
tx.Hash, err = tx.TxHash()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to calculate hash for ethTx: %w at epoch: %d", err, epoch)
|
||||
}
|
||||
|
||||
res, err := insertStmt.Exec(tx.Hash.String(), smsg.Cid().String())
|
||||
if err != nil {
|
||||
return fmt.Errorf("error inserting tx mapping to db: %s at epoch: %d", err, epoch)
|
||||
}
|
||||
|
||||
rowsAffected, err := res.RowsAffected()
|
||||
if err != nil {
|
||||
return fmt.Errorf("error getting rows affected: %s at epoch: %d", err, epoch)
|
||||
}
|
||||
|
||||
if rowsAffected > 0 {
|
||||
log.Debugf("Inserted txhash %s, cid: %s at epoch: %d", tx.Hash.String(), smsg.Cid().String(), epoch)
|
||||
}
|
||||
|
||||
totalRowsAffected += rowsAffected
|
||||
}
|
||||
}
|
||||
|
||||
curTs = execTs
|
||||
}
|
||||
|
||||
log.Infof("Done, inserted %d missing txhashes", totalRowsAffected)
|
||||
|
||||
return nil
|
||||
},
|
||||
}
|
||||
@ -86,7 +86,7 @@ func main() {
|
||||
invariantsCmd,
|
||||
gasTraceCmd,
|
||||
replayOfflineCmd,
|
||||
msgindexCmd,
|
||||
indexesCmd,
|
||||
FevmAnalyticsCmd,
|
||||
mismatchesCmd,
|
||||
}
|
||||
|
||||
@ -2648,7 +2648,6 @@ COMMANDS:
|
||||
call Simulate an eth contract call
|
||||
contract-address Generate contract address from smart contract code
|
||||
bytecode Write the bytecode of a smart contract to a file
|
||||
backfill-txhash
|
||||
help, h Shows a list of commands or help for one command
|
||||
|
||||
OPTIONS:
|
||||
@ -2736,24 +2735,6 @@ OPTIONS:
|
||||
|
||||
```
|
||||
|
||||
### lotus evm backfill-txhash
|
||||
```
|
||||
NAME:
|
||||
lotus evm backfill-txhash
|
||||
|
||||
USAGE:
|
||||
lotus evm backfill-txhash [command options] [arguments...]
|
||||
|
||||
DESCRIPTION:
|
||||
Backfills the txhash.db for a number of epochs starting from a specified height
|
||||
|
||||
OPTIONS:
|
||||
--epochs value the number of epochs to backfill (default: 2000)
|
||||
--from value the tipset height to start backfilling from (0 is head of chain) (default: 0)
|
||||
--repo value path to the repo (default: "~/.lotus")
|
||||
|
||||
```
|
||||
|
||||
## lotus net
|
||||
```
|
||||
NAME:
|
||||
|
||||
Loading…
Reference in New Issue
Block a user