cerc-io/lotus
11
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Merge remote-tracking branch 'origin/master' into feat/worker-task-count-limits

Browse Source
This commit is contained in:
Łukasz Magiera
2022-05-25 18:25:15 +02:00
parent b90b9e97a6 6f2c8d6f5f
commit 2bdc729fe7
49 changed files with 1964 additions and 1075 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/lotus-miner/dagstore.go
+40
View File
@@ -19,6 +19,7 @@ var dagstoreCmd = &cli.Command{
Usage: "Manage the dagstore on the markets subsystem",
Subcommands: []*cli.Command{
dagstoreListShardsCmd,
dagstoreRegisterShardCmd,
dagstoreInitializeShardCmd,
dagstoreRecoverShardCmd,
dagstoreInitializeAllCmd,
@@ -59,6 +60,45 @@ var dagstoreListShardsCmd = &cli.Command{
},
}
var dagstoreRegisterShardCmd = &cli.Command{
Name: "register-shard",
ArgsUsage: "[key]",
Usage: "Register a shard",
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "color",
Usage: "use color in display output",
DefaultText: "depends on output being a TTY",
},
},
Action: func(cctx *cli.Context) error {
if cctx.IsSet("color") {
color.NoColor = !cctx.Bool("color")
}
if cctx.NArg() != 1 {
return fmt.Errorf("must provide a single shard key")
}
marketsAPI, closer, err := lcli.GetMarketsAPI(cctx)
if err != nil {
return err
}
defer closer()
ctx := lcli.ReqContext(cctx)
shardKey := cctx.Args().First()
err = marketsAPI.DagstoreRegisterShard(ctx, shardKey)
if err != nil {
return err
}
fmt.Println("Registered shard " + shardKey)
return nil
},
}
var dagstoreInitializeShardCmd = &cli.Command{
Name: "initialize-shard",
ArgsUsage: "[key]",
cmd/lotus-miner/precommits-info.go
+54
View File
@@ -0,0 +1,54 @@
package main
import (
"fmt"
"sort"
"github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
"github.com/filecoin-project/lotus/chain/types"
lcli "github.com/filecoin-project/lotus/cli"
"github.com/filecoin-project/specs-actors/v7/actors/util/adt"
cbor "github.com/ipfs/go-ipld-cbor"
"github.com/urfave/cli/v2"
)
var sectorPreCommitsCmd = &cli.Command{
Name: "precommits",
Usage: "Print on-chain precommit info",
Action: func(cctx *cli.Context) error {
ctx := lcli.ReqContext(cctx)
mapi, closer, err := lcli.GetFullNodeAPI(cctx)
if err != nil {
return err
}
defer closer()
maddr, err := getActorAddress(ctx, cctx)
if err != nil {
return err
}
mact, err := mapi.StateGetActor(ctx, maddr, types.EmptyTSK)
if err != nil {
return err
}
store := adt.WrapStore(ctx, cbor.NewCborStore(blockstore.NewAPIBlockstore(mapi)))
mst, err := miner.Load(store, mact)
if err != nil {
return err
}
preCommitSector := make([]miner.SectorPreCommitOnChainInfo, 0)
err = mst.ForEachPrecommittedSector(func(info miner.SectorPreCommitOnChainInfo) error {
preCommitSector = append(preCommitSector, info)
return err
})
less := func(i, j int) bool {
return preCommitSector[i].Info.SectorNumber <= preCommitSector[j].Info.SectorNumber
}
sort.Slice(preCommitSector, less)
for _, info := range preCommitSector {
fmt.Printf("%s: %s\n", info.Info.SectorNumber, info.PreCommitEpoch)
}
return nil
},
}
cmd/lotus-miner/sealing.go
+99
View File
@@ -4,20 +4,27 @@ import (
"encoding/hex"
"encoding/json"
"fmt"
"io"
"math"
"net/http"
"os"
"sort"
"strings"
"text/tabwriter"
"time"
"github.com/dustin/go-humanize"
"github.com/fatih/color"
"github.com/google/uuid"
"github.com/mitchellh/go-homedir"
"github.com/urfave/cli/v2"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-padreader"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/extern/sector-storage/sealtasks"
"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
"github.com/filecoin-project/lotus/lib/httpreader"
"github.com/filecoin-project/lotus/chain/types"
lcli "github.com/filecoin-project/lotus/cli"
@@ -31,6 +38,7 @@ var sealingCmd = &cli.Command{
workersCmd(true),
sealingSchedDiagCmd,
sealingAbortCmd,
sealingDataCidCmd,
},
}
@@ -411,3 +419,94 @@ var sealingAbortCmd = &cli.Command{
return nodeApi.SealingAbort(ctx, job.ID)
},
}
var sealingDataCidCmd = &cli.Command{
Name: "data-cid",
Usage: "Compute data CID using workers",
ArgsUsage: "[file/url] <padded piece size>",
Flags: []cli.Flag{
&cli.Uint64Flag{
Name: "file-size",
Usage: "real file size",
},
},
Action: func(cctx *cli.Context) error {
if cctx.Args().Len() < 1 || cctx.Args().Len() > 2 {
return xerrors.Errorf("expected 1 or 2 arguments")
}
nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx)
if err != nil {
return err
}
defer closer()
ctx := lcli.ReqContext(cctx)
var r io.Reader
sz := cctx.Uint64("file-size")
if strings.HasPrefix(cctx.Args().First(), "http://") || strings.HasPrefix(cctx.Args().First(), "https://") {
r = &httpreader.HttpReader{
URL: cctx.Args().First(),
}
if !cctx.IsSet("file-size") {
resp, err := http.Head(cctx.Args().First())
if err != nil {
return xerrors.Errorf("http head: %w", err)
}
if resp.ContentLength < 0 {
return xerrors.Errorf("head response didn't contain content length; specify --file-size")
}
sz = uint64(resp.ContentLength)
}
} else {
p, err := homedir.Expand(cctx.Args().First())
if err != nil {
return xerrors.Errorf("expanding path: %w", err)
}
f, err := os.OpenFile(p, os.O_RDONLY, 0)
if err != nil {
return xerrors.Errorf("opening source file: %w", err)
}
if !cctx.IsSet("file-size") {
st, err := f.Stat()
if err != nil {
return xerrors.Errorf("stat: %w", err)
}
sz = uint64(st.Size())
}
r = f
}
var psize abi.PaddedPieceSize
if cctx.Args().Len() == 2 {
rps, err := humanize.ParseBytes(cctx.Args().Get(1))
if err != nil {
return xerrors.Errorf("parsing piece size: %w", err)
}
psize = abi.PaddedPieceSize(rps)
if err := psize.Validate(); err != nil {
return xerrors.Errorf("checking piece size: %w", err)
}
if sz > uint64(psize.Unpadded()) {
return xerrors.Errorf("file larger than the piece")
}
} else {
psize = padreader.PaddedSize(sz).Padded()
}
pc, err := nodeApi.ComputeDataCid(ctx, psize.Unpadded(), r)
if err != nil {
return xerrors.Errorf("computing data CID: %w", err)
}
fmt.Println(pc.PieceCID, " ", pc.Size)
return nil
},
}
cmd/lotus-miner/sectors.go
+88
View File
@@ -45,6 +45,7 @@ var sectorsCmd = &cli.Command{
sectorsRefsCmd,
sectorsUpdateCmd,
sectorsPledgeCmd,
sectorPreCommitsCmd,
sectorsCheckExpireCmd,
sectorsExpiredCmd,
sectorsRenewCmd,
@@ -58,6 +59,7 @@ var sectorsCmd = &cli.Command{
sectorsCapacityCollateralCmd,
sectorsBatching,
sectorsRefreshPieceMatchingCmd,
sectorsCompactPartitionsCmd,
},
}
@@ -2088,3 +2090,89 @@ func yesno(b bool) string {
}
return color.RedString("NO")
}
var sectorsCompactPartitionsCmd = &cli.Command{
Name: "compact-partitions",
Usage: "removes dead sectors from partitions and reduces the number of partitions used if possible",
Flags: []cli.Flag{
&cli.Uint64Flag{
Name: "deadline",
Usage: "the deadline to compact the partitions in",
Required: true,
},
&cli.Int64SliceFlag{
Name: "partitions",
Usage: "list of partitions to compact sectors in",
Required: true,
},
&cli.BoolFlag{
Name: "really-do-it",
Usage: "Actually send transaction performing the action",
Value: false,
},
},
Action: func(cctx *cli.Context) error {
if !cctx.Bool("really-do-it") {
fmt.Println("Pass --really-do-it to actually execute this action")
return nil
}
api, acloser, err := lcli.GetFullNodeAPI(cctx)
if err != nil {
return err
}
defer acloser()
ctx := lcli.ReqContext(cctx)
maddr, err := getActorAddress(ctx, cctx)
if err != nil {
return err
}
minfo, err := api.StateMinerInfo(ctx, maddr, types.EmptyTSK)
if err != nil {
return err
}
deadline := cctx.Uint64("deadline")
if err != nil {
return err
}
parts := cctx.Int64Slice("partitions")
if len(parts) <= 0 {
return fmt.Errorf("must include at least one partition to compact")
}
partitions := bitfield.BitField{}
for _, partition := range parts {
partitions.Set(uint64(partition))
}
params := miner5.CompactPartitionsParams{
Deadline: deadline,
Partitions: partitions,
}
sp, err := actors.SerializeParams(&params)
if err != nil {
return xerrors.Errorf("serializing params: %w", err)
}
smsg, err := api.MpoolPushMessage(ctx, &types.Message{
From: minfo.Worker,
To: maddr,
Method: miner.Methods.CompactPartitions,
Value: big.Zero(),
Params: sp,
}, nil)
if err != nil {
return xerrors.Errorf("mpool push: %w", err)
}
fmt.Println("Message CID:", smsg.Cid())
return nil
},
}
cmd/lotus-shed/shedgen/cbor_gen.go
+19 -13
View File
@@ -23,18 +23,19 @@ func (t *CarbNode) MarshalCBOR(w io.Writer) error {
_, err := w.Write(cbg.CborNull)
return err
}
if _, err := w.Write([]byte{161}); err != nil {
cw := cbg.NewCborWriter(w)
if _, err := cw.Write([]byte{161}); err != nil {
return err
}
scratch := make([]byte, 9)
// t.Sub ([]cid.Cid) (slice)
if len("Sub") > cbg.MaxLength {
return xerrors.Errorf("Value in field \"Sub\" was too long")
}
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Sub"))); err != nil {
if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Sub"))); err != nil {
return err
}
if _, err := io.WriteString(w, string("Sub")); err != nil {
@@ -45,27 +46,32 @@ func (t *CarbNode) MarshalCBOR(w io.Writer) error {
return xerrors.Errorf("Slice value in field t.Sub was too long")
}
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.Sub))); err != nil {
if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Sub))); err != nil {
return err
}
for _, v := range t.Sub {
if err := cbg.WriteCidBuf(scratch, w, v); err != nil {
if err := cbg.WriteCid(w, v); err != nil {
return xerrors.Errorf("failed writing cid field t.Sub: %w", err)
}
}
return nil
}
func (t *CarbNode) UnmarshalCBOR(r io.Reader) error {
func (t *CarbNode) UnmarshalCBOR(r io.Reader) (err error) {
*t = CarbNode{}
br := cbg.GetPeeker(r)
scratch := make([]byte, 8)
cr := cbg.NewCborReader(r)
maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
maj, extra, err := cr.ReadHeader()
if err != nil {
return err
}
defer func() {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
}()
if maj != cbg.MajMap {
return fmt.Errorf("cbor input should be of type map")
}
@@ -80,7 +86,7 @@ func (t *CarbNode) UnmarshalCBOR(r io.Reader) error {
for i := uint64(0); i < n; i++ {
{
sval, err := cbg.ReadStringBuf(br, scratch)
sval, err := cbg.ReadString(cr)
if err != nil {
return err
}
@@ -92,7 +98,7 @@ func (t *CarbNode) UnmarshalCBOR(r io.Reader) error {
// t.Sub ([]cid.Cid) (slice)
case "Sub":
maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
maj, extra, err = cr.ReadHeader()
if err != nil {
return err
}
@@ -111,7 +117,7 @@ func (t *CarbNode) UnmarshalCBOR(r io.Reader) error {
for i := 0; i < int(extra); i++ {
c, err := cbg.ReadCid(br)
c, err := cbg.ReadCid(cr)
if err != nil {
return xerrors.Errorf("reading cid field t.Sub failed: %w", err)
}
cmd/lotus-shed/verifreg.go
+5 -5
View File
@@ -193,7 +193,7 @@ var verifRegVerifyClientCmd = &cli.Command{
},
},
Action: func(cctx *cli.Context) error {
fmt.Println("DEPRECATED: This behavior is being moved to `lotus verifreg`")
fmt.Println("DEPRECATED: This behavior is being moved to `lotus filplus`")
froms := cctx.String("from")
if froms == "" {
return fmt.Errorf("must specify from address with --from")
@@ -262,7 +262,7 @@ var verifRegListVerifiersCmd = &cli.Command{
Usage: "list all verifiers",
Hidden: true,
Action: func(cctx *cli.Context) error {
fmt.Println("DEPRECATED: This behavior is being moved to `lotus verifreg`")
fmt.Println("DEPRECATED: This behavior is being moved to `lotus filplus`")
api, closer, err := lcli.GetFullNodeAPI(cctx)
if err != nil {
return err
@@ -294,7 +294,7 @@ var verifRegListClientsCmd = &cli.Command{
Usage: "list all verified clients",
Hidden: true,
Action: func(cctx *cli.Context) error {
fmt.Println("DEPRECATED: This behavior is being moved to `lotus verifreg`")
fmt.Println("DEPRECATED: This behavior is being moved to `lotus filplus`")
api, closer, err := lcli.GetFullNodeAPI(cctx)
if err != nil {
return err
@@ -326,7 +326,7 @@ var verifRegCheckClientCmd = &cli.Command{
Usage: "check verified client remaining bytes",
Hidden: true,
Action: func(cctx *cli.Context) error {
fmt.Println("DEPRECATED: This behavior is being moved to `lotus verifreg`")
fmt.Println("DEPRECATED: This behavior is being moved to `lotus filplus`")
if !cctx.Args().Present() {
return fmt.Errorf("must specify client address to check")
}
@@ -362,7 +362,7 @@ var verifRegCheckVerifierCmd = &cli.Command{
Usage: "check verifiers remaining bytes",
Hidden: true,
Action: func(cctx *cli.Context) error {
fmt.Println("DEPRECATED: This behavior is being moved to `lotus verifreg`")
fmt.Println("DEPRECATED: This behavior is being moved to `lotus filplus`")
if !cctx.Args().Present() {
return fmt.Errorf("must specify verifier address to check")
}
cmd/lotus-worker/main.go
+13 -2
View File
@@ -182,12 +182,16 @@ var runCmd = &cli.Command{
Usage: "enable window post",
Value: false,
},
&cli.BoolFlag{
Name: "winningpost",
Usage: "enable winning post",
Value: false,
},
&cli.BoolFlag{
Name: "no-default",
Usage: "disable all default compute tasks, use the worker for storage/fetching only",
Value: false,
},
&cli.IntFlag{
Name: "parallel-fetch-limit",
Usage: "maximum fetch operations to run in parallel",
@@ -308,8 +312,11 @@ var runCmd = &cli.Command{
}
if workerType == "" {
workerType = sealtasks.WorkerSealing
taskTypes = append(taskTypes, sealtasks.TTFetch, sealtasks.TTCommit1, sealtasks.TTProveReplicaUpdate1, sealtasks.TTFinalize, sealtasks.TTFinalizeReplicaUpdate)
if !cctx.Bool("no-default") {
workerType = sealtasks.WorkerSealing
}
}
if (workerType == sealtasks.WorkerSealing || cctx.IsSet("addpiece")) && cctx.Bool("addpiece") {
@@ -337,6 +344,10 @@ var runCmd = &cli.Command{
taskTypes = append(taskTypes, sealtasks.TTRegenSectorKey)
}
if cctx.Bool("no-default") && workerType == "" {
workerType = sealtasks.WorkerSealing
}
if len(taskTypes) == 0 {
return xerrors.Errorf("no task types specified")
}