cerc-io/lotus
11
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Merge branch 'master' into rate-limit-gateway

Browse Source
This commit is contained in:
Cory Schwartz
2022-06-01 00:58:23 -07:00
parent 44264dccba d1a241e5b7
commit e645e8aa7c
105 changed files with 5066 additions and 2070 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/lotus-bench/main.go
+1
View File
@@ -109,6 +109,7 @@ func main() {
Commands: []*cli.Command{
proveCmd,
sealBenchCmd,
simpleCmd,
importBenchCmd,
},
}
cmd/lotus-bench/simple.go
+990
View File
@@ -0,0 +1,990 @@
package main
import (
"context"
"encoding/base64"
"encoding/json"
"fmt"
"io/ioutil"
"os"
"strconv"
"time"
"github.com/docker/go-units"
"github.com/ipfs/go-cid"
"github.com/urfave/cli/v2"
"golang.org/x/xerrors"
ffi "github.com/filecoin-project/filecoin-ffi"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-paramfetch"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/build"
lcli "github.com/filecoin-project/lotus/cli"
"github.com/filecoin-project/lotus/extern/sector-storage/ffiwrapper"
"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
prf "github.com/filecoin-project/specs-actors/actors/runtime/proof"
"github.com/filecoin-project/specs-storage/storage"
)
var simpleCmd = &cli.Command{
Name: "simple",
Usage: "Run basic sector operations",
Description: `Example sealing steps:
> Create unsealed sector file
$ ./lotus-bench simple addpiece --sector-size 2K /dev/zero /tmp/unsealed
AddPiece 25.23225ms (79.26 KiB/s)
baga6ea4seaqpy7usqklokfx2vxuynmupslkeutzexe2uqurdg5vhtebhxqmpqmy 2048
> Run PreCommit1
$ ./lotus-bench simple precommit1 --sector-size 2k /tmp/unsealed /tmp/sealed /tmp/cache baga6ea4seaqpy7usqklokfx2vxuynmupslkeutzexe2uqurdg5vhtebhxqmpqmy 2048
PreCommit1 30.151666ms (66.33 KiB/s)
eyJfbG90dXNfU2VhbFJhbmRvbW5lc3MiOiJBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFB[...]==
> Run PreCommit2
$ ./lotus-bench simple precommit2 --sector-size 2k /tmp/sealed /tmp/cache eyJfbG90dXNfU2VhbFJhbmRvbW5lc3MiOiJBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFB[...]==
PreCommit2 75.320167ms (26.55 KiB/s)
d:baga6ea4seaqpy7usqklokfx2vxuynmupslkeutzexe2uqurdg5vhtebhxqmpqmy r:bagboea4b5abcbrshxgmmpaucffwp2elaofbcrvb7hmcu3653o4lsw2arlor4hn3c
> Run Commit1
$ ./lotus-bench simple commit1 --sector-size 2k /tmp/sl /tmp/cac baga6ea4seaqpy7usqklokfx2vxuynmupslkeutzexe2uqurdg5vhtebhxqmpqmy bagboea4b5abcbrshxgmmpaucffwp2elaofbcrvb7hmcu3653o4lsw2arlor4hn3c /tmp/c1.json
Commit1 20.691875ms (96.66 KiB/s)
> Run Commit2
$ ./lotus-bench simple commit2 /tmp/c1.json
[...]
Commit2: 13.829147792s (148 B/s)
proof: 8b624a6a4b272a6196517f858d07205c586cfae77fc026e8e9340acefbb8fc1d5af25b33724756c0a4481a800e14ff1ea914c3ce20bf6e2932296ad8ffa32867989ceae62e50af1479ca56a1ea5228cc8acf5ca54bc0b8e452bf74194b758b2c12ece76599a8b93f6b3dd9f0b1bb2e023bf311e9a404c7d453aeddf284e46025b63b631610de6ff6621bc6f630a14dd3ad59edbe6e940fdebbca3d97bea2708fd21764ea929f4699ebc93d818037a74be3363bdb2e8cc29b3e386c6376ff98fa
----
Example PoSt steps:
> Try window-post
$ ./lotus-bench simple window-post --sector-size 2k /tmp/sealed /tmp/cache bagboea4b5abcbrshxgmmpaucffwp2elaofbcrvb7hmcu3653o4lsw2arlor4hn3c 1
Vanilla 14.192625ms (140.9 KiB/s)
Proof 387.819333ms (5.156 KiB/s)
mI6TdveK9wMqHwVsRlVa90q44yGEIsNqLpTQLB...
> Try winning-post
$ ./lotus-bench simple winning-post --sector-size 2k /tmp/sealed /tmp/cache bagboea4b5abcbrshxgmmpaucffwp2elaofbcrvb7hmcu3653o4lsw2arlor4hn3c 1
Vanilla 19.266625ms (103.8 KiB/s)
Proof 1.234634708s (1.619 KiB/s)
o4VBUf2wBHuvmm58XY8wgCC/1xBqfujlgmNs...
----
Example SnapDeals steps:
> Create unsealed update file
$ ./lotus-bench simple addpiece --sector-size 2K /dev/random /tmp/new-unsealed
AddPiece 23.845958ms (83.87 KiB/s)
baga6ea4seaqkt24j5gbf2ye2wual5gn7a5yl2tqb52v2sk4nvur4bdy7lg76cdy 2048
> Create updated sealed file
$ ./lotus-bench simple replicaupdate --sector-size 2K /tmp/sealed /tmp/cache /tmp/new-unsealed /tmp/update /tmp/update-cache baga6ea4seaqkt24j5gbf2ye2wual5gn7a5yl2tqb52v2sk4nvur4bdy7lg76cdy 2048
ReplicaUpdate 63.0815ms (31.7 KiB/s)
d:baga6ea4seaqkt24j5gbf2ye2wual5gn7a5yl2tqb52v2sk4nvur4bdy7lg76cdy r:bagboea4b5abcaydcwlbtdx5dph2a3efpqt42emxpn3be76iu4e4lx3ltrpmpi7af
> Run ProveReplicaUpdate1
$ ./lotus-bench simple provereplicaupdate1 --sector-size 2K /tmp/sl /tmp/cac /tmp/update /tmp/update-cache bagboea4b5abcbrshxgmmpaucffwp2elaofbcrvb7hmcu3653o4lsw2arlor4hn3c bagboea4b5abcaydcwlbtdx5dph2a3efpqt42emxpn3be76iu4e4lx3ltrpmpi7af baga6ea4seaqkt24j5gbf2ye2wual5gn7a5yl2tqb52v2sk4nvur4bdy7lg76cdy /tmp/pr1.json
ProveReplicaUpdate1 18.373375ms (108.9 KiB/s)
> Run ProveReplicaUpdate2
$ ./lotus-bench simple provereplicaupdate2 --sector-size 2K bagboea4b5abcbrshxgmmpaucffwp2elaofbcrvb7hmcu3653o4lsw2arlor4hn3c bagboea4b5abcaydcwlbtdx5dph2a3efpqt42emxpn3be76iu4e4lx3ltrpmpi7af baga6ea4seaqkt24j5gbf2ye2wual5gn7a5yl2tqb52v2sk4nvur4bdy7lg76cdy /tmp/pr1.json
ProveReplicaUpdate2 7.339033459s (279 B/s)
p: pvC0JBrEyUqtIIUvB2UUx/2a24c3Cvnu6AZ0D3IMBYAu...
`,
Subcommands: []*cli.Command{
simpleAddPiece,
simplePreCommit1,
simplePreCommit2,
simpleCommit1,
simpleCommit2,
simpleWindowPost,
simpleWinningPost,
simpleReplicaUpdate,
simpleProveReplicaUpdate1,
simpleProveReplicaUpdate2,
},
}
type benchSectorProvider map[storiface.SectorFileType]string
func (b benchSectorProvider) AcquireSector(ctx context.Context, id storage.SectorRef, existing storiface.SectorFileType, allocate storiface.SectorFileType, ptype storiface.PathType) (storiface.SectorPaths, func(), error) {
out := storiface.SectorPaths{
ID: id.ID,
Unsealed: b[storiface.FTUnsealed],
Sealed: b[storiface.FTSealed],
Cache: b[storiface.FTCache],
Update: b[storiface.FTUpdate],
UpdateCache: b[storiface.FTUpdateCache],
}
return out, func() {}, nil
}
var _ ffiwrapper.SectorProvider = &benchSectorProvider{}
var simpleAddPiece = &cli.Command{
Name: "addpiece",
ArgsUsage: "[data] [unsealed]",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "sector-size",
Value: "512MiB",
Usage: "size of the sectors in bytes, i.e. 32GiB",
},
&cli.StringFlag{
Name: "miner-addr",
Usage: "pass miner address (only necessary if using existing sectorbuilder)",
Value: "t01000",
},
},
Action: func(cctx *cli.Context) error {
ctx := cctx.Context
maddr, err := address.NewFromString(cctx.String("miner-addr"))
if err != nil {
return err
}
amid, err := address.IDFromAddress(maddr)
if err != nil {
return err
}
mid := abi.ActorID(amid)
sectorSizeInt, err := units.RAMInBytes(cctx.String("sector-size"))
if err != nil {
return err
}
sectorSize := abi.SectorSize(sectorSizeInt)
pp := benchSectorProvider{
storiface.FTUnsealed: cctx.Args().Get(1),
}
sealer, err := ffiwrapper.New(pp)
if err != nil {
return err
}
sr := storage.SectorRef{
ID: abi.SectorID{
Miner: mid,
Number: 1,
},
ProofType: spt(sectorSize),
}
data, err := os.Open(cctx.Args().First())
if err != nil {
return xerrors.Errorf("open data file: %w", err)
}
start := time.Now()
pi, err := sealer.AddPiece(ctx, sr, []abi.UnpaddedPieceSize{}, abi.PaddedPieceSize(sectorSize).Unpadded(), data)
if err != nil {
return xerrors.Errorf("add piece: %w", err)
}
took := time.Now().Sub(start)
fmt.Printf("AddPiece %s (%s)\n", took, bps(abi.SectorSize(pi.Size), 1, took))
fmt.Printf("%s %d\n", pi.PieceCID, pi.Size)
return nil
},
}
var simplePreCommit1 = &cli.Command{
Name: "precommit1",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "sector-size",
Value: "512MiB",
Usage: "size of the sectors in bytes, i.e. 32GiB",
},
&cli.StringFlag{
Name: "miner-addr",
Usage: "pass miner address (only necessary if using existing sectorbuilder)",
Value: "t01000",
},
},
ArgsUsage: "[unsealed] [sealed] [cache] [[piece cid] [piece size]]...",
Action: func(cctx *cli.Context) error {
ctx := cctx.Context
maddr, err := address.NewFromString(cctx.String("miner-addr"))
if err != nil {
return err
}
amid, err := address.IDFromAddress(maddr)
if err != nil {
return err
}
mid := abi.ActorID(amid)
sectorSizeInt, err := units.RAMInBytes(cctx.String("sector-size"))
if err != nil {
return err
}
sectorSize := abi.SectorSize(sectorSizeInt)
pp := benchSectorProvider{
storiface.FTUnsealed: cctx.Args().Get(0),
storiface.FTSealed: cctx.Args().Get(1),
storiface.FTCache: cctx.Args().Get(2),
}
sealer, err := ffiwrapper.New(pp)
if err != nil {
return err
}
sr := storage.SectorRef{
ID: abi.SectorID{
Miner: mid,
Number: 1,
},
ProofType: spt(sectorSize),
}
var ticket [32]byte // all zero
pieces, err := ParsePieceInfos(cctx, 3)
if err != nil {
return err
}
start := time.Now()
p1o, err := sealer.SealPreCommit1(ctx, sr, ticket[:], pieces)
if err != nil {
return xerrors.Errorf("precommit1: %w", err)
}
took := time.Now().Sub(start)
fmt.Printf("PreCommit1 %s (%s)\n", took, bps(sectorSize, 1, took))
fmt.Println(base64.StdEncoding.EncodeToString(p1o))
return nil
},
}
var simplePreCommit2 = &cli.Command{
Name: "precommit2",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "sector-size",
Value: "512MiB",
Usage: "size of the sectors in bytes, i.e. 32GiB",
},
&cli.StringFlag{
Name: "miner-addr",
Usage: "pass miner address (only necessary if using existing sectorbuilder)",
Value: "t01000",
},
},
ArgsUsage: "[sealed] [cache] [pc1 out]",
Action: func(cctx *cli.Context) error {
ctx := cctx.Context
maddr, err := address.NewFromString(cctx.String("miner-addr"))
if err != nil {
return err
}
amid, err := address.IDFromAddress(maddr)
if err != nil {
return err
}
mid := abi.ActorID(amid)
sectorSizeInt, err := units.RAMInBytes(cctx.String("sector-size"))
if err != nil {
return err
}
sectorSize := abi.SectorSize(sectorSizeInt)
pp := benchSectorProvider{
storiface.FTSealed: cctx.Args().Get(0),
storiface.FTCache: cctx.Args().Get(1),
}
sealer, err := ffiwrapper.New(pp)
if err != nil {
return err
}
p1o, err := base64.StdEncoding.DecodeString(cctx.Args().Get(2))
if err != nil {
return err
}
sr := storage.SectorRef{
ID: abi.SectorID{
Miner: mid,
Number: 1,
},
ProofType: spt(sectorSize),
}
start := time.Now()
p2o, err := sealer.SealPreCommit2(ctx, sr, p1o)
if err != nil {
return xerrors.Errorf("precommit2: %w", err)
}
took := time.Now().Sub(start)
fmt.Printf("PreCommit2 %s (%s)\n", took, bps(sectorSize, 1, took))
fmt.Printf("d:%s r:%s\n", p2o.Unsealed, p2o.Sealed)
return nil
},
}
var simpleCommit1 = &cli.Command{
Name: "commit1",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "sector-size",
Value: "512MiB",
Usage: "size of the sectors in bytes, i.e. 32GiB",
},
&cli.StringFlag{
Name: "miner-addr",
Usage: "pass miner address (only necessary if using existing sectorbuilder)",
Value: "t01000",
},
},
ArgsUsage: "[sealed] [cache] [comm D] [comm R] [c1out.json]",
Action: func(cctx *cli.Context) error {
ctx := cctx.Context
maddr, err := address.NewFromString(cctx.String("miner-addr"))
if err != nil {
return err
}
amid, err := address.IDFromAddress(maddr)
if err != nil {
return err
}
mid := abi.ActorID(amid)
sectorSizeInt, err := units.RAMInBytes(cctx.String("sector-size"))
if err != nil {
return err
}
sectorSize := abi.SectorSize(sectorSizeInt)
pp := benchSectorProvider{
storiface.FTSealed: cctx.Args().Get(0),
storiface.FTCache: cctx.Args().Get(1),
}
sealer, err := ffiwrapper.New(pp)
if err != nil {
return err
}
sr := storage.SectorRef{
ID: abi.SectorID{
Miner: mid,
Number: 1,
},
ProofType: spt(sectorSize),
}
start := time.Now()
var ticket, seed [32]byte // all zero
commd, err := cid.Parse(cctx.Args().Get(2))
if err != nil {
return xerrors.Errorf("parse commr: %w", err)
}
commr, err := cid.Parse(cctx.Args().Get(3))
if err != nil {
return xerrors.Errorf("parse commr: %w", err)
}
c1o, err := sealer.SealCommit1(ctx, sr, ticket[:], seed[:], []abi.PieceInfo{
{
Size: abi.PaddedPieceSize(sectorSize),
PieceCID: commd,
},
}, storage.SectorCids{
Unsealed: commd,
Sealed: commr,
})
if err != nil {
return xerrors.Errorf("commit1: %w", err)
}
took := time.Now().Sub(start)
fmt.Printf("Commit1 %s (%s)\n", took, bps(sectorSize, 1, took))
c2in := Commit2In{
SectorNum: int64(1),
Phase1Out: c1o,
SectorSize: uint64(sectorSize),
}
b, err := json.Marshal(&c2in)
if err != nil {
return err
}
if err := ioutil.WriteFile(cctx.Args().Get(4), b, 0664); err != nil {
log.Warnf("%+v", err)
}
return nil
},
}
var simpleCommit2 = &cli.Command{
Name: "commit2",
ArgsUsage: "[c1out.json]",
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "no-gpu",
Usage: "disable gpu usage for the benchmark run",
},
&cli.StringFlag{
Name: "miner-addr",
Usage: "pass miner address (only necessary if using existing sectorbuilder)",
Value: "t01000",
},
},
Action: func(c *cli.Context) error {
if c.Bool("no-gpu") {
err := os.Setenv("BELLMAN_NO_GPU", "1")
if err != nil {
return xerrors.Errorf("setting no-gpu flag: %w", err)
}
}
if !c.Args().Present() {
return xerrors.Errorf("Usage: lotus-bench prove [input.json]")
}
inb, err := ioutil.ReadFile(c.Args().First())
if err != nil {
return xerrors.Errorf("reading input file: %w", err)
}
var c2in Commit2In
if err := json.Unmarshal(inb, &c2in); err != nil {
return xerrors.Errorf("unmarshalling input file: %w", err)
}
if err := paramfetch.GetParams(lcli.ReqContext(c), build.ParametersJSON(), build.SrsJSON(), c2in.SectorSize); err != nil {
return xerrors.Errorf("getting params: %w", err)
}
maddr, err := address.NewFromString(c.String("miner-addr"))
if err != nil {
return err
}
mid, err := address.IDFromAddress(maddr)
if err != nil {
return err
}
sb, err := ffiwrapper.New(nil)
if err != nil {
return err
}
ref := storage.SectorRef{
ID: abi.SectorID{
Miner: abi.ActorID(mid),
Number: abi.SectorNumber(c2in.SectorNum),
},
ProofType: spt(abi.SectorSize(c2in.SectorSize)),
}
start := time.Now()
proof, err := sb.SealCommit2(context.TODO(), ref, c2in.Phase1Out)
if err != nil {
return err
}
sealCommit2 := time.Now()
dur := sealCommit2.Sub(start)
fmt.Printf("Commit2: %s (%s)\n", dur, bps(abi.SectorSize(c2in.SectorSize), 1, dur))
fmt.Printf("proof: %x\n", proof)
return nil
},
}
var simpleWindowPost = &cli.Command{
Name: "window-post",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "sector-size",
Value: "512MiB",
Usage: "size of the sectors in bytes, i.e. 32GiB",
},
&cli.StringFlag{
Name: "miner-addr",
Usage: "pass miner address (only necessary if using existing sectorbuilder)",
Value: "t01000",
},
},
ArgsUsage: "[sealed] [cache] [comm R] [sector num]",
Action: func(cctx *cli.Context) error {
maddr, err := address.NewFromString(cctx.String("miner-addr"))
if err != nil {
return err
}
amid, err := address.IDFromAddress(maddr)
if err != nil {
return err
}
mid := abi.ActorID(amid)
sectorSizeInt, err := units.RAMInBytes(cctx.String("sector-size"))
if err != nil {
return err
}
sectorSize := abi.SectorSize(sectorSizeInt)
var rand [32]byte // all zero
commr, err := cid.Parse(cctx.Args().Get(2))
if err != nil {
return xerrors.Errorf("parse commr: %w", err)
}
wpt, err := spt(sectorSize).RegisteredWindowPoStProof()
if err != nil {
return err
}
snum, err := strconv.ParseUint(cctx.Args().Get(3), 10, 64)
if err != nil {
return xerrors.Errorf("parsing sector num: %w", err)
}
sn := abi.SectorNumber(snum)
ch, err := ffi.GeneratePoStFallbackSectorChallenges(wpt, mid, rand[:], []abi.SectorNumber{sn})
if err != nil {
return xerrors.Errorf("generating challenges: %w", err)
}
start := time.Now()
vp, err := ffi.GenerateSingleVanillaProof(ffi.PrivateSectorInfo{
SectorInfo: prf.SectorInfo{
SealProof: spt(sectorSize),
SectorNumber: sn,
SealedCID: commr,
},
CacheDirPath: cctx.Args().Get(1),
PoStProofType: wpt,
SealedSectorPath: cctx.Args().Get(0),
}, ch.Challenges[sn])
if err != nil {
return err
}
challenge := time.Now()
proof, err := ffi.GenerateSinglePartitionWindowPoStWithVanilla(wpt, mid, rand[:], [][]byte{vp}, 0)
if err != nil {
return xerrors.Errorf("generate post: %w", err)
}
end := time.Now()
fmt.Printf("Vanilla %s (%s)\n", challenge.Sub(start), bps(sectorSize, 1, challenge.Sub(start)))
fmt.Printf("Proof %s (%s)\n", end.Sub(challenge), bps(sectorSize, 1, end.Sub(challenge)))
fmt.Println(base64.StdEncoding.EncodeToString(proof.ProofBytes))
return nil
},
}
var simpleWinningPost = &cli.Command{
Name: "winning-post",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "sector-size",
Value: "512MiB",
Usage: "size of the sectors in bytes, i.e. 32GiB",
},
&cli.StringFlag{
Name: "miner-addr",
Usage: "pass miner address (only necessary if using existing sectorbuilder)",
Value: "t01000",
},
},
ArgsUsage: "[sealed] [cache] [comm R] [sector num]",
Action: func(cctx *cli.Context) error {
maddr, err := address.NewFromString(cctx.String("miner-addr"))
if err != nil {
return err
}
amid, err := address.IDFromAddress(maddr)
if err != nil {
return err
}
mid := abi.ActorID(amid)
sectorSizeInt, err := units.RAMInBytes(cctx.String("sector-size"))
if err != nil {
return err
}
sectorSize := abi.SectorSize(sectorSizeInt)
var rand [32]byte // all zero
commr, err := cid.Parse(cctx.Args().Get(2))
if err != nil {
return xerrors.Errorf("parse commr: %w", err)
}
wpt, err := spt(sectorSize).RegisteredWinningPoStProof()
if err != nil {
return err
}
snum, err := strconv.ParseUint(cctx.Args().Get(3), 10, 64)
if err != nil {
return xerrors.Errorf("parsing sector num: %w", err)
}
sn := abi.SectorNumber(snum)
ch, err := ffi.GeneratePoStFallbackSectorChallenges(wpt, mid, rand[:], []abi.SectorNumber{sn})
if err != nil {
return xerrors.Errorf("generating challenges: %w", err)
}
start := time.Now()
vp, err := ffi.GenerateSingleVanillaProof(ffi.PrivateSectorInfo{
SectorInfo: prf.SectorInfo{
SealProof: spt(sectorSize),
SectorNumber: sn,
SealedCID: commr,
},
CacheDirPath: cctx.Args().Get(1),
PoStProofType: wpt,
SealedSectorPath: cctx.Args().Get(0),
}, ch.Challenges[sn])
if err != nil {
return err
}
challenge := time.Now()
proof, err := ffi.GenerateWinningPoStWithVanilla(wpt, mid, rand[:], [][]byte{vp})
if err != nil {
return xerrors.Errorf("generate post: %w", err)
}
end := time.Now()
fmt.Printf("Vanilla %s (%s)\n", challenge.Sub(start), bps(sectorSize, 1, challenge.Sub(start)))
fmt.Printf("Proof %s (%s)\n", end.Sub(challenge), bps(sectorSize, 1, end.Sub(challenge)))
fmt.Println(base64.StdEncoding.EncodeToString(proof[0].ProofBytes))
return nil
},
}
var simpleReplicaUpdate = &cli.Command{
Name: "replicaupdate",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "sector-size",
Value: "512MiB",
Usage: "size of the sectors in bytes, i.e. 32GiB",
},
&cli.StringFlag{
Name: "miner-addr",
Usage: "pass miner address (only necessary if using existing sectorbuilder)",
Value: "t01000",
},
},
ArgsUsage: "[sealed] [cache] [unsealed] [update] [updatecache] [[piece cid] [piece size]]...",
Action: func(cctx *cli.Context) error {
ctx := cctx.Context
maddr, err := address.NewFromString(cctx.String("miner-addr"))
if err != nil {
return err
}
amid, err := address.IDFromAddress(maddr)
if err != nil {
return err
}
mid := abi.ActorID(amid)
sectorSizeInt, err := units.RAMInBytes(cctx.String("sector-size"))
if err != nil {
return err
}
sectorSize := abi.SectorSize(sectorSizeInt)
pp := benchSectorProvider{
storiface.FTSealed: cctx.Args().Get(0),
storiface.FTCache: cctx.Args().Get(1),
storiface.FTUnsealed: cctx.Args().Get(2),
storiface.FTUpdate: cctx.Args().Get(3),
storiface.FTUpdateCache: cctx.Args().Get(4),
}
sealer, err := ffiwrapper.New(pp)
if err != nil {
return err
}
pieces, err := ParsePieceInfos(cctx, 5)
if err != nil {
return err
}
sr := storage.SectorRef{
ID: abi.SectorID{
Miner: mid,
Number: 1,
},
ProofType: spt(sectorSize),
}
start := time.Now()
ruo, err := sealer.ReplicaUpdate(ctx, sr, pieces)
if err != nil {
return xerrors.Errorf("replica update: %w", err)
}
took := time.Now().Sub(start)
fmt.Printf("ReplicaUpdate %s (%s)\n", took, bps(sectorSize, 1, took))
fmt.Printf("d:%s r:%s\n", ruo.NewUnsealed, ruo.NewSealed)
return nil
},
}
var simpleProveReplicaUpdate1 = &cli.Command{
Name: "provereplicaupdate1",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "sector-size",
Value: "512MiB",
Usage: "size of the sectors in bytes, i.e. 32GiB",
},
&cli.StringFlag{
Name: "miner-addr",
Usage: "pass miner address (only necessary if using existing sectorbuilder)",
Value: "t01000",
},
},
ArgsUsage: "[sealed] [cache] [update] [updatecache] [sectorKey] [newSealed] [newUnsealed] [vproofs.json]",
Action: func(cctx *cli.Context) error {
ctx := cctx.Context
maddr, err := address.NewFromString(cctx.String("miner-addr"))
if err != nil {
return err
}
amid, err := address.IDFromAddress(maddr)
if err != nil {
return err
}
mid := abi.ActorID(amid)
sectorSizeInt, err := units.RAMInBytes(cctx.String("sector-size"))
if err != nil {
return err
}
sectorSize := abi.SectorSize(sectorSizeInt)
pp := benchSectorProvider{
storiface.FTSealed: cctx.Args().Get(0),
storiface.FTCache: cctx.Args().Get(1),
storiface.FTUpdate: cctx.Args().Get(2),
storiface.FTUpdateCache: cctx.Args().Get(3),
}
sealer, err := ffiwrapper.New(pp)
if err != nil {
return err
}
sr := storage.SectorRef{
ID: abi.SectorID{
Miner: mid,
Number: 1,
},
ProofType: spt(sectorSize),
}
start := time.Now()
oldcommr, err := cid.Parse(cctx.Args().Get(4))
if err != nil {
return xerrors.Errorf("parse commr: %w", err)
}
commr, err := cid.Parse(cctx.Args().Get(5))
if err != nil {
return xerrors.Errorf("parse commr: %w", err)
}
commd, err := cid.Parse(cctx.Args().Get(6))
if err != nil {
return xerrors.Errorf("parse commr: %w", err)
}
rvp, err := sealer.ProveReplicaUpdate1(ctx, sr, oldcommr, commr, commd)
if err != nil {
return xerrors.Errorf("replica update: %w", err)
}
took := time.Now().Sub(start)
fmt.Printf("ProveReplicaUpdate1 %s (%s)\n", took, bps(sectorSize, 1, took))
vpjb, err := json.Marshal(&rvp)
if err != nil {
return xerrors.Errorf("json marshal vanilla proofs: %w", err)
}
if err := ioutil.WriteFile(cctx.Args().Get(7), vpjb, 0666); err != nil {
return xerrors.Errorf("writing vanilla proofs file: %w", err)
}
return nil
},
}
var simpleProveReplicaUpdate2 = &cli.Command{
Name: "provereplicaupdate2",
Flags: []cli.Flag{
&cli.StringFlag{
Name: "sector-size",
Value: "512MiB",
Usage: "size of the sectors in bytes, i.e. 32GiB",
},
&cli.StringFlag{
Name: "miner-addr",
Usage: "pass miner address (only necessary if using existing sectorbuilder)",
Value: "t01000",
},
},
ArgsUsage: "[sectorKey] [newSealed] [newUnsealed] [vproofs.json]",
Action: func(cctx *cli.Context) error {
ctx := cctx.Context
maddr, err := address.NewFromString(cctx.String("miner-addr"))
if err != nil {
return err
}
amid, err := address.IDFromAddress(maddr)
if err != nil {
return err
}
mid := abi.ActorID(amid)
sectorSizeInt, err := units.RAMInBytes(cctx.String("sector-size"))
if err != nil {
return err
}
sectorSize := abi.SectorSize(sectorSizeInt)
pp := benchSectorProvider{}
sealer, err := ffiwrapper.New(pp)
if err != nil {
return err
}
sr := storage.SectorRef{
ID: abi.SectorID{
Miner: mid,
Number: 1,
},
ProofType: spt(sectorSize),
}
start := time.Now()
oldcommr, err := cid.Parse(cctx.Args().Get(0))
if err != nil {
return xerrors.Errorf("parse commr: %w", err)
}
commr, err := cid.Parse(cctx.Args().Get(1))
if err != nil {
return xerrors.Errorf("parse commr: %w", err)
}
commd, err := cid.Parse(cctx.Args().Get(2))
if err != nil {
return xerrors.Errorf("parse commr: %w", err)
}
vpb, err := ioutil.ReadFile(cctx.Args().Get(3))
if err != nil {
return xerrors.Errorf("reading valilla proof file: %w", err)
}
var vp storage.ReplicaVanillaProofs
if err := json.Unmarshal(vpb, &vp); err != nil {
return xerrors.Errorf("unmarshalling vanilla proofs: %w", err)
}
p, err := sealer.ProveReplicaUpdate2(ctx, sr, oldcommr, commr, commd, vp)
if err != nil {
return xerrors.Errorf("prove replica update2: %w", err)
}
took := time.Now().Sub(start)
fmt.Printf("ProveReplicaUpdate2 %s (%s)\n", took, bps(sectorSize, 1, took))
fmt.Println("p:", base64.StdEncoding.EncodeToString(p))
return nil
},
}
func ParsePieceInfos(cctx *cli.Context, firstArg int) ([]abi.PieceInfo, error) {
args := cctx.Args().Len() - firstArg
if args%2 != 0 {
return nil, xerrors.Errorf("piece info argunemts need to be supplied in pairs")
}
if args < 2 {
return nil, xerrors.Errorf("need at least one piece info argument")
}
out := make([]abi.PieceInfo, args/2)
for i := 0; i < args/2; i++ {
c, err := cid.Parse(cctx.Args().Get(firstArg + (i * 2)))
if err != nil {
return nil, xerrors.Errorf("parse piece cid: %w", err)
}
psize, err := strconv.ParseUint(cctx.Args().Get(firstArg+(i*2)+1), 10, 64)
if err != nil {
return nil, xerrors.Errorf("parse piece size: %w", err)
}
out[i] = abi.PieceInfo{
Size: abi.PaddedPieceSize(psize),
PieceCID: c,
}
}
return out, nil
}
cmd/lotus-gateway/main.go
+1 -1
View File
@@ -198,7 +198,7 @@ var runCmd = &cli.Command{
}
gwapi := gateway.NewNode(api, lookbackCap, waitLookback, rateLimit, rateLimitTimeout)
h, err := gateway.Handler(gwapi, perConnRateLimit, connPerMinute, serverOptions...)
h, err := gateway.Handler(gwapi, api, perConnRateLimit, connPerMinute, serverOptions...)
if err != nil {
return xerrors.Errorf("failed to set up gateway HTTP handler")
}
cmd/lotus-miner/dagstore.go
+40
View File
@@ -19,6 +19,7 @@ var dagstoreCmd = &cli.Command{
Usage: "Manage the dagstore on the markets subsystem",
Subcommands: []*cli.Command{
dagstoreListShardsCmd,
dagstoreRegisterShardCmd,
dagstoreInitializeShardCmd,
dagstoreRecoverShardCmd,
dagstoreInitializeAllCmd,
@@ -59,6 +60,45 @@ var dagstoreListShardsCmd = &cli.Command{
},
}
var dagstoreRegisterShardCmd = &cli.Command{
Name: "register-shard",
ArgsUsage: "[key]",
Usage: "Register a shard",
Flags: []cli.Flag{
&cli.BoolFlag{
Name: "color",
Usage: "use color in display output",
DefaultText: "depends on output being a TTY",
},
},
Action: func(cctx *cli.Context) error {
if cctx.IsSet("color") {
color.NoColor = !cctx.Bool("color")
}
if cctx.NArg() != 1 {
return fmt.Errorf("must provide a single shard key")
}
marketsAPI, closer, err := lcli.GetMarketsAPI(cctx)
if err != nil {
return err
}
defer closer()
ctx := lcli.ReqContext(cctx)
shardKey := cctx.Args().First()
err = marketsAPI.DagstoreRegisterShard(ctx, shardKey)
if err != nil {
return err
}
fmt.Println("Registered shard " + shardKey)
return nil
},
}
var dagstoreInitializeShardCmd = &cli.Command{
Name: "initialize-shard",
ArgsUsage: "[key]",
cmd/lotus-miner/precommits-info.go
+54
View File
@@ -0,0 +1,54 @@
package main
import (
"fmt"
"sort"
"github.com/filecoin-project/lotus/blockstore"
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
"github.com/filecoin-project/lotus/chain/types"
lcli "github.com/filecoin-project/lotus/cli"
"github.com/filecoin-project/specs-actors/v7/actors/util/adt"
cbor "github.com/ipfs/go-ipld-cbor"
"github.com/urfave/cli/v2"
)
var sectorPreCommitsCmd = &cli.Command{
Name: "precommits",
Usage: "Print on-chain precommit info",
Action: func(cctx *cli.Context) error {
ctx := lcli.ReqContext(cctx)
mapi, closer, err := lcli.GetFullNodeAPI(cctx)
if err != nil {
return err
}
defer closer()
maddr, err := getActorAddress(ctx, cctx)
if err != nil {
return err
}
mact, err := mapi.StateGetActor(ctx, maddr, types.EmptyTSK)
if err != nil {
return err
}
store := adt.WrapStore(ctx, cbor.NewCborStore(blockstore.NewAPIBlockstore(mapi)))
mst, err := miner.Load(store, mact)
if err != nil {
return err
}
preCommitSector := make([]miner.SectorPreCommitOnChainInfo, 0)
err = mst.ForEachPrecommittedSector(func(info miner.SectorPreCommitOnChainInfo) error {
preCommitSector = append(preCommitSector, info)
return err
})
less := func(i, j int) bool {
return preCommitSector[i].Info.SectorNumber <= preCommitSector[j].Info.SectorNumber
}
sort.Slice(preCommitSector, less)
for _, info := range preCommitSector {
fmt.Printf("%s: %s\n", info.Info.SectorNumber, info.PreCommitEpoch)
}
return nil
},
}
cmd/lotus-miner/sealing.go
+167 -6
View File
@@ -4,20 +4,27 @@ import (
"encoding/hex"
"encoding/json"
"fmt"
"io"
"math"
"net/http"
"os"
"sort"
"strings"
"text/tabwriter"
"time"
"github.com/dustin/go-humanize"
"github.com/fatih/color"
"github.com/google/uuid"
"github.com/mitchellh/go-homedir"
"github.com/urfave/cli/v2"
"golang.org/x/xerrors"
"github.com/filecoin-project/go-padreader"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/lotus/extern/sector-storage/sealtasks"
"github.com/filecoin-project/lotus/extern/sector-storage/storiface"
"github.com/filecoin-project/lotus/lib/httpreader"
"github.com/filecoin-project/lotus/chain/types"
lcli "github.com/filecoin-project/lotus/cli"
@@ -31,6 +38,7 @@ var sealingCmd = &cli.Command{
workersCmd(true),
sealingSchedDiagCmd,
sealingAbortCmd,
sealingDataCidCmd,
},
}
@@ -97,13 +105,20 @@ func workersCmd(sealing bool) *cli.Command {
return st[i].id.String() < st[j].id.String()
})
/*
Example output:
Worker c4d65451-07f8-4230-98ad-4f33dea2a8cc, host myhostname
TASK: PC1(1/4) AP(15/15) GET(3)
CPU: [|||||||| ] 16/128 core(s) in use
RAM: [|||||||| ] 12% 125.8 GiB/1008 GiB
VMEM: [|||||||| ] 12% 125.8 GiB/1008 GiB
GPU: [ ] 0% 0.00/1 gpu(s) in use
GPU: NVIDIA GeForce RTX 3090, not used
*/
for _, stat := range st {
gpuUse := "not "
gpuCol := color.FgBlue
if stat.GpuUsed > 0 {
gpuCol = color.FgGreen
gpuUse = ""
}
// Worker uuid + name
var disabled string
if !stat.Enabled {
@@ -112,9 +127,53 @@ func workersCmd(sealing bool) *cli.Command {
fmt.Printf("Worker %s, host %s%s\n", stat.id, color.MagentaString(stat.Info.Hostname), disabled)
// Task counts
tc := make([][]string, 0, len(stat.TaskCounts))
for st, c := range stat.TaskCounts {
if c == 0 {
continue
}
stt, err := sealtasks.SttFromString(st)
if err != nil {
return err
}
str := fmt.Sprint(c)
if max := stat.Info.Resources.ResourceSpec(stt.RegisteredSealProof, stt.TaskType).MaxConcurrent; max > 0 {
switch {
case c < max:
str = color.GreenString(str)
case c >= max:
str = color.YellowString(str)
}
str = fmt.Sprintf("%s/%d", str, max)
} else {
str = color.CyanString(str)
}
str = fmt.Sprintf("%s(%s)", color.BlueString(stt.Short()), str)
tc = append(tc, []string{string(stt.TaskType), str})
}
sort.Slice(tc, func(i, j int) bool {
return sealtasks.TaskType(tc[i][0]).Less(sealtasks.TaskType(tc[j][0]))
})
var taskStr string
for _, t := range tc {
taskStr = t[1] + " "
}
if taskStr != "" {
fmt.Printf("\tTASK: %s\n", taskStr)
}
// CPU use
fmt.Printf("\tCPU: [%s] %d/%d core(s) in use\n",
barString(float64(stat.Info.Resources.CPUs), 0, float64(stat.CpuUse)), stat.CpuUse, stat.Info.Resources.CPUs)
// RAM use
ramTotal := stat.Info.Resources.MemPhysical
ramTasks := stat.MemUsedMin
ramUsed := stat.Info.Resources.MemUsed
@@ -129,6 +188,8 @@ func workersCmd(sealing bool) *cli.Command {
types.SizeStr(types.NewInt(ramTasks+ramUsed)),
types.SizeStr(types.NewInt(stat.Info.Resources.MemPhysical)))
// VMEM use (ram+swap)
vmemTotal := stat.Info.Resources.MemPhysical + stat.Info.Resources.MemSwap
vmemTasks := stat.MemUsedMax
vmemUsed := stat.Info.Resources.MemUsed + stat.Info.Resources.MemSwapUsed
@@ -143,12 +204,21 @@ func workersCmd(sealing bool) *cli.Command {
types.SizeStr(types.NewInt(vmemTasks+vmemReserved)),
types.SizeStr(types.NewInt(vmemTotal)))
// GPU use
if len(stat.Info.Resources.GPUs) > 0 {
gpuBar := barString(float64(len(stat.Info.Resources.GPUs)), 0, stat.GpuUsed)
fmt.Printf("\tGPU: [%s] %.f%% %.2f/%d gpu(s) in use\n", color.GreenString(gpuBar),
stat.GpuUsed*100/float64(len(stat.Info.Resources.GPUs)),
stat.GpuUsed, len(stat.Info.Resources.GPUs))
}
gpuUse := "not "
gpuCol := color.FgBlue
if stat.GpuUsed > 0 {
gpuCol = color.FgGreen
gpuUse = ""
}
for _, gpu := range stat.Info.Resources.GPUs {
fmt.Printf("\tGPU: %s\n", color.New(gpuCol).Sprintf("%s, %sused", gpu, gpuUse))
}
@@ -349,3 +419,94 @@ var sealingAbortCmd = &cli.Command{
return nodeApi.SealingAbort(ctx, job.ID)
},
}
var sealingDataCidCmd = &cli.Command{
Name: "data-cid",
Usage: "Compute data CID using workers",
ArgsUsage: "[file/url] <padded piece size>",
Flags: []cli.Flag{
&cli.Uint64Flag{
Name: "file-size",
Usage: "real file size",
},
},
Action: func(cctx *cli.Context) error {
if cctx.Args().Len() < 1 || cctx.Args().Len() > 2 {
return xerrors.Errorf("expected 1 or 2 arguments")
}
nodeApi, closer, err := lcli.GetStorageMinerAPI(cctx)
if err != nil {
return err
}
defer closer()
ctx := lcli.ReqContext(cctx)
var r io.Reader
sz := cctx.Uint64("file-size")
if strings.HasPrefix(cctx.Args().First(), "http://") || strings.HasPrefix(cctx.Args().First(), "https://") {
r = &httpreader.HttpReader{
URL: cctx.Args().First(),
}
if !cctx.IsSet("file-size") {
resp, err := http.Head(cctx.Args().First())
if err != nil {
return xerrors.Errorf("http head: %w", err)
}
if resp.ContentLength < 0 {
return xerrors.Errorf("head response didn't contain content length; specify --file-size")
}
sz = uint64(resp.ContentLength)
}
} else {
p, err := homedir.Expand(cctx.Args().First())
if err != nil {
return xerrors.Errorf("expanding path: %w", err)
}
f, err := os.OpenFile(p, os.O_RDONLY, 0)
if err != nil {
return xerrors.Errorf("opening source file: %w", err)
}
if !cctx.IsSet("file-size") {
st, err := f.Stat()
if err != nil {
return xerrors.Errorf("stat: %w", err)
}
sz = uint64(st.Size())
}
r = f
}
var psize abi.PaddedPieceSize
if cctx.Args().Len() == 2 {
rps, err := humanize.ParseBytes(cctx.Args().Get(1))
if err != nil {
return xerrors.Errorf("parsing piece size: %w", err)
}
psize = abi.PaddedPieceSize(rps)
if err := psize.Validate(); err != nil {
return xerrors.Errorf("checking piece size: %w", err)
}
if sz > uint64(psize.Unpadded()) {
return xerrors.Errorf("file larger than the piece")
}
} else {
psize = padreader.PaddedSize(sz).Padded()
}
pc, err := nodeApi.ComputeDataCid(ctx, psize.Unpadded(), r)
if err != nil {
return xerrors.Errorf("computing data CID: %w", err)
}
fmt.Println(pc.PieceCID, " ", pc.Size)
return nil
},
}
cmd/lotus-miner/sectors.go
+88
View File
@@ -45,6 +45,7 @@ var sectorsCmd = &cli.Command{
sectorsRefsCmd,
sectorsUpdateCmd,
sectorsPledgeCmd,
sectorPreCommitsCmd,
sectorsCheckExpireCmd,
sectorsExpiredCmd,
sectorsRenewCmd,
@@ -58,6 +59,7 @@ var sectorsCmd = &cli.Command{
sectorsCapacityCollateralCmd,
sectorsBatching,
sectorsRefreshPieceMatchingCmd,
sectorsCompactPartitionsCmd,
},
}
@@ -2088,3 +2090,89 @@ func yesno(b bool) string {
}
return color.RedString("NO")
}
var sectorsCompactPartitionsCmd = &cli.Command{
Name: "compact-partitions",
Usage: "removes dead sectors from partitions and reduces the number of partitions used if possible",
Flags: []cli.Flag{
&cli.Uint64Flag{
Name: "deadline",
Usage: "the deadline to compact the partitions in",
Required: true,
},
&cli.Int64SliceFlag{
Name: "partitions",
Usage: "list of partitions to compact sectors in",
Required: true,
},
&cli.BoolFlag{
Name: "really-do-it",
Usage: "Actually send transaction performing the action",
Value: false,
},
},
Action: func(cctx *cli.Context) error {
if !cctx.Bool("really-do-it") {
fmt.Println("Pass --really-do-it to actually execute this action")
return nil
}
api, acloser, err := lcli.GetFullNodeAPI(cctx)
if err != nil {
return err
}
defer acloser()
ctx := lcli.ReqContext(cctx)
maddr, err := getActorAddress(ctx, cctx)
if err != nil {
return err
}
minfo, err := api.StateMinerInfo(ctx, maddr, types.EmptyTSK)
if err != nil {
return err
}
deadline := cctx.Uint64("deadline")
if err != nil {
return err
}
parts := cctx.Int64Slice("partitions")
if len(parts) <= 0 {
return fmt.Errorf("must include at least one partition to compact")
}
partitions := bitfield.BitField{}
for _, partition := range parts {
partitions.Set(uint64(partition))
}
params := miner5.CompactPartitionsParams{
Deadline: deadline,
Partitions: partitions,
}
sp, err := actors.SerializeParams(&params)
if err != nil {
return xerrors.Errorf("serializing params: %w", err)
}
smsg, err := api.MpoolPushMessage(ctx, &types.Message{
From: minfo.Worker,
To: maddr,
Method: miner.Methods.CompactPartitions,
Value: big.Zero(),
Params: sp,
}, nil)
if err != nil {
return xerrors.Errorf("mpool push: %w", err)
}
fmt.Println("Message CID:", smsg.Cid())
return nil
},
}
cmd/lotus-shed/ledger.go
+66
View File
@@ -28,6 +28,7 @@ var ledgerCmd = &cli.Command{
ledgerKeyInfoCmd,
ledgerSignTestCmd,
ledgerShowCmd,
ledgerNewAddressesCmd,
},
}
@@ -291,3 +292,68 @@ var ledgerShowCmd = &cli.Command{
return nil
},
}
var ledgerNewAddressesCmd = &cli.Command{
Name: "new",
Flags: []cli.Flag{},
Action: func(cctx *cli.Context) error {
ctx := lcli.ReqContext(cctx)
if cctx.NArg() != 1 {
return fmt.Errorf("must pass account index")
}
index, err := strconv.ParseUint(cctx.Args().First(), 10, 32)
if err != nil {
return err
}
api, closer, err := lcli.GetFullNodeAPI(cctx)
if err != nil {
return err
}
defer closer()
fl, err := ledgerfil.FindLedgerFilecoinApp()
if err != nil {
return err
}
defer fl.Close() // nolint
if err := ctx.Err(); err != nil {
return err
}
p := []uint32{hdHard | 44, hdHard | 461, hdHard, 0, uint32(index)}
pubk, err := fl.GetPublicKeySECP256K1(p)
if err != nil {
return err
}
addr, err := address.NewSecp256k1Address(pubk)
if err != nil {
return err
}
var pd ledgerwallet.LedgerKeyInfo
pd.Address = addr
pd.Path = p
b, err := json.Marshal(pd)
if err != nil {
return err
}
var ki types.KeyInfo
ki.Type = types.KTSecp256k1Ledger
ki.PrivateKey = b
_, err = api.WalletImport(ctx, &ki)
if err != nil {
return err
}
fmt.Printf("%s %s\n", addr, printHDPath(p))
return nil
},
}
cmd/lotus-shed/shedgen/cbor_gen.go
+19 -13
View File
@@ -23,18 +23,19 @@ func (t *CarbNode) MarshalCBOR(w io.Writer) error {
_, err := w.Write(cbg.CborNull)
return err
}
if _, err := w.Write([]byte{161}); err != nil {
cw := cbg.NewCborWriter(w)
if _, err := cw.Write([]byte{161}); err != nil {
return err
}
scratch := make([]byte, 9)
// t.Sub ([]cid.Cid) (slice)
if len("Sub") > cbg.MaxLength {
return xerrors.Errorf("Value in field \"Sub\" was too long")
}
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajTextString, uint64(len("Sub"))); err != nil {
if err := cw.WriteMajorTypeHeader(cbg.MajTextString, uint64(len("Sub"))); err != nil {
return err
}
if _, err := io.WriteString(w, string("Sub")); err != nil {
@@ -45,27 +46,32 @@ func (t *CarbNode) MarshalCBOR(w io.Writer) error {
return xerrors.Errorf("Slice value in field t.Sub was too long")
}
if err := cbg.WriteMajorTypeHeaderBuf(scratch, w, cbg.MajArray, uint64(len(t.Sub))); err != nil {
if err := cw.WriteMajorTypeHeader(cbg.MajArray, uint64(len(t.Sub))); err != nil {
return err
}
for _, v := range t.Sub {
if err := cbg.WriteCidBuf(scratch, w, v); err != nil {
if err := cbg.WriteCid(w, v); err != nil {
return xerrors.Errorf("failed writing cid field t.Sub: %w", err)
}
}
return nil
}
func (t *CarbNode) UnmarshalCBOR(r io.Reader) error {
func (t *CarbNode) UnmarshalCBOR(r io.Reader) (err error) {
*t = CarbNode{}
br := cbg.GetPeeker(r)
scratch := make([]byte, 8)
cr := cbg.NewCborReader(r)
maj, extra, err := cbg.CborReadHeaderBuf(br, scratch)
maj, extra, err := cr.ReadHeader()
if err != nil {
return err
}
defer func() {
if err == io.EOF {
err = io.ErrUnexpectedEOF
}
}()
if maj != cbg.MajMap {
return fmt.Errorf("cbor input should be of type map")
}
@@ -80,7 +86,7 @@ func (t *CarbNode) UnmarshalCBOR(r io.Reader) error {
for i := uint64(0); i < n; i++ {
{
sval, err := cbg.ReadStringBuf(br, scratch)
sval, err := cbg.ReadString(cr)
if err != nil {
return err
}
@@ -92,7 +98,7 @@ func (t *CarbNode) UnmarshalCBOR(r io.Reader) error {
// t.Sub ([]cid.Cid) (slice)
case "Sub":
maj, extra, err = cbg.CborReadHeaderBuf(br, scratch)
maj, extra, err = cr.ReadHeader()
if err != nil {
return err
}
@@ -111,7 +117,7 @@ func (t *CarbNode) UnmarshalCBOR(r io.Reader) error {
for i := 0; i < int(extra); i++ {
c, err := cbg.ReadCid(br)
c, err := cbg.ReadCid(cr)
if err != nil {
return xerrors.Errorf("reading cid field t.Sub failed: %w", err)
}
cmd/lotus-shed/verifreg.go
+5 -5
View File
@@ -193,7 +193,7 @@ var verifRegVerifyClientCmd = &cli.Command{
},
},
Action: func(cctx *cli.Context) error {
fmt.Println("DEPRECATED: This behavior is being moved to `lotus verifreg`")
fmt.Println("DEPRECATED: This behavior is being moved to `lotus filplus`")
froms := cctx.String("from")
if froms == "" {
return fmt.Errorf("must specify from address with --from")
@@ -262,7 +262,7 @@ var verifRegListVerifiersCmd = &cli.Command{
Usage: "list all verifiers",
Hidden: true,
Action: func(cctx *cli.Context) error {
fmt.Println("DEPRECATED: This behavior is being moved to `lotus verifreg`")
fmt.Println("DEPRECATED: This behavior is being moved to `lotus filplus`")
api, closer, err := lcli.GetFullNodeAPI(cctx)
if err != nil {
return err
@@ -294,7 +294,7 @@ var verifRegListClientsCmd = &cli.Command{
Usage: "list all verified clients",
Hidden: true,
Action: func(cctx *cli.Context) error {
fmt.Println("DEPRECATED: This behavior is being moved to `lotus verifreg`")
fmt.Println("DEPRECATED: This behavior is being moved to `lotus filplus`")
api, closer, err := lcli.GetFullNodeAPI(cctx)
if err != nil {
return err
@@ -326,7 +326,7 @@ var verifRegCheckClientCmd = &cli.Command{
Usage: "check verified client remaining bytes",
Hidden: true,
Action: func(cctx *cli.Context) error {
fmt.Println("DEPRECATED: This behavior is being moved to `lotus verifreg`")
fmt.Println("DEPRECATED: This behavior is being moved to `lotus filplus`")
if !cctx.Args().Present() {
return fmt.Errorf("must specify client address to check")
}
@@ -362,7 +362,7 @@ var verifRegCheckVerifierCmd = &cli.Command{
Usage: "check verifiers remaining bytes",
Hidden: true,
Action: func(cctx *cli.Context) error {
fmt.Println("DEPRECATED: This behavior is being moved to `lotus verifreg`")
fmt.Println("DEPRECATED: This behavior is being moved to `lotus filplus`")
if !cctx.Args().Present() {
return fmt.Errorf("must specify verifier address to check")
}
cmd/lotus-worker/main.go
+24 -8
View File
@@ -182,12 +182,16 @@ var runCmd = &cli.Command{
Usage: "enable window post",
Value: false,
},
&cli.BoolFlag{
Name: "winningpost",
Usage: "enable winning post",
Value: false,
},
&cli.BoolFlag{
Name: "no-default",
Usage: "disable all default compute tasks, use the worker for storage/fetching only",
Value: false,
},
&cli.IntFlag{
Name: "parallel-fetch-limit",
Usage: "maximum fetch operations to run in parallel",
@@ -289,27 +293,27 @@ var runCmd = &cli.Command{
return err
}
if cctx.Bool("commit") || cctx.Bool("prove-replica-update2") {
if err := paramfetch.GetParams(ctx, build.ParametersJSON(), build.SrsJSON(), uint64(ssize)); err != nil {
return xerrors.Errorf("get params: %w", err)
}
}
var taskTypes []sealtasks.TaskType
var workerType string
var needParams bool
if cctx.Bool("windowpost") {
needParams = true
workerType = sealtasks.WorkerWindowPoSt
taskTypes = append(taskTypes, sealtasks.TTGenerateWindowPoSt)
}
if cctx.Bool("winningpost") {
needParams = true
workerType = sealtasks.WorkerWinningPoSt
taskTypes = append(taskTypes, sealtasks.TTGenerateWinningPoSt)
}
if workerType == "" {
workerType = sealtasks.WorkerSealing
taskTypes = append(taskTypes, sealtasks.TTFetch, sealtasks.TTCommit1, sealtasks.TTProveReplicaUpdate1, sealtasks.TTFinalize, sealtasks.TTFinalizeReplicaUpdate)
if !cctx.Bool("no-default") {
workerType = sealtasks.WorkerSealing
}
}
if (workerType == sealtasks.WorkerSealing || cctx.IsSet("addpiece")) && cctx.Bool("addpiece") {
@@ -325,18 +329,24 @@ var runCmd = &cli.Command{
taskTypes = append(taskTypes, sealtasks.TTPreCommit2)
}
if (workerType == sealtasks.WorkerSealing || cctx.IsSet("commit")) && cctx.Bool("commit") {
needParams = true
taskTypes = append(taskTypes, sealtasks.TTCommit2)
}
if (workerType == sealtasks.WorkerSealing || cctx.IsSet("replica-update")) && cctx.Bool("replica-update") {
taskTypes = append(taskTypes, sealtasks.TTReplicaUpdate)
}
if (workerType == sealtasks.WorkerSealing || cctx.IsSet("prove-replica-update2")) && cctx.Bool("prove-replica-update2") {
needParams = true
taskTypes = append(taskTypes, sealtasks.TTProveReplicaUpdate2)
}
if (workerType == sealtasks.WorkerSealing || cctx.IsSet("regen-sector-key")) && cctx.Bool("regen-sector-key") {
taskTypes = append(taskTypes, sealtasks.TTRegenSectorKey)
}
if cctx.Bool("no-default") && workerType == "" {
workerType = sealtasks.WorkerSealing
}
if len(taskTypes) == 0 {
return xerrors.Errorf("no task types specified")
}
@@ -346,6 +356,12 @@ var runCmd = &cli.Command{
}
}
if needParams {
if err := paramfetch.GetParams(ctx, build.ParametersJSON(), build.SrsJSON(), uint64(ssize)); err != nil {
return xerrors.Errorf("get params: %w", err)
}
}
// Open repo
repoPath := cctx.String(FlagWorkerRepo)
cmd/lotus-worker/sealworker/rpc.go
+1
View File
@@ -34,6 +34,7 @@ func WorkerHandler(authv func(ctx context.Context, token string) ([]auth.Permiss
}
rpcServer.Register("Filecoin", wapi)
rpcServer.AliasMethod("rpc.discover", "Filecoin.Discover")
mux.Handle("/rpc/v0", rpcServer)
mux.Handle("/rpc/streams/v0/push/{uuid}", readerHandler)