cerc-io/lotus
16
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Merge pull request #8389 from filecoin-project/feat/manual-wdpost

Browse Source 
feat: miner: API/CLI to compute window-post
This commit is contained in:
Łukasz Magiera 2022-03-28 21:20:06 -04:00 committed by GitHub
commit d502eeba2b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 287 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
api/api_storage.go
View File

@ -6,6 +6,7 @@ import (
"time"
"github.com/filecoin-project/lotus/chain/actors/builtin"
"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
"github.com/google/uuid"
"github.com/ipfs/go-cid"
@ -51,6 +52,8 @@ type StorageMiner interface {
MiningBase(context.Context) (*types.TipSet, error) //perm:read
ComputeWindowPoSt(ctx context.Context, dlIdx uint64, tsk types.TipSetKey) ([]miner.SubmitWindowedPoStParams, error) //perm:admin
// Temp api for testing
PledgeSector(context.Context) (abi.SectorID, error) //perm:write

13
api/proxy_gen.go
View File

@ -641,6 +641,8 @@ type StorageMinerStruct struct {
ComputeProof func(p0 context.Context, p1 []builtin.ExtendedSectorInfo, p2 abi.PoStRandomness, p3 abi.ChainEpoch, p4 abinetwork.Version) ([]builtin.PoStProof, error) `perm:"read"`
ComputeWindowPoSt func(p0 context.Context, p1 uint64, p2 types.TipSetKey) ([]miner.SubmitWindowedPoStParams, error) `perm:"admin"`
CreateBackup func(p0 context.Context, p1 string) error `perm:"admin"`
DagstoreGC func(p0 context.Context) ([]DagstoreShardResult, error) `perm:"admin"`
@ -3857,6 +3859,17 @@ func (s *StorageMinerStub) ComputeProof(p0 context.Context, p1 []builtin.Extende
return *new([]builtin.PoStProof), ErrNotSupported
}
func (s *StorageMinerStruct) ComputeWindowPoSt(p0 context.Context, p1 uint64, p2 types.TipSetKey) ([]miner.SubmitWindowedPoStParams, error) {
if s.Internal.ComputeWindowPoSt == nil {
return *new([]miner.SubmitWindowedPoStParams), ErrNotSupported
}
return s.Internal.ComputeWindowPoSt(p0, p1, p2)
}
func (s *StorageMinerStub) ComputeWindowPoSt(p0 context.Context, p1 uint64, p2 types.TipSetKey) ([]miner.SubmitWindowedPoStParams, error) {
return *new([]miner.SubmitWindowedPoStParams), ErrNotSupported
}
func (s *StorageMinerStruct) CreateBackup(p0 context.Context, p1 string) error {
if s.Internal.CreateBackup == nil {
return ErrNotSupported

BIN
build/openrpc/full.json.gz
View File

Binary file not shown.

BIN
build/openrpc/miner.json.gz
View File

Binary file not shown.

BIN
build/openrpc/worker.json.gz
View File

Binary file not shown.

50
cmd/lotus-miner/proving.go
View File

@ -1,10 +1,12 @@
package main
import (
"encoding/json"
"fmt"
"os"
"strconv"
"text/tabwriter"
"time"
"github.com/fatih/color"
"github.com/urfave/cli/v2"
@ -31,6 +33,7 @@ var provingCmd = &cli.Command{
provingFaultsCmd,
provingCheckProvableCmd,
workersCmd(false),
provingComputeCmd,
},
}
@ -510,3 +513,50 @@ var provingCheckProvableCmd = &cli.Command{
return tw.Flush()
},
}
var provingComputeCmd = &cli.Command{
Name: "compute",
Subcommands: []*cli.Command{
provingComputeWindowPoStCmd,
},
}
var provingComputeWindowPoStCmd = &cli.Command{
Name: "window-post",
Usage: "Compute WindowPoSt for a specific deadline",
Description: `Note: This command is intended to be used to verify PoSt compute performance.
It will not send any messages to the chain.`,
ArgsUsage: "[deadline index]",
Action: func(cctx *cli.Context) error {
if cctx.Args().Len() != 1 {
return xerrors.Errorf("must pass deadline index")
}
dlIdx, err := strconv.ParseUint(cctx.Args().Get(0), 10, 64)
if err != nil {
return xerrors.Errorf("could not parse deadline index: %w", err)
}
sapi, scloser, err := lcli.GetStorageMinerAPI(cctx)
if err != nil {
return err
}
defer scloser()
ctx := lcli.ReqContext(cctx)
start := time.Now()
res, err := sapi.ComputeWindowPoSt(ctx, dlIdx, types.EmptyTSK)
fmt.Printf("Took %s\n", time.Now().Sub(start))
if err != nil {
return err
}
jr, err := json.Marshal(res)
if err != nil {
return err
}
fmt.Println(string(jr))
return nil
},
}

47
documentation/en/api-v0-methods-miner.md
View File

@ -16,6 +16,7 @@
* [CheckProvable](#CheckProvable)
* [Compute](#Compute)
* [ComputeProof](#ComputeProof)
* [ComputeWindowPoSt](#ComputeWindowPoSt)
* [Create](#Create)
* [CreateBackup](#CreateBackup)
* [Dagstore](#Dagstore)
@ -394,6 +395,52 @@ Response:
]
```
### ComputeWindowPoSt
Perms: admin
Inputs:
```json
[
42,
[
{
"/": "bafy2bzacea3wsdh6y3a36tb3skempjoxqpuyompjbmfeyf34fi3uy6uue42v4"
},
{
"/": "bafy2bzacebp3shtrn43k7g3unredz7fxn4gj533d3o43tqn2p2ipxxhrvchve"
}
]
]
```
Response:
```json
[
{
"Deadline": 42,
"Partitions": [
{
"Index": 42,
"Skipped": [
5,
1
]
}
],
"Proofs": [
{
"PoStProof": 8,
"ProofBytes": "Ynl0ZSBhcnJheQ=="
}
],
"ChainCommitEpoch": 10101,
"ChainCommitRand": "Bw=="
}
]
```
## Create

35
documentation/en/cli-lotus-miner.md
View File

@ -2041,6 +2041,7 @@ COMMANDS:
faults View the currently known proving faulty sectors information
check Check sectors provable
workers list workers
compute
help, h Shows a list of commands or help for one command
OPTIONS:
@ -2131,6 +2132,40 @@ OPTIONS:
```
### lotus-miner proving compute
```
NAME:
lotus-miner proving compute - A new cli application
USAGE:
lotus-miner proving compute command [command options] [arguments...]
COMMANDS:
window-post Compute WindowPoSt for a specific deadline
help, h Shows a list of commands or help for one command
OPTIONS:
--help, -h show help (default: false)
```
#### lotus-miner proving compute window-post
```
NAME:
lotus-miner proving compute window-post - Compute WindowPoSt for a specific deadline
USAGE:
lotus-miner proving compute window-post [command options] [deadline index]
DESCRIPTION:
Note: This command is intended to be used to verify PoSt compute performance.
It will not send any messages to the chain.
OPTIONS:
--help, -h show help (default: false)
```
## lotus-miner storage
```
NAME:

46
itests/worker_test.go
View File

@ -296,3 +296,49 @@ func TestWindowPostWorkerSkipBadSector(t *testing.T) {
require.Equal(t, p.MinerPower, p.TotalPower)
require.Equal(t, p.MinerPower.RawBytePower, types.NewInt(uint64(ssz)*uint64(sectors-1)))
}
func TestWindowPostWorkerManualPoSt(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
_ = logging.SetLogLevel("storageminer", "INFO")
sectors := 2 * 48 * 2
client, miner, _, ens := kit.EnsembleWorker(t,
kit.PresealSectors(sectors), // 2 sectors per partition, 2 partitions in all 48 deadlines
kit.LatestActorsAt(-1),
kit.ThroughRPC(),
kit.WithTaskTypes([]sealtasks.TaskType{sealtasks.TTGenerateWindowPoSt}))
maddr, err := miner.ActorAddress(ctx)
require.NoError(t, err)
di, err := client.StateMinerProvingDeadline(ctx, maddr, types.EmptyTSK)
require.NoError(t, err)
bm := ens.InterconnectAll().BeginMiningMustPost(2 * time.Millisecond)[0]
di = di.NextNotElapsed()
t.Log("Running one proving period")
waitUntil := di.Open + di.WPoStChallengeWindow*2 - 2
client.WaitTillChain(ctx, kit.HeightAtLeast(waitUntil))
t.Log("Waiting for post message")
bm.Stop()
tryDl := func(dl uint64) {
p, err := miner.ComputeWindowPoSt(ctx, dl, types.EmptyTSK)
require.NoError(t, err)
require.Len(t, p, 1)
require.Equal(t, dl, p[0].Deadline)
}
tryDl(0)
tryDl(40)
tryDl(di.Index + 4)
lastPending, err := client.MpoolPending(ctx, types.EmptyTSK)
require.NoError(t, err)
require.Len(t, lastPending, 0)
}

2
node/builder_miner.go
View File

@ -108,10 +108,10 @@ func ConfigStorageMiner(c interface{}) Option {
// Mining / proving
Override(new(*slashfilter.SlashFilter), modules.NewSlashFilter),
Override(new(*storage.Miner), modules.StorageMiner(config.DefaultStorageMiner().Fees)),
Override(new(*miner.Miner), modules.SetupBlockProducer),
Override(new(gen.WinningPoStProver), storage.NewWinningPoStProver),
Override(new(*storage.Miner), modules.StorageMiner(cfg.Fees)),
Override(new(*storage.WindowPoStScheduler), modules.WindowPostScheduler(cfg.Fees)),
Override(new(sectorblocks.SectorBuilder), From(new(*storage.Miner))),
),

18
node/impl/storminer.go
View File

@ -16,6 +16,7 @@ import (
"github.com/filecoin-project/go-jsonrpc/auth"
"github.com/filecoin-project/lotus/chain/actors/builtin"
lminer "github.com/filecoin-project/lotus/chain/actors/builtin/miner"
"github.com/filecoin-project/lotus/chain/gen"
"github.com/google/uuid"
@ -92,6 +93,8 @@ type StorageMinerAPI struct {
storiface.WorkerReturn `optional:"true"`
AddrSel *storage.AddressSelector
WdPoSt *storage.WindowPoStScheduler
Epp gen.WinningPoStProver `optional:"true"`
DS dtypes.MetadataDS
@ -407,6 +410,21 @@ func (sm *StorageMinerAPI) SectorMatchPendingPiecesToOpenSectors(ctx context.Con
return sm.Miner.SectorMatchPendingPiecesToOpenSectors(ctx)
}
func (sm *StorageMinerAPI) ComputeWindowPoSt(ctx context.Context, dlIdx uint64, tsk types.TipSetKey) ([]lminer.SubmitWindowedPoStParams, error) {
var ts *types.TipSet
var err error
if tsk == types.EmptyTSK {
ts, err = sm.Full.ChainHead(ctx)
} else {
ts, err = sm.Full.ChainGetTipSet(ctx, tsk)
}
if err != nil {
return nil, err
}
return sm.WdPoSt.ComputePoSt(ctx, dlIdx, ts)
}
func (sm *StorageMinerAPI) WorkerConnect(ctx context.Context, url string) error {
w, err := connectRemoteWorker(ctx, sm, url)
if err != nil {

44
node/modules/storageminer.go
View File

@ -215,6 +215,7 @@ type StorageMinerParams struct {
GetSealingConfigFn dtypes.GetSealingConfigFunc
Journal journal.Journal
AddrSel *storage.AddressSelector
Maddr dtypes.MinerAddress
}
func StorageMiner(fc config.MinerFeeConfig) func(params StorageMinerParams) (*storage.Miner, error) {
@ -231,20 +232,11 @@ func StorageMiner(fc config.MinerFeeConfig) func(params StorageMinerParams) (*st
gsd = params.GetSealingConfigFn
j = params.Journal
as = params.AddrSel
maddr = address.Address(params.Maddr)
)
maddr, err := minerAddrFromDS(ds)
if err != nil {
return nil, err
}
ctx := helpers.LifecycleCtx(mctx, lc)
fps, err := storage.NewWindowedPoStScheduler(api, fc, as, sealer, verif, sealer, j, maddr)
if err != nil {
return nil, err
}
sm, err := storage.NewMiner(api, maddr, ds, sealer, sc, verif, prover, gsd, fc, j, as)
if err != nil {
return nil, err
@ -252,7 +244,6 @@ func StorageMiner(fc config.MinerFeeConfig) func(params StorageMinerParams) (*st
lc.Append(fx.Hook{
OnStart: func(context.Context) error {
go fps.Run(ctx)
return sm.Run(ctx)
},
OnStop: sm.Stop,
@ -262,6 +253,37 @@ func StorageMiner(fc config.MinerFeeConfig) func(params StorageMinerParams) (*st
}
}
func WindowPostScheduler(fc config.MinerFeeConfig) func(params StorageMinerParams) (*storage.WindowPoStScheduler, error) {
return func(params StorageMinerParams) (*storage.WindowPoStScheduler, error) {
var (
mctx = params.MetricsCtx
lc = params.Lifecycle
api = params.API
sealer = params.Sealer
verif = params.Verifier
j = params.Journal
as = params.AddrSel
maddr = address.Address(params.Maddr)
)
ctx := helpers.LifecycleCtx(mctx, lc)
fps, err := storage.NewWindowedPoStScheduler(api, fc, as, sealer, verif, sealer, j, maddr)
if err != nil {
return nil, err
}
lc.Append(fx.Hook{
OnStart: func(context.Context) error {
go fps.Run(ctx)
return nil
},
})
return fps, nil
}
}
func HandleRetrieval(host host.Host, lc fx.Lifecycle, m retrievalmarket.RetrievalProvider, j journal.Journal) {
m.OnReady(marketevents.ReadyLogger("retrieval provider"))
lc.Append(fx.Hook{

54
storage/wdpost_run.go
View File

@ -93,7 +93,7 @@ func (s *WindowPoStScheduler) runGeneratePoST(
ctx, span := trace.StartSpan(ctx, "WindowPoStScheduler.generatePoST")
defer span.End()
posts, err := s.runPoStCycle(ctx, *deadline, ts)
posts, err := s.runPoStCycle(ctx, false, *deadline, ts)
if err != nil {
log.Errorf("runPoStCycle failed: %+v", err)
return nil, err
@ -449,19 +449,8 @@ func (s *WindowPoStScheduler) declareFaults(ctx context.Context, dlIdx uint64, p
return faults, sm, nil
}
// runPoStCycle runs a full cycle of the PoSt process:
//
// 1. performs recovery declarations for the next deadline.
// 2. performs fault declarations for the next deadline.
// 3. computes and submits proofs, batching partitions and making sure they
// don't exceed message capacity.
func (s *WindowPoStScheduler) runPoStCycle(ctx context.Context, di dline.Info, ts *types.TipSet) ([]miner.SubmitWindowedPoStParams, error) {
ctx, span := trace.StartSpan(ctx, "storage.runPoStCycle")
defer span.End()
func (s *WindowPoStScheduler) asyncFaultRecover(di dline.Info, ts *types.TipSet) {
go func() {
// TODO: extract from runPoStCycle, run on fault cutoff boundaries
// check faults / recoveries for the *next* deadline. It's already too
// late to declare them for this deadline
declDeadline := (di.Index + 2) % di.WPoStPeriodDeadlines
@ -520,6 +509,24 @@ func (s *WindowPoStScheduler) runPoStCycle(ctx context.Context, di dline.Info, t
}
})
}()
}
// runPoStCycle runs a full cycle of the PoSt process:
//
// 1. performs recovery declarations for the next deadline.
// 2. performs fault declarations for the next deadline.
// 3. computes and submits proofs, batching partitions and making sure they
// don't exceed message capacity.
//
// When `manual` is set, no messages (fault/recover) will be automatically sent
func (s *WindowPoStScheduler) runPoStCycle(ctx context.Context, manual bool, di dline.Info, ts *types.TipSet) ([]miner.SubmitWindowedPoStParams, error) {
ctx, span := trace.StartSpan(ctx, "storage.runPoStCycle")
defer span.End()
if !manual {
// TODO: extract from runPoStCycle, run on fault cutoff boundaries
s.asyncFaultRecover(di, ts)
}
buf := new(bytes.Buffer)
if err := s.actor.MarshalCBOR(buf); err != nil {
@ -941,3 +948,24 @@ func (s *WindowPoStScheduler) prepareMessage(ctx context.Context, msg *types.Mes
}
return nil
}
func (s *WindowPoStScheduler) ComputePoSt(ctx context.Context, dlIdx uint64, ts *types.TipSet) ([]miner.SubmitWindowedPoStParams, error) {
dl, err := s.api.StateMinerProvingDeadline(ctx, s.actor, ts.Key())
if err != nil {
return nil, xerrors.Errorf("getting deadline: %w", err)
}
curIdx := dl.Index
dl.Index = dlIdx
dlDiff := dl.Index - curIdx
if dl.Index > curIdx {
dlDiff -= dl.WPoStPeriodDeadlines
dl.PeriodStart -= dl.WPoStProvingPeriod
}
epochDiff := (dl.WPoStProvingPeriod / abi.ChainEpoch(dl.WPoStPeriodDeadlines)) * abi.ChainEpoch(dlDiff)
// runPoStCycle only needs dl.Index and dl.Challenge
dl.Challenge += epochDiff
return s.runPoStCycle(ctx, true, *dl, ts)
}