From fac122c35842639b292f50a3a9c5fb39845112ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=81ukasz=20Magiera?= Date: Fri, 18 Aug 2023 16:53:59 +0200 Subject: [PATCH] sealer: Support external PC2 executor commands --- cmd/lotus-bench/simple.go | 42 +++++++++++- storage/sealer/ffiwrapper/extern_pc2.go | 87 +++++++++++++++++++++++++ storage/sealer/ffiwrapper/sealer.go | 15 ++++- storage/sealer/ffiwrapper/sealer_cgo.go | 37 +++++++++-- 4 files changed, 175 insertions(+), 6 deletions(-) create mode 100644 storage/sealer/ffiwrapper/extern_pc2.go diff --git a/cmd/lotus-bench/simple.go b/cmd/lotus-bench/simple.go index 8ae5713ad..0e9c05f12 100644 --- a/cmd/lotus-bench/simple.go +++ b/cmd/lotus-bench/simple.go @@ -305,7 +305,36 @@ var simplePreCommit2 = &cli.Command{ Name: "synthetic", Usage: "generate synthetic PoRep proofs", }, + &cli.StringFlag{ + Name: "external-pc2", + Usage: "command for computing PC2 externally", + }, }, + Description: `Compute PreCommit2 inputs and seal a sector. + +--external-pc2 can be used to compute the PreCommit2 inputs externally. +The flag behaves similarly to the related lotus-worker flag, using it in +lotus-bench may be useful for testing if the external PreCommit2 command is +invoked correctly. + +The command will be called with a number of environment variables set: +* EXTSEAL_PC2_SECTOR_NUM: the sector number +* EXTSEAL_PC2_SECTOR_MINER: the miner id +* EXTSEAL_PC2_PROOF_TYPE: the proof type +* EXTSEAL_PC2_SECTOR_SIZE: the sector size in bytes +* EXTSEAL_PC2_CACHE: the path to the cache directory +* EXTSEAL_PC2_SEALED: the path to the sealed sector file (initialized with unsealed data by the caller) +* EXTSEAL_PC2_PC1OUT: output from rust-fil-proofs precommit1 phase (base64 encoded json) + +The command is expected to: +* Create cache sc-02-data-tree-r* files +* Create cache sc-02-data-tree-c* files +* Create cache p_aux / t_aux files +* Transform the sealed file in place + +Example invocation of lotus-bench as external executor: +'./lotus-bench simple precommit2 --sector-size $EXTSEAL_PC2_SECTOR_SIZE $EXTSEAL_PC2_SEALED $EXTSEAL_PC2_CACHE $EXTSEAL_PC2_PC1OUT' +`, ArgsUsage: "[sealed] [cache] [pc1 out]", Action: func(cctx *cli.Context) error { ctx := cctx.Context @@ -330,7 +359,18 @@ var simplePreCommit2 = &cli.Command{ storiface.FTSealed: cctx.Args().Get(0), storiface.FTCache: cctx.Args().Get(1), } - sealer, err := ffiwrapper.New(pp) + + var opts []ffiwrapper.FFIWrapperOpt + + if cctx.IsSet("external-pc2") { + extSeal := ffiwrapper.ExternalSealer{ + PreCommit2: ffiwrapper.MakeExternPrecommit2(cctx.String("external-pc2")), + } + + opts = append(opts, ffiwrapper.WithExternalSealCalls(extSeal)) + } + + sealer, err := ffiwrapper.New(pp, opts...) if err != nil { return err } diff --git a/storage/sealer/ffiwrapper/extern_pc2.go b/storage/sealer/ffiwrapper/extern_pc2.go new file mode 100644 index 000000000..fc6873188 --- /dev/null +++ b/storage/sealer/ffiwrapper/extern_pc2.go @@ -0,0 +1,87 @@ +package ffiwrapper + +import ( + "context" + "encoding/base64" + "fmt" + commcid "github.com/filecoin-project/go-fil-commcid" + "github.com/filecoin-project/lotus/storage/sealer/commitment" + "golang.org/x/xerrors" + "os" + "os/exec" + + "github.com/ipfs/go-cid" + + "github.com/filecoin-project/lotus/storage/sealer/storiface" +) + +// MakeExternPrecommit2 creates an implementation of ExternPrecommit2 backed by +// an external command specified by the command string. +// +// The command will be called with a number of environment variables set: +// * EXTSEAL_PC2_SECTOR_NUM: the sector number +// * EXTSEAL_PC2_SECTOR_MINER: the miner id +// * EXTSEAL_PC2_PROOF_TYPE: the proof type +// * EXTSEAL_PC2_SECTOR_SIZE: the sector size in bytes +// * EXTSEAL_PC2_CACHE: the path to the cache directory +// * EXTSEAL_PC2_SEALED: the path to the sealed sector file (initialized with unsealed data by the caller) +// * EXTSEAL_PC2_PC1OUT: output from rust-fil-proofs precommit1 phase (base64 encoded json) +// +// The command is expected to: +// * Create cache sc-02-data-tree-r* files +// * Create cache sc-02-data-tree-c* files +// * Create cache p_aux / t_aux files +// * Transform the sealed file in place +func MakeExternPrecommit2(command string) ExternPrecommit2 { + return func(ctx context.Context, sector storiface.SectorRef, cache, sealed string, pc1out storiface.PreCommit1Out) (sealedCID cid.Cid, unsealedCID cid.Cid, err error) { + ssize, err := sector.ProofType.SectorSize() + if err != nil { + return cid.Undef, cid.Undef, err + } + + // Set environment variables for the external command + env := []string{ + "EXTSEAL_PC2_SECTOR_NUM=" + sector.ID.Number.String(), + "EXTSEAL_PC2_SECTOR_MINER=" + sector.ID.Miner.String(), + "EXTSEAL_PC2_PROOF_TYPE=" + fmt.Sprintf("%d", sector.ProofType), + "EXTSEAL_PC2_SECTOR_SIZE=" + fmt.Sprintf("%d", ssize), + "EXTSEAL_PC2_CACHE=" + cache, + "EXTSEAL_PC2_SEALED=" + sealed, + "EXTSEAL_PC2_PC1OUT=" + base64.StdEncoding.EncodeToString(pc1out), + } + + log.Infow("running external sealing call", "method", "precommit2", "command", command, "env", env) + + // Create and run the external command + cmd := exec.CommandContext(ctx, "sh", "-c", command) + cmd.Env = env + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + + if err := cmd.Run(); err != nil { + return cid.Cid{}, cid.Cid{}, xerrors.Errorf("external command error: %w", err) + } + + commr, err := commitment.PAuxCommR(cache) + if err != nil { + return cid.Undef, cid.Undef, xerrors.Errorf("reading p_aux: %w", err) + } + + sealedCID, err = commcid.ReplicaCommitmentV1ToCID(commr[:]) + if err != nil { + return cid.Cid{}, cid.Cid{}, err + } + + commd, err := commitment.TreeDCommD(cache) + if err != nil { + return cid.Undef, cid.Undef, xerrors.Errorf("reading CommD from tree-d: %w", err) + } + + unsealedCID, err = commcid.DataCommitmentV1ToCID(commd[:]) + if err != nil { + return cid.Cid{}, cid.Cid{}, err + } + + return sealedCID, unsealedCID, nil + } +} diff --git a/storage/sealer/ffiwrapper/sealer.go b/storage/sealer/ffiwrapper/sealer.go index 39cb8fa1b..87311e1fe 100644 --- a/storage/sealer/ffiwrapper/sealer.go +++ b/storage/sealer/ffiwrapper/sealer.go @@ -1,13 +1,26 @@ package ffiwrapper import ( + "context" + "github.com/filecoin-project/lotus/storage/sealer/storiface" + "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" ) var log = logging.Logger("ffiwrapper") +type ExternPrecommit2 func(ctx context.Context, sector storiface.SectorRef, cache, sealed string, pc1out storiface.PreCommit1Out) (sealedCID cid.Cid, unsealedCID cid.Cid, err error) + +type ExternalSealer struct { + PreCommit2 ExternPrecommit2 +} + type Sealer struct { - sectors SectorProvider + sectors SectorProvider + + // externCalls cointain overrides for calling alternative sealing logic + externCalls ExternalSealer + stopping chan struct{} } diff --git a/storage/sealer/ffiwrapper/sealer_cgo.go b/storage/sealer/ffiwrapper/sealer_cgo.go index 8e970adaa..acd325ecc 100644 --- a/storage/sealer/ffiwrapper/sealer_cgo.go +++ b/storage/sealer/ffiwrapper/sealer_cgo.go @@ -40,10 +40,30 @@ import ( var _ storiface.Storage = &Sealer{} -func New(sectors SectorProvider) (*Sealer, error) { +type FFIWrapperOpts struct { + ext ExternalSealer +} + +type FFIWrapperOpt func(*FFIWrapperOpts) + +func WithExternalSealCalls(ext ExternalSealer) FFIWrapperOpt { + return func(o *FFIWrapperOpts) { + o.ext = ext + } +} + +func New(sectors SectorProvider, opts ...FFIWrapperOpt) (*Sealer, error) { + options := &FFIWrapperOpts{} + + for _, o := range opts { + o(options) + } + sb := &Sealer{ sectors: sectors, + externCalls: options.ext, + stopping: make(chan struct{}), } @@ -881,9 +901,18 @@ func (sb *Sealer) SealPreCommit2(ctx context.Context, sector storiface.SectorRef } defer done() - sealedCID, unsealedCID, err := ffi.SealPreCommitPhase2(phase1Out, paths.Cache, paths.Sealed) - if err != nil { - return storiface.SectorCids{}, xerrors.Errorf("presealing sector %d (%s): %w", sector.ID.Number, paths.Unsealed, err) + var sealedCID, unsealedCID cid.Cid + + if sb.externCalls.PreCommit2 == nil { + sealedCID, unsealedCID, err = ffi.SealPreCommitPhase2(phase1Out, paths.Cache, paths.Sealed) + if err != nil { + return storiface.SectorCids{}, xerrors.Errorf("presealing sector %d (%s): %w", sector.ID.Number, paths.Unsealed, err) + } + } else { + sealedCID, unsealedCID, err = sb.externCalls.PreCommit2(ctx, sector, paths.Cache, paths.Sealed, phase1Out) + if err != nil { + return storiface.SectorCids{}, xerrors.Errorf("presealing sector (extern-pc2) %d (%s): %w", sector.ID.Number, paths.Unsealed, err) + } } ssize, err := sector.ProofType.SectorSize()