From beaa3dffab9b5aafab529d2f1a67d9dda14b298c Mon Sep 17 00:00:00 2001 From: Jeromy Date: Thu, 2 Apr 2020 14:18:57 -0700 Subject: [PATCH] add a command to lotus-bench to benchmark importing and validating a chain --- api/api_full.go | 1 + chain/stmgr/call.go | 1 + chain/stmgr/stmgr.go | 1 + chain/vm/vm.go | 21 +- cmd/lotus-bench/import.go | 140 +++++++ cmd/lotus-bench/main.go | 830 +++++++++++++++++++------------------- go.sum | 4 + node/impl/full/state.go | 1 + 8 files changed, 582 insertions(+), 417 deletions(-) create mode 100644 cmd/lotus-bench/import.go diff --git a/api/api_full.go b/api/api_full.go index a1c21344c..6798ba580 100644 --- a/api/api_full.go +++ b/api/api_full.go @@ -314,6 +314,7 @@ type InvocResult struct { MsgRct *types.MessageReceipt InternalExecutions []*vm.ExecutionResult Error string + Duration time.Duration } type MethodCall struct { diff --git a/chain/stmgr/call.go b/chain/stmgr/call.go index ac5c12ca0..293f18ef7 100644 --- a/chain/stmgr/call.go +++ b/chain/stmgr/call.go @@ -68,6 +68,7 @@ func (sm *StateManager) CallRaw(ctx context.Context, msg *types.Message, bstate MsgRct: &ret.MessageReceipt, InternalExecutions: ret.InternalExecutions, Error: errs, + Duration: ret.Duration, }, nil } diff --git a/chain/stmgr/stmgr.go b/chain/stmgr/stmgr.go index 636d5af33..e81213500 100644 --- a/chain/stmgr/stmgr.go +++ b/chain/stmgr/stmgr.go @@ -124,6 +124,7 @@ func (sm *StateManager) ExecutionTrace(ctx context.Context, ts *types.TipSet) (c Msg: msg, MsgRct: &ret.MessageReceipt, InternalExecutions: ret.InternalExecutions, + Duration: ret.Duration, } if ret.ActorErr != nil { ir.Error = ret.ActorErr.Error() diff --git a/chain/vm/vm.go b/chain/vm/vm.go index f132bb149..0666caea1 100644 --- a/chain/vm/vm.go +++ b/chain/vm/vm.go @@ -5,6 +5,7 @@ import ( "context" "fmt" "reflect" + "time" "github.com/filecoin-project/specs-actors/actors/builtin" @@ -186,6 +187,7 @@ type ApplyRet struct { ActorErr aerrors.ActorError Penalty types.BigInt InternalExecutions []*ExecutionResult + Duration time.Duration } func (vm *VM) send(ctx context.Context, msg *types.Message, parent *Runtime, @@ -261,6 +263,7 @@ func checkMessage(msg *types.Message) error { } func (vm *VM) ApplyImplicitMessage(ctx context.Context, msg *types.Message) (*ApplyRet, error) { + start := time.Now() ret, actorErr, _ := vm.send(ctx, msg, nil, 0) return &ApplyRet{ MessageReceipt: types.MessageReceipt{ @@ -271,10 +274,12 @@ func (vm *VM) ApplyImplicitMessage(ctx context.Context, msg *types.Message) (*Ap ActorErr: actorErr, InternalExecutions: nil, Penalty: types.NewInt(0), + Duration: time.Since(start), }, actorErr } func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, error) { + start := time.Now() ctx, span := trace.StartSpan(ctx, "vm.ApplyMessage") defer span.End() msg := cmsg.VMMessage() @@ -299,7 +304,8 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, ExitCode: exitcode.SysErrOutOfGas, GasUsed: 0, }, - Penalty: types.BigMul(msg.GasPrice, types.NewInt(uint64(msgGasCost))), + Penalty: types.BigMul(msg.GasPrice, types.NewInt(uint64(msgGasCost))), + Duration: time.Since(start), }, nil } @@ -314,7 +320,8 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, ExitCode: exitcode.SysErrSenderInvalid, GasUsed: 0, }, - Penalty: minerPenaltyAmount, + Penalty: minerPenaltyAmount, + Duration: time.Since(start), }, nil } return nil, xerrors.Errorf("failed to look up from actor: %w", err) @@ -326,7 +333,8 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, ExitCode: exitcode.SysErrSenderInvalid, GasUsed: 0, }, - Penalty: minerPenaltyAmount, + Penalty: minerPenaltyAmount, + Duration: time.Since(start), }, nil } @@ -336,7 +344,8 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, ExitCode: exitcode.SysErrSenderStateInvalid, GasUsed: 0, }, - Penalty: minerPenaltyAmount, + Penalty: minerPenaltyAmount, + Duration: time.Since(start), }, nil } @@ -348,7 +357,8 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, ExitCode: exitcode.SysErrSenderStateInvalid, GasUsed: 0, }, - Penalty: minerPenaltyAmount, + Penalty: minerPenaltyAmount, + Duration: time.Since(start), }, nil } @@ -423,6 +433,7 @@ func (vm *VM) ApplyMessage(ctx context.Context, cmsg types.ChainMsg) (*ApplyRet, ActorErr: actorErr, InternalExecutions: rt.internalExecutions, Penalty: types.NewInt(0), + Duration: time.Since(start), }, nil } diff --git a/cmd/lotus-bench/import.go b/cmd/lotus-bench/import.go new file mode 100644 index 000000000..8d4d5d8e5 --- /dev/null +++ b/cmd/lotus-bench/import.go @@ -0,0 +1,140 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "io/ioutil" + "os" + "runtime/pprof" + "time" + + "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/chain/stmgr" + "github.com/filecoin-project/lotus/chain/store" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/chain/vm" + "github.com/filecoin-project/sector-storage/ffiwrapper" + "github.com/filecoin-project/specs-actors/actors/abi" + "golang.org/x/xerrors" + + "github.com/ipfs/go-datastore" + badger "github.com/ipfs/go-ds-badger2" + blockstore "github.com/ipfs/go-ipfs-blockstore" + "gopkg.in/urfave/cli.v2" +) + +type TipSetExec struct { + TipSet types.TipSetKey + Trace []*api.InvocResult + Duration time.Duration +} + +var importBenchCmd = &cli.Command{ + Name: "import", + Usage: "benchmark chain import and validation", + Flags: []cli.Flag{ + &cli.Int64Flag{ + Name: "height", + Usage: "halt validation after given height", + }, + }, + Action: func(cctx *cli.Context) error { + if !cctx.Args().Present() { + fmt.Println("must pass car file of chain to benchmark importing") + return nil + } + + cfi, err := os.Open(cctx.Args().First()) + if err != nil { + return err + } + defer cfi.Close() + + tdir, err := ioutil.TempDir("", "lotus-import-bench") + if err != nil { + return err + } + + bds, err := badger.NewDatastore(tdir, nil) + if err != nil { + return err + } + bs := blockstore.NewBlockstore(bds) + ds := datastore.NewMapDatastore() + cs := store.NewChainStore(bs, ds, vm.Syscalls(ffiwrapper.ProofVerifier)) + stm := stmgr.NewStateManager(cs) + + prof, err := os.Create("import-bench.prof") + if err != nil { + return err + } + defer prof.Close() + + if err := pprof.StartCPUProfile(prof); err != nil { + return err + } + + head, err := cs.Import(cfi) + if err != nil { + return err + } + + if h := cctx.Int64("height"); h != 0 { + tsh, err := cs.GetTipsetByHeight(context.TODO(), abi.ChainEpoch(h), head) + if err != nil { + return err + } + head = tsh + } + + ts := head + tschain := []*types.TipSet{ts} + for ts.Height() != 0 { + next, err := cs.LoadTipSet(ts.Parents()) + if err != nil { + return err + } + + tschain = append(tschain, next) + ts = next + } + + out := make([]TipSetExec, 0, len(tschain)) + + lastState := tschain[len(tschain)-1].ParentState() + for i := len(tschain) - 2; i >= 0; i-- { + cur := tschain[i] + log.Infof("computing state (height: %d, ts=%s)", cur.Height(), cur.Cids()) + if cur.ParentState() != lastState { + return xerrors.Errorf("tipset chain had state mismatch at height %d", cur.Height()) + } + start := time.Now() + st, trace, err := stm.ExecutionTrace(context.TODO(), cur) + if err != nil { + return err + } + out = append(out, TipSetExec{ + TipSet: cur.Key(), + Trace: trace, + Duration: time.Since(start), + }) + lastState = st + } + + pprof.StopCPUProfile() + + ibj, err := os.Create("import-bench.json") + if err != nil { + return err + } + defer ibj.Close() + + if err := json.NewEncoder(ibj).Encode(out); err != nil { + return err + } + + return nil + + }, +} diff --git a/cmd/lotus-bench/main.go b/cmd/lotus-bench/main.go index 995a9c5d9..26ff5e81a 100644 --- a/cmd/lotus-bench/main.go +++ b/cmd/lotus-bench/main.go @@ -74,418 +74,8 @@ func main() { Version: build.UserVersion, Commands: []*cli.Command{ proveCmd, - }, - Flags: []cli.Flag{ - &cli.StringFlag{ - Name: "storage-dir", - Value: "~/.lotus-bench", - Usage: "Path to the storage directory that will store sectors long term", - }, - &cli.StringFlag{ - Name: "sector-size", - Value: "512MiB", - Usage: "size of the sectors in bytes, i.e. 32GiB", - }, - &cli.BoolFlag{ - Name: "no-gpu", - Usage: "disable gpu usage for the benchmark run", - }, - &cli.StringFlag{ - Name: "miner-addr", - Usage: "pass miner address (only necessary if using existing sectorbuilder)", - Value: "t01000", - }, - &cli.StringFlag{ - Name: "benchmark-existing-sectorbuilder", - Usage: "pass a directory to run election-post timings on an existing sectorbuilder", - }, - &cli.BoolFlag{ - Name: "json-out", - Usage: "output results in json format", - }, - &cli.BoolFlag{ - Name: "skip-commit2", - Usage: "skip the commit2 (snark) portion of the benchmark", - }, - &cli.BoolFlag{ - Name: "skip-unseal", - Usage: "skip the unseal portion of the benchmark", - }, - &cli.StringFlag{ - Name: "save-commit2-input", - Usage: "Save commit2 input to a file", - }, - }, - Action: func(c *cli.Context) error { - if c.Bool("no-gpu") { - os.Setenv("BELLMAN_NO_GPU", "1") - } - - robench := c.String("benchmark-existing-sectorbuilder") - - var sbdir string - - if robench == "" { - sdir, err := homedir.Expand(c.String("storage-dir")) - if err != nil { - return err - } - - os.MkdirAll(sdir, 0775) - - tsdir, err := ioutil.TempDir(sdir, "bench") - if err != nil { - return err - } - defer func() { - if err := os.RemoveAll(tsdir); err != nil { - log.Warn("remove all: ", err) - } - }() - sbdir = tsdir - } else { - exp, err := homedir.Expand(robench) - if err != nil { - return err - } - sbdir = exp - } - - maddr, err := address.NewFromString(c.String("miner-addr")) - if err != nil { - return err - } - - sectorSizeInt, err := units.RAMInBytes(c.String("sector-size")) - if err != nil { - return err - } - sectorSize := abi.SectorSize(sectorSizeInt) - - ppt, spt, err := ffiwrapper.ProofTypeFromSectorSize(sectorSize) - if err != nil { - return err - } - - cfg := &ffiwrapper.Config{ - SealProofType: spt, - PoStProofType: ppt, - } - - if robench == "" { - if err := os.MkdirAll(sbdir, 0775); err != nil { - return err - } - } - - if !c.Bool("skip-commit2") { - if err := paramfetch.GetParams(build.ParametersJson(), uint64(sectorSize)); err != nil { - return xerrors.Errorf("getting params: %w", err) - } - } - - sbfs := &basicfs.Provider{ - Root: sbdir, - } - - sb, err := ffiwrapper.New(sbfs, cfg) - if err != nil { - return err - } - - amid, err := address.IDFromAddress(maddr) - if err != nil { - return err - } - - mid := abi.ActorID(amid) - - var sealTimings []SealingResult - var sealedSectors []abi.SectorInfo - numSectors := abi.SectorNumber(1) - for i := abi.SectorNumber(1); i <= numSectors && robench == ""; i++ { - sid := abi.SectorID{ - Miner: mid, - Number: i, - } - - start := time.Now() - log.Info("Writing piece into sector...") - - r := rand.New(rand.NewSource(100 + int64(i))) - - pi, err := sb.AddPiece(context.TODO(), sid, nil, abi.PaddedPieceSize(sectorSize).Unpadded(), r) - if err != nil { - return err - } - - addpiece := time.Now() - - trand := sha256.Sum256([]byte(c.String("ticket-preimage"))) - ticket := abi.SealRandomness(trand[:]) - - log.Info("Running replication(1)...") - pieces := []abi.PieceInfo{pi} - pc1o, err := sb.SealPreCommit1(context.TODO(), sid, ticket, pieces) - if err != nil { - return xerrors.Errorf("commit: %w", err) - } - - precommit1 := time.Now() - - log.Info("Running replication(2)...") - cids, err := sb.SealPreCommit2(context.TODO(), sid, pc1o) - if err != nil { - return xerrors.Errorf("commit: %w", err) - } - - precommit2 := time.Now() - - sealedSectors = append(sealedSectors, abi.SectorInfo{ - RegisteredProof: spt, - SectorNumber: i, - SealedCID: cids.Sealed, - }) - - seed := lapi.SealSeed{ - Epoch: 101, - Value: []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 255}, - } - - log.Info("Generating PoRep for sector (1)") - c1o, err := sb.SealCommit1(context.TODO(), sid, ticket, seed.Value, pieces, cids) - if err != nil { - return err - } - - sealcommit1 := time.Now() - - log.Info("Generating PoRep for sector (2)") - - if c.String("save-commit2-input") != "" { - c2in := Commit2In{ - SectorNum: int64(i), - Phase1Out: c1o, - SectorSize: uint64(sectorSize), - } - - b, err := json.Marshal(&c2in) - if err != nil { - return err - } - - if err := ioutil.WriteFile(c.String("save-commit2-input"), b, 0664); err != nil { - log.Warnf("%+v", err) - } - } - - var proof storage.Proof - if !c.Bool("skip-commit2") { - proof, err = sb.SealCommit2(context.TODO(), sid, c1o) - if err != nil { - return err - } - } - - sealcommit2 := time.Now() - - if !c.Bool("skip-commit2") { - - svi := abi.SealVerifyInfo{ - SectorID: abi.SectorID{Miner: mid, Number: i}, - OnChain: abi.OnChainSealVerifyInfo{ - SealedCID: cids.Sealed, - InteractiveEpoch: seed.Epoch, - RegisteredProof: spt, - Proof: proof, - DealIDs: nil, - SectorNumber: i, - SealRandEpoch: 0, - }, - Randomness: ticket, - InteractiveRandomness: seed.Value, - UnsealedCID: cids.Unsealed, - } - - ok, err := ffiwrapper.ProofVerifier.VerifySeal(svi) - if err != nil { - return err - } - if !ok { - return xerrors.Errorf("porep proof for sector %d was invalid", i) - } - } - - verifySeal := time.Now() - - if !c.Bool("skip-unseal") { - log.Info("Unsealing sector") - // TODO: RM unsealed sector first - rc, err := sb.ReadPieceFromSealedSector(context.TODO(), abi.SectorID{Miner: mid, Number: 1}, 0, abi.UnpaddedPieceSize(sectorSize), ticket, cids.Unsealed) - if err != nil { - return err - } - - if err := rc.Close(); err != nil { - return err - } - } - unseal := time.Now() - - sealTimings = append(sealTimings, SealingResult{ - AddPiece: addpiece.Sub(start), - PreCommit1: precommit1.Sub(addpiece), - PreCommit2: precommit2.Sub(precommit1), - Commit1: sealcommit1.Sub(precommit2), - Commit2: sealcommit2.Sub(sealcommit1), - Verify: verifySeal.Sub(sealcommit2), - Unseal: unseal.Sub(verifySeal), - }) - } - - beforePost := time.Now() - - var challenge [32]byte - rand.Read(challenge[:]) - - if robench != "" { - // TODO: this assumes we only ever benchmark a preseal - // sectorbuilder directory... we need a better way to handle - // this in other cases - - fdata, err := ioutil.ReadFile(filepath.Join(sbdir, "pre-seal-"+maddr.String()+".json")) - if err != nil { - return err - } - - var genmm map[string]genesis.Miner - if err := json.Unmarshal(fdata, &genmm); err != nil { - return err - } - - genm, ok := genmm[maddr.String()] - if !ok { - return xerrors.Errorf("preseal file didnt have expected miner in it") - } - - for _, s := range genm.Sectors { - sealedSectors = append(sealedSectors, abi.SectorInfo{ - SealedCID: s.CommR, - SectorNumber: s.SectorID, - }) - } - } - - bo := BenchResults{ - SectorSize: sectorSize, - SealingResults: sealTimings, - } - - if !c.Bool("skip-commit2") { - log.Info("generating election post candidates") - fcandidates, err := sb.GenerateEPostCandidates(context.TODO(), mid, sealedSectors, challenge[:], []abi.SectorNumber{}) - if err != nil { - return err - } - - var candidates []abi.PoStCandidate - for _, c := range fcandidates { - c.Candidate.RegisteredProof = ppt - candidates = append(candidates, c.Candidate) - } - - gencandidates := time.Now() - - log.Info("computing election post snark (cold)") - proof1, err := sb.ComputeElectionPoSt(context.TODO(), mid, sealedSectors, challenge[:], candidates[:1]) - if err != nil { - return err - } - - epost1 := time.Now() - - log.Info("computing election post snark (hot)") - proof2, err := sb.ComputeElectionPoSt(context.TODO(), mid, sealedSectors, challenge[:], candidates[:1]) - if err != nil { - return err - } - - epost2 := time.Now() - - ccount := ffiwrapper.ElectionPostChallengeCount(uint64(len(sealedSectors)), 0) - - pvi1 := abi.PoStVerifyInfo{ - Randomness: abi.PoStRandomness(challenge[:]), - Candidates: candidates[:1], - Proofs: proof1, - EligibleSectors: sealedSectors, - Prover: mid, - ChallengeCount: ccount, - } - ok, err := ffiwrapper.ProofVerifier.VerifyElectionPost(context.TODO(), pvi1) - if err != nil { - return err - } - if !ok { - log.Error("post verification failed") - } - - verifypost1 := time.Now() - - pvi2 := abi.PoStVerifyInfo{ - Randomness: abi.PoStRandomness(challenge[:]), - Candidates: candidates[:1], - Proofs: proof2, - EligibleSectors: sealedSectors, - Prover: mid, - ChallengeCount: ccount, - } - - ok, err = ffiwrapper.ProofVerifier.VerifyElectionPost(context.TODO(), pvi2) - if err != nil { - return err - } - if !ok { - log.Error("post verification failed") - } - verifypost2 := time.Now() - - bo.PostGenerateCandidates = gencandidates.Sub(beforePost) - bo.PostEProofCold = epost1.Sub(gencandidates) - bo.PostEProofHot = epost2.Sub(epost1) - bo.VerifyEPostCold = verifypost1.Sub(epost2) - bo.VerifyEPostHot = verifypost2.Sub(verifypost1) - } - - if c.Bool("json-out") { - data, err := json.MarshalIndent(bo, "", " ") - if err != nil { - return err - } - - fmt.Println(string(data)) - } else { - fmt.Printf("----\nresults (v24) (%d)\n", sectorSize) - if robench == "" { - fmt.Printf("seal: addPiece: %s (%s)\n", bo.SealingResults[0].AddPiece, bps(bo.SectorSize, bo.SealingResults[0].AddPiece)) // TODO: average across multiple sealings - fmt.Printf("seal: preCommit phase 1: %s (%s)\n", bo.SealingResults[0].PreCommit1, bps(bo.SectorSize, bo.SealingResults[0].PreCommit1)) - fmt.Printf("seal: preCommit phase 2: %s (%s)\n", bo.SealingResults[0].PreCommit2, bps(bo.SectorSize, bo.SealingResults[0].PreCommit2)) - fmt.Printf("seal: commit phase 1: %s (%s)\n", bo.SealingResults[0].Commit1, bps(bo.SectorSize, bo.SealingResults[0].Commit1)) - fmt.Printf("seal: commit phase 2: %s (%s)\n", bo.SealingResults[0].Commit2, bps(bo.SectorSize, bo.SealingResults[0].Commit2)) - fmt.Printf("seal: verify: %s\n", bo.SealingResults[0].Verify) - if !c.Bool("skip-unseal") { - fmt.Printf("unseal: %s (%s)\n", bo.SealingResults[0].Unseal, bps(bo.SectorSize, bo.SealingResults[0].Unseal)) - } - } - if !c.Bool("skip-commit2") { - fmt.Printf("generate candidates: %s (%s)\n", bo.PostGenerateCandidates, bps(bo.SectorSize*abi.SectorSize(len(bo.SealingResults)), bo.PostGenerateCandidates)) - fmt.Printf("compute epost proof (cold): %s\n", bo.PostEProofCold) - fmt.Printf("compute epost proof (hot): %s\n", bo.PostEProofHot) - fmt.Printf("verify epost proof (cold): %s\n", bo.VerifyEPostCold) - fmt.Printf("verify epost proof (hot): %s\n", bo.VerifyEPostHot) - } - } - return nil + sealBenchCmd, + importBenchCmd, }, } @@ -495,6 +85,422 @@ func main() { } } +var sealBenchCmd = &cli.Command{ + Name: "sealing", + Flags: []cli.Flag{ + &cli.StringFlag{ + Name: "storage-dir", + Value: "~/.lotus-bench", + Usage: "Path to the storage directory that will store sectors long term", + }, + &cli.StringFlag{ + Name: "sector-size", + Value: "512MiB", + Usage: "size of the sectors in bytes, i.e. 32GiB", + }, + &cli.BoolFlag{ + Name: "no-gpu", + Usage: "disable gpu usage for the benchmark run", + }, + &cli.StringFlag{ + Name: "miner-addr", + Usage: "pass miner address (only necessary if using existing sectorbuilder)", + Value: "t01000", + }, + &cli.StringFlag{ + Name: "benchmark-existing-sectorbuilder", + Usage: "pass a directory to run election-post timings on an existing sectorbuilder", + }, + &cli.BoolFlag{ + Name: "json-out", + Usage: "output results in json format", + }, + &cli.BoolFlag{ + Name: "skip-commit2", + Usage: "skip the commit2 (snark) portion of the benchmark", + }, + &cli.BoolFlag{ + Name: "skip-unseal", + Usage: "skip the unseal portion of the benchmark", + }, + &cli.StringFlag{ + Name: "save-commit2-input", + Usage: "Save commit2 input to a file", + }, + }, + Action: func(c *cli.Context) error { + if c.Bool("no-gpu") { + os.Setenv("BELLMAN_NO_GPU", "1") + } + + robench := c.String("benchmark-existing-sectorbuilder") + + var sbdir string + + if robench == "" { + sdir, err := homedir.Expand(c.String("storage-dir")) + if err != nil { + return err + } + + os.MkdirAll(sdir, 0775) + + tsdir, err := ioutil.TempDir(sdir, "bench") + if err != nil { + return err + } + defer func() { + if err := os.RemoveAll(tsdir); err != nil { + log.Warn("remove all: ", err) + } + }() + sbdir = tsdir + } else { + exp, err := homedir.Expand(robench) + if err != nil { + return err + } + sbdir = exp + } + + maddr, err := address.NewFromString(c.String("miner-addr")) + if err != nil { + return err + } + + sectorSizeInt, err := units.RAMInBytes(c.String("sector-size")) + if err != nil { + return err + } + sectorSize := abi.SectorSize(sectorSizeInt) + + ppt, spt, err := ffiwrapper.ProofTypeFromSectorSize(sectorSize) + if err != nil { + return err + } + + cfg := &ffiwrapper.Config{ + SealProofType: spt, + PoStProofType: ppt, + } + + if robench == "" { + if err := os.MkdirAll(sbdir, 0775); err != nil { + return err + } + } + + if !c.Bool("skip-commit2") { + if err := paramfetch.GetParams(build.ParametersJson(), uint64(sectorSize)); err != nil { + return xerrors.Errorf("getting params: %w", err) + } + } + + sbfs := &basicfs.Provider{ + Root: sbdir, + } + + sb, err := ffiwrapper.New(sbfs, cfg) + if err != nil { + return err + } + + amid, err := address.IDFromAddress(maddr) + if err != nil { + return err + } + + mid := abi.ActorID(amid) + + var sealTimings []SealingResult + var sealedSectors []abi.SectorInfo + numSectors := abi.SectorNumber(1) + for i := abi.SectorNumber(1); i <= numSectors && robench == ""; i++ { + sid := abi.SectorID{ + Miner: mid, + Number: i, + } + + start := time.Now() + log.Info("Writing piece into sector...") + + r := rand.New(rand.NewSource(100 + int64(i))) + + pi, err := sb.AddPiece(context.TODO(), sid, nil, abi.PaddedPieceSize(sectorSize).Unpadded(), r) + if err != nil { + return err + } + + addpiece := time.Now() + + trand := sha256.Sum256([]byte(c.String("ticket-preimage"))) + ticket := abi.SealRandomness(trand[:]) + + log.Info("Running replication(1)...") + pieces := []abi.PieceInfo{pi} + pc1o, err := sb.SealPreCommit1(context.TODO(), sid, ticket, pieces) + if err != nil { + return xerrors.Errorf("commit: %w", err) + } + + precommit1 := time.Now() + + log.Info("Running replication(2)...") + cids, err := sb.SealPreCommit2(context.TODO(), sid, pc1o) + if err != nil { + return xerrors.Errorf("commit: %w", err) + } + + precommit2 := time.Now() + + sealedSectors = append(sealedSectors, abi.SectorInfo{ + RegisteredProof: spt, + SectorNumber: i, + SealedCID: cids.Sealed, + }) + + seed := lapi.SealSeed{ + Epoch: 101, + Value: []byte{1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 255}, + } + + log.Info("Generating PoRep for sector (1)") + c1o, err := sb.SealCommit1(context.TODO(), sid, ticket, seed.Value, pieces, cids) + if err != nil { + return err + } + + sealcommit1 := time.Now() + + log.Info("Generating PoRep for sector (2)") + + if c.String("save-commit2-input") != "" { + c2in := Commit2In{ + SectorNum: int64(i), + Phase1Out: c1o, + SectorSize: uint64(sectorSize), + } + + b, err := json.Marshal(&c2in) + if err != nil { + return err + } + + if err := ioutil.WriteFile(c.String("save-commit2-input"), b, 0664); err != nil { + log.Warnf("%+v", err) + } + } + + var proof storage.Proof + if !c.Bool("skip-commit2") { + proof, err = sb.SealCommit2(context.TODO(), sid, c1o) + if err != nil { + return err + } + } + + sealcommit2 := time.Now() + + if !c.Bool("skip-commit2") { + + svi := abi.SealVerifyInfo{ + SectorID: abi.SectorID{Miner: mid, Number: i}, + OnChain: abi.OnChainSealVerifyInfo{ + SealedCID: cids.Sealed, + InteractiveEpoch: seed.Epoch, + RegisteredProof: spt, + Proof: proof, + DealIDs: nil, + SectorNumber: i, + SealRandEpoch: 0, + }, + Randomness: ticket, + InteractiveRandomness: seed.Value, + UnsealedCID: cids.Unsealed, + } + + ok, err := ffiwrapper.ProofVerifier.VerifySeal(svi) + if err != nil { + return err + } + if !ok { + return xerrors.Errorf("porep proof for sector %d was invalid", i) + } + } + + verifySeal := time.Now() + + if !c.Bool("skip-unseal") { + log.Info("Unsealing sector") + // TODO: RM unsealed sector first + rc, err := sb.ReadPieceFromSealedSector(context.TODO(), abi.SectorID{Miner: mid, Number: 1}, 0, abi.UnpaddedPieceSize(sectorSize), ticket, cids.Unsealed) + if err != nil { + return err + } + + if err := rc.Close(); err != nil { + return err + } + } + unseal := time.Now() + + sealTimings = append(sealTimings, SealingResult{ + AddPiece: addpiece.Sub(start), + PreCommit1: precommit1.Sub(addpiece), + PreCommit2: precommit2.Sub(precommit1), + Commit1: sealcommit1.Sub(precommit2), + Commit2: sealcommit2.Sub(sealcommit1), + Verify: verifySeal.Sub(sealcommit2), + Unseal: unseal.Sub(verifySeal), + }) + } + + beforePost := time.Now() + + var challenge [32]byte + rand.Read(challenge[:]) + + if robench != "" { + // TODO: this assumes we only ever benchmark a preseal + // sectorbuilder directory... we need a better way to handle + // this in other cases + + fdata, err := ioutil.ReadFile(filepath.Join(sbdir, "pre-seal-"+maddr.String()+".json")) + if err != nil { + return err + } + + var genmm map[string]genesis.Miner + if err := json.Unmarshal(fdata, &genmm); err != nil { + return err + } + + genm, ok := genmm[maddr.String()] + if !ok { + return xerrors.Errorf("preseal file didnt have expected miner in it") + } + + for _, s := range genm.Sectors { + sealedSectors = append(sealedSectors, abi.SectorInfo{ + SealedCID: s.CommR, + SectorNumber: s.SectorID, + }) + } + } + + bo := BenchResults{ + SectorSize: sectorSize, + SealingResults: sealTimings, + } + + if !c.Bool("skip-commit2") { + log.Info("generating election post candidates") + fcandidates, err := sb.GenerateEPostCandidates(context.TODO(), mid, sealedSectors, challenge[:], []abi.SectorNumber{}) + if err != nil { + return err + } + + var candidates []abi.PoStCandidate + for _, c := range fcandidates { + c.Candidate.RegisteredProof = ppt + candidates = append(candidates, c.Candidate) + } + + gencandidates := time.Now() + + log.Info("computing election post snark (cold)") + proof1, err := sb.ComputeElectionPoSt(context.TODO(), mid, sealedSectors, challenge[:], candidates[:1]) + if err != nil { + return err + } + + epost1 := time.Now() + + log.Info("computing election post snark (hot)") + proof2, err := sb.ComputeElectionPoSt(context.TODO(), mid, sealedSectors, challenge[:], candidates[:1]) + if err != nil { + return err + } + + epost2 := time.Now() + + ccount := ffiwrapper.ElectionPostChallengeCount(uint64(len(sealedSectors)), 0) + + pvi1 := abi.PoStVerifyInfo{ + Randomness: abi.PoStRandomness(challenge[:]), + Candidates: candidates[:1], + Proofs: proof1, + EligibleSectors: sealedSectors, + Prover: mid, + ChallengeCount: ccount, + } + ok, err := ffiwrapper.ProofVerifier.VerifyElectionPost(context.TODO(), pvi1) + if err != nil { + return err + } + if !ok { + log.Error("post verification failed") + } + + verifypost1 := time.Now() + + pvi2 := abi.PoStVerifyInfo{ + Randomness: abi.PoStRandomness(challenge[:]), + Candidates: candidates[:1], + Proofs: proof2, + EligibleSectors: sealedSectors, + Prover: mid, + ChallengeCount: ccount, + } + + ok, err = ffiwrapper.ProofVerifier.VerifyElectionPost(context.TODO(), pvi2) + if err != nil { + return err + } + if !ok { + log.Error("post verification failed") + } + verifypost2 := time.Now() + + bo.PostGenerateCandidates = gencandidates.Sub(beforePost) + bo.PostEProofCold = epost1.Sub(gencandidates) + bo.PostEProofHot = epost2.Sub(epost1) + bo.VerifyEPostCold = verifypost1.Sub(epost2) + bo.VerifyEPostHot = verifypost2.Sub(verifypost1) + } + + if c.Bool("json-out") { + data, err := json.MarshalIndent(bo, "", " ") + if err != nil { + return err + } + + fmt.Println(string(data)) + } else { + fmt.Printf("----\nresults (v24) (%d)\n", sectorSize) + if robench == "" { + fmt.Printf("seal: addPiece: %s (%s)\n", bo.SealingResults[0].AddPiece, bps(bo.SectorSize, bo.SealingResults[0].AddPiece)) // TODO: average across multiple sealings + fmt.Printf("seal: preCommit phase 1: %s (%s)\n", bo.SealingResults[0].PreCommit1, bps(bo.SectorSize, bo.SealingResults[0].PreCommit1)) + fmt.Printf("seal: preCommit phase 2: %s (%s)\n", bo.SealingResults[0].PreCommit2, bps(bo.SectorSize, bo.SealingResults[0].PreCommit2)) + fmt.Printf("seal: commit phase 1: %s (%s)\n", bo.SealingResults[0].Commit1, bps(bo.SectorSize, bo.SealingResults[0].Commit1)) + fmt.Printf("seal: commit phase 2: %s (%s)\n", bo.SealingResults[0].Commit2, bps(bo.SectorSize, bo.SealingResults[0].Commit2)) + fmt.Printf("seal: verify: %s\n", bo.SealingResults[0].Verify) + if !c.Bool("skip-unseal") { + fmt.Printf("unseal: %s (%s)\n", bo.SealingResults[0].Unseal, bps(bo.SectorSize, bo.SealingResults[0].Unseal)) + } + } + if !c.Bool("skip-commit2") { + fmt.Printf("generate candidates: %s (%s)\n", bo.PostGenerateCandidates, bps(bo.SectorSize*abi.SectorSize(len(bo.SealingResults)), bo.PostGenerateCandidates)) + fmt.Printf("compute epost proof (cold): %s\n", bo.PostEProofCold) + fmt.Printf("compute epost proof (hot): %s\n", bo.PostEProofHot) + fmt.Printf("verify epost proof (cold): %s\n", bo.VerifyEPostCold) + fmt.Printf("verify epost proof (hot): %s\n", bo.VerifyEPostHot) + } + } + return nil + }, +} + var proveCmd = &cli.Command{ Name: "prove", Usage: "Benchmark a proof computation", diff --git a/go.sum b/go.sum index 5d25465d7..9054f8ad2 100644 --- a/go.sum +++ b/go.sum @@ -642,6 +642,7 @@ github.com/mattn/go-runewidth v0.0.7 h1:Ei8KR0497xHyKJPAv59M1dkC+rOZCMBJ+t3fZ+tw github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= +github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g= @@ -791,6 +792,7 @@ github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tL github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/src-d/envconfig v1.0.0 h1:/AJi6DtjFhZKNx3OB2qMsq7y4yT5//AeSZIe7rk+PX8= github.com/src-d/envconfig v1.0.0/go.mod h1:Q9YQZ7BKITldTBnoxsE5gOeB5y66RyPXeue/R4aaNBc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= @@ -847,6 +849,7 @@ github.com/whyrusleeping/pubsub v0.0.0-20131020042734-02de8aa2db3d/go.mod h1:g7c github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee h1:lYbXeSvJi5zk5GLKVuid9TVjS9a0OmLIDKTfoZBL6Ow= github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee/go.mod h1:m2aV4LZI4Aez7dP5PMyVKEHhUyEJ/RjmPEDOpDvudHg= github.com/whyrusleeping/yamux v1.1.5/go.mod h1:E8LnQQ8HKx5KD29HZFUwM1PxCOdPRzGwur1mcYhXcD8= +github.com/x-cray/logrus-prefixed-formatter v0.5.2 h1:00txxvfBM9muc0jiLIEAkAcIMJzfthRT6usrui8uGmg= github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -1035,6 +1038,7 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/src-d/go-cli.v0 v0.0.0-20181105080154-d492247bbc0d/go.mod h1:z+K8VcOYVYcSwSjGebuDL6176A1XskgbtNl64NSg+n8= +gopkg.in/src-d/go-log.v1 v1.0.1 h1:heWvX7J6qbGWbeFS/aRmiy1eYaT+QMV6wNvHDyMjQV4= gopkg.in/src-d/go-log.v1 v1.0.1/go.mod h1:GN34hKP0g305ysm2/hctJ0Y8nWP3zxXXJ8GFabTyABE= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= diff --git a/node/impl/full/state.go b/node/impl/full/state.go index 023faa3b2..ade22e228 100644 --- a/node/impl/full/state.go +++ b/node/impl/full/state.go @@ -193,6 +193,7 @@ func (a *StateAPI) StateReplay(ctx context.Context, tsk types.TipSetKey, mc cid. MsgRct: &r.MessageReceipt, InternalExecutions: r.InternalExecutions, Error: errstr, + Duration: r.Duration, }, nil }