From 22267eb45dd5fcabb31ec52f49bc3892491b2a9c Mon Sep 17 00:00:00 2001 From: Steven Allen Date: Mon, 14 Jun 2021 13:05:28 -0700 Subject: [PATCH] feat(lotus-sim): split info command file --- chain/actors/builtin/miner/miner.go | 2 + cmd/lotus-sim/info.go | 234 +--------------------------- cmd/lotus-sim/info_capacity.go | 63 ++++++++ cmd/lotus-sim/info_commit.go | 144 +++++++++++++++++ cmd/lotus-sim/info_wdpost.go | 65 ++++++++ 5 files changed, 275 insertions(+), 233 deletions(-) create mode 100644 cmd/lotus-sim/info_capacity.go create mode 100644 cmd/lotus-sim/info_commit.go create mode 100644 cmd/lotus-sim/info_wdpost.go diff --git a/chain/actors/builtin/miner/miner.go b/chain/actors/builtin/miner/miner.go index bb7f80340..995dc78cb 100644 --- a/chain/actors/builtin/miner/miner.go +++ b/chain/actors/builtin/miner/miner.go @@ -22,6 +22,7 @@ import ( miner0 "github.com/filecoin-project/specs-actors/actors/builtin/miner" miner2 "github.com/filecoin-project/specs-actors/v2/actors/builtin/miner" miner3 "github.com/filecoin-project/specs-actors/v3/actors/builtin/miner" + miner5 "github.com/filecoin-project/specs-actors/v5/actors/builtin/miner" builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" @@ -239,6 +240,7 @@ type DeclareFaultsRecoveredParams = miner0.DeclareFaultsRecoveredParams type SubmitWindowedPoStParams = miner0.SubmitWindowedPoStParams type ProveCommitSectorParams = miner0.ProveCommitSectorParams type DisputeWindowedPoStParams = miner3.DisputeWindowedPoStParams +type ProveCommitAggregateParams = miner5.ProveCommitAggregateParams func PreferredSealProofTypeFromWindowPoStType(nver network.Version, proof abi.RegisteredPoStProof) (abi.RegisteredSealProof, error) { // We added support for the new proofs in network version 7, and removed support for the old diff --git a/cmd/lotus-sim/info.go b/cmd/lotus-sim/info.go index ef20b7f26..4cd453440 100644 --- a/cmd/lotus-sim/info.go +++ b/cmd/lotus-sim/info.go @@ -1,29 +1,21 @@ package main import ( - "bytes" "context" "fmt" "io" - "os" - "syscall" "text/tabwriter" "time" - "github.com/ipfs/go-cid" - "github.com/streadway/quantile" "github.com/urfave/cli/v2" "github.com/filecoin-project/go-state-types/big" - "github.com/filecoin-project/go-state-types/exitcode" - "github.com/filecoin-project/specs-actors/v5/actors/builtin" - "github.com/filecoin-project/specs-actors/v5/actors/builtin/miner" + "github.com/filecoin-project/lotus/chain/actors/builtin" "github.com/filecoin-project/lotus/chain/actors/builtin/power" "github.com/filecoin-project/lotus/chain/stmgr" "github.com/filecoin-project/lotus/chain/types" "github.com/filecoin-project/lotus/cmd/lotus-sim/simulation" - "github.com/filecoin-project/lotus/lib/stati" ) func getTotalPower(ctx context.Context, sm *stmgr.StateManager, ts *types.TipSet) (power.Claim, error) { @@ -110,227 +102,3 @@ var infoSimCommand = &cli.Command{ return printInfo(cctx.Context, sim, cctx.App.Writer) }, } - -var infoWindowPostBandwidthSimCommand = &cli.Command{ - Name: "post-bandwidth", - Description: "List average chain bandwidth used by window posts for each day of the simulation.", - Action: func(cctx *cli.Context) error { - node, err := open(cctx) - if err != nil { - return err - } - defer node.Close() - - sim, err := node.LoadSim(cctx.Context, cctx.String("simulation")) - if err != nil { - return err - } - - var postGas, totalGas int64 - printStats := func() { - fmt.Fprintf(cctx.App.Writer, "%.4f%%\n", float64(100*postGas)/float64(totalGas)) - } - idx := 0 - err = sim.Walk(cctx.Context, 0, func( - sm *stmgr.StateManager, ts *types.TipSet, stCid cid.Cid, - messages []*simulation.AppliedMessage, - ) error { - for _, m := range messages { - totalGas += m.GasUsed - if m.ExitCode != exitcode.Ok { - continue - } - if m.Method == builtin.MethodsMiner.SubmitWindowedPoSt { - postGas += m.GasUsed - } - } - idx++ - idx %= builtin.EpochsInDay - if idx == 0 { - printStats() - postGas = 0 - totalGas = 0 - } - return nil - }) - if idx > 0 { - printStats() - } - return err - }, -} - -var infoCapacityGrowthSimCommand = &cli.Command{ - Name: "capacity-growth", - Description: "List daily capacity growth over the course of the simulation starting at the end.", - Action: func(cctx *cli.Context) error { - node, err := open(cctx) - if err != nil { - return err - } - defer node.Close() - - sim, err := node.LoadSim(cctx.Context, cctx.String("simulation")) - if err != nil { - return err - } - - firstEpoch := sim.GetStart().Height() - ts := sim.GetHead() - lastPower, err := getTotalPower(cctx.Context, sim.StateManager, ts) - if err != nil { - return err - } - lastHeight := ts.Height() - - for ts.Height() > firstEpoch && cctx.Err() == nil { - ts, err = sim.Chainstore.LoadTipSet(ts.Parents()) - if err != nil { - return err - } - newEpoch := ts.Height() - if newEpoch != firstEpoch && newEpoch+builtin.EpochsInDay > lastHeight { - continue - } - - newPower, err := getTotalPower(cctx.Context, sim.StateManager, ts) - if err != nil { - return err - } - - growthRate := big.Div( - big.Mul(big.Sub(lastPower.RawBytePower, newPower.RawBytePower), - big.NewInt(builtin.EpochsInDay)), - big.NewInt(int64(lastHeight-newEpoch)), - ) - lastPower = newPower - lastHeight = newEpoch - fmt.Fprintf(cctx.App.Writer, "%s/day\n", types.SizeStr(growthRate)) - } - return cctx.Err() - }, -} - -var infoCommitGasSimCommand = &cli.Command{ - Name: "commit-gas", - Description: "Output information about the gas for commits", - Flags: []cli.Flag{ - &cli.Int64Flag{ - Name: "lookback", - Value: 0, - }, - }, - Action: func(cctx *cli.Context) error { - log := func(f string, i ...interface{}) { - fmt.Fprintf(os.Stderr, f, i...) - } - node, err := open(cctx) - if err != nil { - return err - } - defer node.Close() - - go profileOnSignal(cctx, syscall.SIGUSR2) - - sim, err := node.LoadSim(cctx.Context, cctx.String("simulation")) - if err != nil { - return err - } - - var gasAgg, proofsAgg uint64 - var gasAggMax, proofsAggMax uint64 - var gasSingle, proofsSingle uint64 - - qpoints := []struct{ q, tol float64 }{ - {0.01, 0.0005}, - {0.05, 0.001}, - {0.20, 0.01}, - {0.25, 0.01}, - {0.30, 0.01}, - {0.40, 0.01}, - {0.45, 0.01}, - {0.50, 0.01}, - {0.60, 0.01}, - {0.80, 0.01}, - {0.95, 0.001}, - {0.99, 0.0005}, - } - estims := make([]quantile.Estimate, len(qpoints)) - for i, p := range qpoints { - estims[i] = quantile.Known(p.q, p.tol) - } - qua := quantile.New(estims...) - hist, err := stati.NewHistogram([]float64{ - 1, 3, 5, 7, 15, 30, 50, 100, 200, 400, 600, 700, 819}) - if err != nil { - return err - } - - err = sim.Walk(cctx.Context, cctx.Int64("lookback"), func( - sm *stmgr.StateManager, ts *types.TipSet, stCid cid.Cid, - messages []*simulation.AppliedMessage, - ) error { - for _, m := range messages { - if m.ExitCode != exitcode.Ok { - continue - } - if m.Method == builtin.MethodsMiner.ProveCommitAggregate { - param := miner.ProveCommitAggregateParams{} - err := param.UnmarshalCBOR(bytes.NewReader(m.Params)) - if err != nil { - log("failed to decode params: %+v", err) - return nil - } - c, err := param.SectorNumbers.Count() - if err != nil { - log("failed to count sectors") - return nil - } - gasAgg += uint64(m.GasUsed) - proofsAgg += c - if c == 819 { - gasAggMax += uint64(m.GasUsed) - proofsAggMax += c - } - for i := uint64(0); i < c; i++ { - qua.Add(float64(c)) - } - hist.Observe(float64(c)) - } - - if m.Method == builtin.MethodsMiner.ProveCommitSector { - gasSingle += uint64(m.GasUsed) - proofsSingle++ - qua.Add(1) - hist.Observe(1) - } - } - - return nil - }) - if err != nil { - return err - } - idealGassUsed := float64(gasAggMax) / float64(proofsAggMax) * float64(proofsAgg+proofsSingle) - - fmt.Printf("Gas usage efficiency in comparison to all 819: %f%%\n", 100*idealGassUsed/float64(gasAgg+gasSingle)) - - fmt.Printf("Proofs in singles: %d\n", proofsSingle) - fmt.Printf("Proofs in Aggs: %d\n", proofsAgg) - fmt.Printf("Proofs in Aggs(819): %d\n", proofsAggMax) - - fmt.Println() - fmt.Println("Quantiles of proofs in given aggregate size:") - for _, p := range qpoints { - fmt.Printf("%.0f%%\t%.0f\n", p.q*100, qua.Get(p.q)) - } - fmt.Println() - fmt.Println("Histogram of messages:") - fmt.Printf("Total\t%d\n", hist.Total()) - for i, b := range hist.Buckets[1:] { - fmt.Printf("%.0f\t%d\n", b, hist.Get(i)) - } - - return nil - }, -} diff --git a/cmd/lotus-sim/info_capacity.go b/cmd/lotus-sim/info_capacity.go new file mode 100644 index 000000000..14ee36f08 --- /dev/null +++ b/cmd/lotus-sim/info_capacity.go @@ -0,0 +1,63 @@ +package main + +import ( + "fmt" + + "github.com/urfave/cli/v2" + + "github.com/filecoin-project/go-state-types/big" + + "github.com/filecoin-project/lotus/chain/actors/builtin" + "github.com/filecoin-project/lotus/chain/types" +) + +var infoCapacityGrowthSimCommand = &cli.Command{ + Name: "capacity-growth", + Description: "List daily capacity growth over the course of the simulation starting at the end.", + Action: func(cctx *cli.Context) error { + node, err := open(cctx) + if err != nil { + return err + } + defer node.Close() + + sim, err := node.LoadSim(cctx.Context, cctx.String("simulation")) + if err != nil { + return err + } + + firstEpoch := sim.GetStart().Height() + ts := sim.GetHead() + lastPower, err := getTotalPower(cctx.Context, sim.StateManager, ts) + if err != nil { + return err + } + lastHeight := ts.Height() + + for ts.Height() > firstEpoch && cctx.Err() == nil { + ts, err = sim.Chainstore.LoadTipSet(ts.Parents()) + if err != nil { + return err + } + newEpoch := ts.Height() + if newEpoch != firstEpoch && newEpoch+builtin.EpochsInDay > lastHeight { + continue + } + + newPower, err := getTotalPower(cctx.Context, sim.StateManager, ts) + if err != nil { + return err + } + + growthRate := big.Div( + big.Mul(big.Sub(lastPower.RawBytePower, newPower.RawBytePower), + big.NewInt(builtin.EpochsInDay)), + big.NewInt(int64(lastHeight-newEpoch)), + ) + lastPower = newPower + lastHeight = newEpoch + fmt.Fprintf(cctx.App.Writer, "%s/day\n", types.SizeStr(growthRate)) + } + return cctx.Err() + }, +} diff --git a/cmd/lotus-sim/info_commit.go b/cmd/lotus-sim/info_commit.go new file mode 100644 index 000000000..f6b08ea05 --- /dev/null +++ b/cmd/lotus-sim/info_commit.go @@ -0,0 +1,144 @@ +package main + +import ( + "bytes" + "fmt" + "os" + + "github.com/ipfs/go-cid" + "github.com/streadway/quantile" + "github.com/urfave/cli/v2" + "syscall" + + "github.com/filecoin-project/go-state-types/exitcode" + + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" + "github.com/filecoin-project/lotus/chain/stmgr" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/cmd/lotus-sim/simulation" + "github.com/filecoin-project/lotus/lib/stati" +) + +var infoCommitGasSimCommand = &cli.Command{ + Name: "commit-gas", + Description: "Output information about the gas for commits", + Flags: []cli.Flag{ + &cli.Int64Flag{ + Name: "lookback", + Value: 0, + }, + }, + Action: func(cctx *cli.Context) error { + log := func(f string, i ...interface{}) { + fmt.Fprintf(os.Stderr, f, i...) + } + node, err := open(cctx) + if err != nil { + return err + } + defer node.Close() + + go profileOnSignal(cctx, syscall.SIGUSR2) + + sim, err := node.LoadSim(cctx.Context, cctx.String("simulation")) + if err != nil { + return err + } + + var gasAgg, proofsAgg uint64 + var gasAggMax, proofsAggMax uint64 + var gasSingle, proofsSingle uint64 + + qpoints := []struct{ q, tol float64 }{ + {0.01, 0.0005}, + {0.05, 0.001}, + {0.20, 0.01}, + {0.25, 0.01}, + {0.30, 0.01}, + {0.40, 0.01}, + {0.45, 0.01}, + {0.50, 0.01}, + {0.60, 0.01}, + {0.80, 0.01}, + {0.95, 0.001}, + {0.99, 0.0005}, + } + estims := make([]quantile.Estimate, len(qpoints)) + for i, p := range qpoints { + estims[i] = quantile.Known(p.q, p.tol) + } + qua := quantile.New(estims...) + hist, err := stati.NewHistogram([]float64{ + 1, 3, 5, 7, 15, 30, 50, 100, 200, 400, 600, 700, 819}) + if err != nil { + return err + } + + err = sim.Walk(cctx.Context, cctx.Int64("lookback"), func( + sm *stmgr.StateManager, ts *types.TipSet, stCid cid.Cid, + messages []*simulation.AppliedMessage, + ) error { + for _, m := range messages { + if m.ExitCode != exitcode.Ok { + continue + } + if m.Method == miner.Methods.ProveCommitAggregate { + param := miner.ProveCommitAggregateParams{} + err := param.UnmarshalCBOR(bytes.NewReader(m.Params)) + if err != nil { + log("failed to decode params: %+v", err) + return nil + } + c, err := param.SectorNumbers.Count() + if err != nil { + log("failed to count sectors") + return nil + } + gasAgg += uint64(m.GasUsed) + proofsAgg += c + if c == 819 { + gasAggMax += uint64(m.GasUsed) + proofsAggMax += c + } + for i := uint64(0); i < c; i++ { + qua.Add(float64(c)) + } + hist.Observe(float64(c)) + } + + if m.Method == miner.Methods.ProveCommitSector { + gasSingle += uint64(m.GasUsed) + proofsSingle++ + qua.Add(1) + hist.Observe(1) + } + } + + return nil + }) + if err != nil { + return err + } + idealGassUsed := float64(gasAggMax) / float64(proofsAggMax) * float64(proofsAgg+proofsSingle) + + fmt.Printf("Gas usage efficiency in comparison to all 819: %f%%\n", 100*idealGassUsed/float64(gasAgg+gasSingle)) + + fmt.Printf("Proofs in singles: %d\n", proofsSingle) + fmt.Printf("Proofs in Aggs: %d\n", proofsAgg) + fmt.Printf("Proofs in Aggs(819): %d\n", proofsAggMax) + + fmt.Println() + fmt.Println("Quantiles of proofs in given aggregate size:") + for _, p := range qpoints { + fmt.Printf("%.0f%%\t%.0f\n", p.q*100, qua.Get(p.q)) + } + fmt.Println() + fmt.Println("Histogram of messages:") + fmt.Printf("Total\t%d\n", hist.Total()) + for i, b := range hist.Buckets[1:] { + fmt.Printf("%.0f\t%d\n", b, hist.Get(i)) + } + + return nil + }, +} diff --git a/cmd/lotus-sim/info_wdpost.go b/cmd/lotus-sim/info_wdpost.go new file mode 100644 index 000000000..d52cd5a8c --- /dev/null +++ b/cmd/lotus-sim/info_wdpost.go @@ -0,0 +1,65 @@ +package main + +import ( + "fmt" + + "github.com/ipfs/go-cid" + "github.com/urfave/cli/v2" + + "github.com/filecoin-project/go-state-types/exitcode" + + "github.com/filecoin-project/lotus/chain/actors/builtin" + "github.com/filecoin-project/lotus/chain/actors/builtin/miner" + "github.com/filecoin-project/lotus/chain/stmgr" + "github.com/filecoin-project/lotus/chain/types" + "github.com/filecoin-project/lotus/cmd/lotus-sim/simulation" +) + +var infoWindowPostBandwidthSimCommand = &cli.Command{ + Name: "post-bandwidth", + Description: "List average chain bandwidth used by window posts for each day of the simulation.", + Action: func(cctx *cli.Context) error { + node, err := open(cctx) + if err != nil { + return err + } + defer node.Close() + + sim, err := node.LoadSim(cctx.Context, cctx.String("simulation")) + if err != nil { + return err + } + + var postGas, totalGas int64 + printStats := func() { + fmt.Fprintf(cctx.App.Writer, "%.4f%%\n", float64(100*postGas)/float64(totalGas)) + } + idx := 0 + err = sim.Walk(cctx.Context, 0, func( + sm *stmgr.StateManager, ts *types.TipSet, stCid cid.Cid, + messages []*simulation.AppliedMessage, + ) error { + for _, m := range messages { + totalGas += m.GasUsed + if m.ExitCode != exitcode.Ok { + continue + } + if m.Method == miner.Methods.SubmitWindowedPoSt { + postGas += m.GasUsed + } + } + idx++ + idx %= builtin.EpochsInDay + if idx == 0 { + printStats() + postGas = 0 + totalGas = 0 + } + return nil + }) + if idx > 0 { + printStats() + } + return err + }, +}