Merge pull request #833 from filecoin-project/feat/ro-bench-epost
Allow lotus-bench to run election post benchmark presealed sector directory
This commit is contained in:
Łukasz Magiera
GitHub
commit
2943b7da5d
@ -4,6 +4,7 @@ import (
|
||||
"bytes"
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"math/rand"
|
||||
@ -20,6 +21,7 @@ import (
|
||||
|
||||
"github.com/filecoin-project/lotus/build"
|
||||
"github.com/filecoin-project/lotus/chain/address"
|
||||
"github.com/filecoin-project/lotus/genesis"
|
||||
"github.com/filecoin-project/lotus/lib/sectorbuilder"
|
||||
)
|
||||
|
||||
@ -68,29 +70,56 @@ func main() {
|
||||
Name: "no-gpu",
|
||||
Usage: "disable gpu usage for the benchmark run",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "miner-addr",
|
||||
Usage: "pass miner address (only necessary if using existing sectorbuilder)",
|
||||
Value: "t0101",
|
||||
},
|
||||
&cli.StringFlag{
|
||||
Name: "benchmark-existing-sectorbuilder",
|
||||
Usage: "pass a directory to run election-post timings on an existing sectorbuilder",
|
||||
},
|
||||
&cli.BoolFlag{
|
||||
Name: "json-out",
|
||||
Usage: "output results in json format",
|
||||
},
|
||||
},
|
||||
Action: func(c *cli.Context) error {
|
||||
if c.Bool("no-gpu") {
|
||||
os.Setenv("BELLMAN_NO_GPU", "1")
|
||||
}
|
||||
sdir, err := homedir.Expand(c.String("storage-dir"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
os.MkdirAll(sdir, 0775)
|
||||
robench := c.String("benchmark-existing-sectorbuilder")
|
||||
|
||||
tsdir, err := ioutil.TempDir(sdir, "bench")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
if err := os.RemoveAll(tsdir); err != nil {
|
||||
log.Warn("remove all: ", err)
|
||||
var sbdir string
|
||||
|
||||
if robench == "" {
|
||||
sdir, err := homedir.Expand(c.String("storage-dir"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
}()
|
||||
|
||||
maddr, err := address.NewFromString("t0101")
|
||||
os.MkdirAll(sdir, 0775)
|
||||
|
||||
tsdir, err := ioutil.TempDir(sdir, "bench")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer func() {
|
||||
if err := os.RemoveAll(tsdir); err != nil {
|
||||
log.Warn("remove all: ", err)
|
||||
}
|
||||
}()
|
||||
sbdir = tsdir
|
||||
} else {
|
||||
exp, err := homedir.Expand(robench)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
sbdir = exp
|
||||
}
|
||||
|
||||
maddr, err := address.NewFromString(c.String("miner-addr"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -102,14 +131,17 @@ func main() {
|
||||
Miner: maddr,
|
||||
SectorSize: sectorSize,
|
||||
WorkerThreads: 2,
|
||||
CacheDir: filepath.Join(tsdir, "cache"),
|
||||
SealedDir: filepath.Join(tsdir, "sealed"),
|
||||
StagedDir: filepath.Join(tsdir, "staged"),
|
||||
UnsealedDir: filepath.Join(tsdir, "unsealed"),
|
||||
CacheDir: filepath.Join(sbdir, "cache"),
|
||||
SealedDir: filepath.Join(sbdir, "sealed"),
|
||||
StagedDir: filepath.Join(sbdir, "staged"),
|
||||
UnsealedDir: filepath.Join(sbdir, "unsealed"),
|
||||
}
|
||||
for _, d := range []string{cfg.CacheDir, cfg.SealedDir, cfg.StagedDir, cfg.UnsealedDir} {
|
||||
if err := os.MkdirAll(d, 0775); err != nil {
|
||||
return err
|
||||
|
||||
if robench == "" {
|
||||
for _, d := range []string{cfg.CacheDir, cfg.SealedDir, cfg.StagedDir, cfg.UnsealedDir} {
|
||||
if err := os.MkdirAll(d, 0775); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@ -126,7 +158,7 @@ func main() {
|
||||
var sealTimings []SealingResult
|
||||
var sealedSectors []ffi.PublicSectorInfo
|
||||
numSectors := uint64(1)
|
||||
for i := uint64(1); i <= numSectors; i++ {
|
||||
for i := uint64(1); i <= numSectors && robench == ""; i++ {
|
||||
start := time.Now()
|
||||
log.Info("Writing piece into sector...")
|
||||
|
||||
@ -207,6 +239,34 @@ func main() {
|
||||
var challenge [32]byte
|
||||
rand.Read(challenge[:])
|
||||
|
||||
if robench != "" {
|
||||
// TODO: this assumes we only ever benchmark a preseal
|
||||
// sectorbuilder directory... we need a better way to handle
|
||||
// this in other cases
|
||||
|
||||
fdata, err := ioutil.ReadFile(filepath.Join(sbdir, "pre-seal-"+maddr.String()+".json"))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var genmm map[string]genesis.GenesisMiner
|
||||
if err := json.Unmarshal(fdata, &genmm); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
genm, ok := genmm[maddr.String()]
|
||||
if !ok {
|
||||
return xerrors.Errorf("preseal file didnt have expected miner in it")
|
||||
}
|
||||
|
||||
for _, s := range genm.Sectors {
|
||||
sealedSectors = append(sealedSectors, ffi.PublicSectorInfo{
|
||||
CommR: s.CommR,
|
||||
SectorID: s.SectorID,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
log.Info("generating election post candidates")
|
||||
sinfos := sectorbuilder.NewSortedPublicSectorInfo(sealedSectors)
|
||||
candidates, err := sb.GenerateEPostCandidates(sinfos, challenge, []uint64{})
|
||||
@ -266,17 +326,28 @@ func main() {
|
||||
VerifyEPostHot: verifypost2.Sub(verifypost1),
|
||||
} // TODO: optionally write this as json to a file
|
||||
|
||||
fmt.Println("results")
|
||||
fmt.Printf("seal: addPiece: %s\n", benchout.SealingResults[0].AddPiece) // TODO: average across multiple sealings
|
||||
fmt.Printf("seal: preCommit: %s\n", benchout.SealingResults[0].PreCommit)
|
||||
fmt.Printf("seal: Commit: %s\n", benchout.SealingResults[0].Commit)
|
||||
fmt.Printf("seal: Verify: %s\n", benchout.SealingResults[0].Verify)
|
||||
fmt.Printf("unseal: %s\n", benchout.SealingResults[0].Unseal)
|
||||
fmt.Printf("generate candidates: %s\n", benchout.PostGenerateCandidates)
|
||||
fmt.Printf("compute epost proof (cold): %s\n", benchout.PostEProofCold)
|
||||
fmt.Printf("compute epost proof (hot): %s\n", benchout.PostEProofHot)
|
||||
fmt.Printf("verify epost proof (cold): %s\n", benchout.VerifyEPostCold)
|
||||
fmt.Printf("verify epost proof (hot): %s\n", benchout.VerifyEPostHot)
|
||||
if c.Bool("json-out") {
|
||||
data, err := json.MarshalIndent(benchout, "", " ")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
fmt.Println(string(data))
|
||||
} else {
|
||||
fmt.Printf("results (%d)\n", sectorSize)
|
||||
if robench == "" {
|
||||
fmt.Printf("seal: addPiece: %s\n", benchout.SealingResults[0].AddPiece) // TODO: average across multiple sealings
|
||||
fmt.Printf("seal: preCommit: %s\n", benchout.SealingResults[0].PreCommit)
|
||||
fmt.Printf("seal: Commit: %s\n", benchout.SealingResults[0].Commit)
|
||||
fmt.Printf("seal: Verify: %s\n", benchout.SealingResults[0].Verify)
|
||||
fmt.Printf("unseal: %s\n", benchout.SealingResults[0].Unseal)
|
||||
}
|
||||
fmt.Printf("generate candidates: %s\n", benchout.PostGenerateCandidates)
|
||||
fmt.Printf("compute epost proof (cold): %s\n", benchout.PostEProofCold)
|
||||
fmt.Printf("compute epost proof (hot): %s\n", benchout.PostEProofHot)
|
||||
fmt.Printf("verify epost proof (cold): %s\n", benchout.VerifyEPostCold)
|
||||
fmt.Printf("verify epost proof (hot): %s\n", benchout.VerifyEPostHot)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
@ -204,6 +204,7 @@ var aggregateSectorDirsCmd = &cli.Command{
|
||||
return err
|
||||
}
|
||||
|
||||
var aggrGenMiner genesis.GenesisMiner
|
||||
var highestSectorID uint64
|
||||
for _, dir := range cctx.Args().Slice() {
|
||||
dir, err := homedir.Expand(dir)
|
||||
@ -244,6 +245,8 @@ var aggregateSectorDirsCmd = &cli.Command{
|
||||
}
|
||||
}
|
||||
|
||||
aggrGenMiner = mergeGenMiners(aggrGenMiner, genm)
|
||||
|
||||
opts := badger.DefaultOptions
|
||||
opts.ReadOnly = true
|
||||
mds, err := badger.NewDatastore(filepath.Join(dir, "badger"), &opts)
|
||||
@ -274,6 +277,10 @@ var aggregateSectorDirsCmd = &cli.Command{
|
||||
return err
|
||||
}
|
||||
|
||||
if err := seed.WriteGenesisMiner(maddr, destdir, &aggrGenMiner); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
},
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user