cerc-io/lotus
11
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Merge pull request #934 from filecoin-project/feat/sb-diskmgr

Browse Source 
sectorbuilder: Check free space before creating sectors
This commit is contained in:
Łukasz Magiera
2019-12-16 21:37:18 +01:00
committed by GitHub
parent 4036f09610 1383bfa0eb
commit 3fda442bb3
19 changed files with 230 additions and 181 deletions
Download Patch File Download Diff File Expand all files Collapse all files
cmd/lotus-bench/main.go
+5 -11
View File
@@ -6,7 +6,6 @@ import (
"crypto/sha256"
"encoding/json"
"fmt"
"github.com/docker/go-units"
"io/ioutil"
"math/big"
"math/rand"
@@ -14,8 +13,8 @@ import (
"path/filepath"
"time"
"github.com/docker/go-units"
ffi "github.com/filecoin-project/filecoin-ffi"
lcli "github.com/filecoin-project/lotus/cli"
"github.com/ipfs/go-datastore"
logging "github.com/ipfs/go-log"
"github.com/mitchellh/go-homedir"
@@ -140,17 +139,12 @@ func main() {
Miner: maddr,
SectorSize: sectorSize,
WorkerThreads: 2,
CacheDir: filepath.Join(sbdir, "cache"),
SealedDir: filepath.Join(sbdir, "sealed"),
StagedDir: filepath.Join(sbdir, "staged"),
UnsealedDir: filepath.Join(sbdir, "unsealed"),
Dir: sbdir,
}
if robench == "" {
for _, d := range []string{cfg.CacheDir, cfg.SealedDir, cfg.StagedDir, cfg.UnsealedDir} {
if err := os.MkdirAll(d, 0775); err != nil {
return err
}
if err := os.MkdirAll(sbdir, 0775); err != nil {
return err
}
}
@@ -371,5 +365,5 @@ func bps(data uint64, d time.Duration) string {
bdata := new(big.Int).SetUint64(data)
bdata = bdata.Mul(bdata, big.NewInt(time.Second.Nanoseconds()))
bps := bdata.Div(bdata, big.NewInt(d.Nanoseconds()))
return lcli.SizeStr(types.BigInt{bps}) + "/s"
return (types.BigInt{bps}).SizeStr() + "/s"
}
cmd/lotus-seal-worker/sub.go
+2 -7
View File
@@ -2,10 +2,8 @@ package main
import (
"context"
"net/http"
"path/filepath"
"golang.org/x/xerrors"
"net/http"
lapi "github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/build"
@@ -35,10 +33,7 @@ func acceptJobs(ctx context.Context, api lapi.StorageMiner, endpoint string, aut
SectorSize: ssize,
Miner: act,
WorkerThreads: 1,
CacheDir: filepath.Join(repo, "cache"),
SealedDir: filepath.Join(repo, "sealed"),
StagedDir: filepath.Join(repo, "staged"),
UnsealedDir: filepath.Join(repo, "unsealed"),
Dir: repo,
})
if err != nil {
return err
cmd/lotus-seal-worker/transfer.go
+1 -1
View File
@@ -132,7 +132,7 @@ func (w *worker) fetchSector(sectorID uint64, typ sectorbuilder.WorkerTaskType)
var err error
switch typ {
case sectorbuilder.WorkerPreCommit:
err = w.fetch("staged", sectorID)
err = w.fetch("staging", sectorID)
case sectorbuilder.WorkerCommit:
err = w.fetch("sealed", sectorID)
if err != nil {
cmd/lotus-seed/main.go
+2 -8
View File
@@ -194,10 +194,7 @@ var aggregateSectorDirsCmd = &cli.Command{
agsb, err := sectorbuilder.New(&sectorbuilder.Config{
Miner: maddr,
SectorSize: ssize,
CacheDir: filepath.Join(destdir, "cache"),
SealedDir: filepath.Join(destdir, "sealed"),
StagedDir: filepath.Join(destdir, "staging"),
UnsealedDir: filepath.Join(destdir, "unsealed"),
Dir: destdir,
WorkerThreads: 2,
}, agmds)
if err != nil {
@@ -258,10 +255,7 @@ var aggregateSectorDirsCmd = &cli.Command{
sb, err := sectorbuilder.New(&sectorbuilder.Config{
Miner: maddr,
SectorSize: genm.SectorSize,
CacheDir: filepath.Join(dir, "cache"),
SealedDir: filepath.Join(dir, "sealed"),
StagedDir: filepath.Join(dir, "staging"),
UnsealedDir: filepath.Join(dir, "unsealed"),
Dir: dir,
WorkerThreads: 2,
}, mds)
if err != nil {
cmd/lotus-seed/seed/seed.go
+3 -8
View File
@@ -30,17 +30,12 @@ func PreSeal(maddr address.Address, ssize uint64, offset uint64, sectors int, sb
Miner: maddr,
SectorSize: ssize,
FallbackLastID: offset,
CacheDir: filepath.Join(sbroot, "cache"),
SealedDir: filepath.Join(sbroot, "sealed"),
StagedDir: filepath.Join(sbroot, "staging"),
UnsealedDir: filepath.Join(sbroot, "unsealed"),
Dir: sbroot,
WorkerThreads: 2,
}
for _, d := range []string{cfg.CacheDir, cfg.SealedDir, cfg.StagedDir, cfg.UnsealedDir} {
if err := os.MkdirAll(d, 0775); err != nil {
return nil, err
}
if err := os.MkdirAll(sbroot, 0775); err != nil {
return nil, err
}
mds, err := badger.NewDatastore(filepath.Join(sbroot, "badger"), nil)
cmd/lotus-storage-miner/info.go
+4 -4
View File
@@ -43,7 +43,7 @@ var infoCmd = &cli.Command{
return err
}
fmt.Printf("Sector Size: %s\n", lcli.SizeStr(types.NewInt(sizeByte)))
fmt.Printf("Sector Size: %s\n", types.NewInt(sizeByte).SizeStr())
pow, err := api.StateMinerPower(ctx, maddr, nil)
if err != nil {
@@ -51,14 +51,14 @@ var infoCmd = &cli.Command{
}
percI := types.BigDiv(types.BigMul(pow.MinerPower, types.NewInt(1000)), pow.TotalPower)
fmt.Printf("Power: %s / %s (%0.4f%%)\n", lcli.SizeStr(pow.MinerPower), lcli.SizeStr(pow.TotalPower), float64(percI.Int64())/100000*10000)
fmt.Printf("Power: %s / %s (%0.4f%%)\n", pow.MinerPower.SizeStr(), pow.TotalPower.SizeStr(), float64(percI.Int64())/100000*10000)
secCounts, err := api.StateMinerSectorCount(ctx, maddr, nil)
if err != nil {
return err
}
fmt.Printf("\tCommitted: %s\n", lcli.SizeStr(types.BigMul(types.NewInt(secCounts.Sset), types.NewInt(sizeByte))))
fmt.Printf("\tProving: %s\n", lcli.SizeStr(types.BigMul(types.NewInt(secCounts.Pset), types.NewInt(sizeByte))))
fmt.Printf("\tCommitted: %s\n", types.BigMul(types.NewInt(secCounts.Sset), types.NewInt(sizeByte)).SizeStr())
fmt.Printf("\tProving: %s\n", types.BigMul(types.NewInt(secCounts.Pset), types.NewInt(sizeByte)).SizeStr())
// TODO: indicate whether the post worker is in use
wstat, err := nodeApi.WorkerStats(ctx)
cmd/lotus-storage-miner/init.go
+2 -8
View File
@@ -172,10 +172,7 @@ var initCmd = &cli.Command{
oldsb, err := sectorbuilder.New(&sectorbuilder.Config{
SectorSize: ssize,
WorkerThreads: 2,
SealedDir: filepath.Join(pssb, "sealed"),
CacheDir: filepath.Join(pssb, "cache"),
StagedDir: filepath.Join(pssb, "staging"),
UnsealedDir: filepath.Join(pssb, "unsealed"),
Dir: pssb,
}, oldmds)
if err != nil {
return xerrors.Errorf("failed to open up preseal sectorbuilder: %w", err)
@@ -184,10 +181,7 @@ var initCmd = &cli.Command{
nsb, err := sectorbuilder.New(&sectorbuilder.Config{
SectorSize: ssize,
WorkerThreads: 2,
SealedDir: filepath.Join(lr.Path(), "sealed"),
CacheDir: filepath.Join(lr.Path(), "cache"),
StagedDir: filepath.Join(lr.Path(), "staging"),
UnsealedDir: filepath.Join(lr.Path(), "unsealed"),
Dir: lr.Path(),
}, mds)
if err != nil {
return xerrors.Errorf("failed to open up sectorbuilder: %w", err)
cmd/lotus-storage-miner/run.go
+1 -1
View File
@@ -61,7 +61,7 @@ var runCmd = &cli.Command{
}
if v.APIVersion != build.APIVersion {
return xerrors.Errorf("lotus-daemon API version doesn't match: local: ", api.Version{APIVersion: build.APIVersion})
return xerrors.Errorf("lotus-daemon API version doesn't match: local: %s", api.Version{APIVersion: build.APIVersion})
}
log.Info("Checking full node sync status")