cerc-io/lotus
11
0
Fork 0
Code Issues Pull Requests Packages Projects Releases 27 Wiki Activity

Merge branch 'testnet/3' into feat/update-markets

Browse Source
This commit is contained in:
Łukasz Magiera
2020-02-04 07:17:22 +01:00
parent 00941e4461 f837cb5513
commit 0e81c1cf7e
63 changed files with 1324 additions and 308 deletions
Download Patch File Download Diff File Expand all files Collapse all files
node/builder.go
+12 -4
View File
@@ -7,6 +7,7 @@ import (
sectorbuilder "github.com/filecoin-project/go-sectorbuilder"
blockstore "github.com/ipfs/go-ipfs-blockstore"
logging "github.com/ipfs/go-log"
ci "github.com/libp2p/go-libp2p-core/crypto"
"github.com/libp2p/go-libp2p-core/host"
"github.com/libp2p/go-libp2p-core/peer"
@@ -54,6 +55,8 @@ import (
"github.com/filecoin-project/lotus/storage/sectorblocks"
)
var log = logging.Logger("builder")
// special is a type used to give keys to modules which
// can't really be identified by the returned type
type special struct{ id int }
@@ -343,15 +346,20 @@ func ConfigStorageMiner(c interface{}, lr repo.LockedRepo) Option {
return Error(xerrors.Errorf("invalid config from repo, got: %T", c))
}
path := cfg.SectorBuilder.Path
if path == "" {
path = lr.Path()
scfg := sectorbuilder.SimplePath(lr.Path())
if cfg.SectorBuilder.Path == "" {
if len(cfg.SectorBuilder.Storage) > 0 {
scfg = cfg.SectorBuilder.Storage
}
} else {
scfg = sectorbuilder.SimplePath(cfg.SectorBuilder.Path)
log.Warn("LEGACY SectorBuilder.Path FOUND IN CONFIG. Please use the new storage config")
}
return Options(
ConfigCommon(&cfg.Common),
Override(new(*sectorbuilder.Config), modules.SectorBuilderConfig(path,
Override(new(*sectorbuilder.Config), modules.SectorBuilderConfig(scfg,
cfg.SectorBuilder.WorkerCount,
cfg.SectorBuilder.DisableLocalPreCommit,
cfg.SectorBuilder.DisableLocalCommit)),
node/config/def.go
+4 -1
View File
@@ -3,6 +3,8 @@ package config
import (
"encoding"
"time"
"github.com/filecoin-project/go-sectorbuilder/fs"
)
// Common is common config between full node and miner
@@ -54,7 +56,8 @@ type Metrics struct {
// // Storage Miner
type SectorBuilder struct {
Path string
Path string // TODO: remove // FORK (-ish)
Storage []fs.PathConfig
WorkerCount uint
DisableLocalPreCommit bool
node/impl/full/chain.go
+4
View File
@@ -170,6 +170,10 @@ func (a *ChainAPI) ChainReadObj(ctx context.Context, obj cid.Cid) ([]byte, error
return blk.RawData(), nil
}
func (a *ChainAPI) ChainHasObj(ctx context.Context, obj cid.Cid) (bool, error) {
return a.Chain.Blockstore().Has(obj)
}
func (a *ChainAPI) ChainSetHead(ctx context.Context, ts *types.TipSet) error {
return a.Chain.SetHead(ts)
}
node/impl/full/state.go
+34
View File
@@ -3,6 +3,7 @@ package full
import (
"bytes"
"context"
"fmt"
"strconv"
"github.com/filecoin-project/go-amt-ipld"
@@ -84,6 +85,10 @@ func (a *StateAPI) StateMinerSectorSize(ctx context.Context, actor address.Addre
return stmgr.GetMinerSectorSize(ctx, a.StateManager, ts, actor)
}
func (a *StateAPI) StateMinerFaults(ctx context.Context, addr address.Address, ts *types.TipSet) ([]uint64, error) {
return stmgr.GetMinerFaults(ctx, a.StateManager, ts, addr)
}
func (a *StateAPI) StatePledgeCollateral(ctx context.Context, ts *types.TipSet) (types.BigInt, error) {
param, err := actors.SerializeParams(&actors.PledgeCollateralParams{Size: types.NewInt(0)})
if err != nil {
@@ -404,3 +409,32 @@ func (a *StateAPI) StateListMessages(ctx context.Context, match *types.Message,
func (a *StateAPI) StateCompute(ctx context.Context, height uint64, msgs []*types.Message, ts *types.TipSet) (cid.Cid, error) {
return stmgr.ComputeState(ctx, a.StateManager, height, msgs, ts)
}
func (a *StateAPI) MsigGetAvailableBalance(ctx context.Context, addr address.Address, ts *types.TipSet) (types.BigInt, error) {
if ts == nil {
ts = a.Chain.GetHeaviestTipSet()
}
var st actors.MultiSigActorState
act, err := a.StateManager.LoadActorState(ctx, addr, &st, ts)
if err != nil {
return types.EmptyInt, xerrors.Errorf("failed to load multisig actor state: %w", err)
}
if act.Code != actors.MultisigCodeCid {
return types.EmptyInt, fmt.Errorf("given actor was not a multisig")
}
if st.UnlockDuration == 0 {
return act.Balance, nil
}
offset := ts.Height() - st.StartingBlock
if offset > st.UnlockDuration {
return act.Balance, nil
}
minBalance := types.BigDiv(st.InitialBalance, types.NewInt(st.UnlockDuration))
minBalance = types.BigMul(minBalance, types.NewInt(offset))
return types.BigSub(act.Balance, minBalance), nil
}
node/impl/storminer.go
+28 -12
View File
@@ -9,14 +9,15 @@ import (
"os"
"strconv"
"github.com/filecoin-project/lotus/api/apistruct"
"github.com/gorilla/mux"
files "github.com/ipfs/go-ipfs-files"
"github.com/filecoin-project/go-address"
"github.com/filecoin-project/go-sectorbuilder"
"github.com/filecoin-project/go-sectorbuilder/fs"
"github.com/filecoin-project/lotus/api"
"github.com/filecoin-project/lotus/api/apistruct"
"github.com/filecoin-project/lotus/lib/tarutil"
"github.com/filecoin-project/lotus/miner"
"github.com/filecoin-project/lotus/storage"
@@ -44,8 +45,8 @@ func (sm *StorageMinerAPI) ServeRemote(w http.ResponseWriter, r *http.Request) {
mux := mux.NewRouter()
mux.HandleFunc("/remote/{type}/{sname}", sm.remoteGetSector).Methods("GET")
mux.HandleFunc("/remote/{type}/{sname}", sm.remotePutSector).Methods("PUT")
mux.HandleFunc("/remote/{type}/{id}", sm.remoteGetSector).Methods("GET")
mux.HandleFunc("/remote/{type}/{id}", sm.remotePutSector).Methods("PUT")
log.Infof("SERVEGETREMOTE %s", r.URL)
@@ -55,14 +56,21 @@ func (sm *StorageMinerAPI) ServeRemote(w http.ResponseWriter, r *http.Request) {
func (sm *StorageMinerAPI) remoteGetSector(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
path, err := sm.SectorBuilder.GetPath(vars["type"], vars["sname"])
id, err := strconv.ParseUint(vars["id"], 10, 64)
if err != nil {
log.Error("parsing sector id: ", err)
w.WriteHeader(500)
return
}
path, err := sm.SectorBuilder.SectorPath(fs.DataType(vars["type"]), id)
if err != nil {
log.Error(err)
w.WriteHeader(500)
return
}
stat, err := os.Stat(path)
stat, err := os.Stat(string(path))
if err != nil {
log.Error(err)
w.WriteHeader(500)
@@ -71,10 +79,10 @@ func (sm *StorageMinerAPI) remoteGetSector(w http.ResponseWriter, r *http.Reques
var rd io.Reader
if stat.IsDir() {
rd, err = tarutil.TarDirectory(path)
rd, err = tarutil.TarDirectory(string(path))
w.Header().Set("Content-Type", "application/x-tar")
} else {
rd, err = os.OpenFile(path, os.O_RDONLY, 0644)
rd, err = os.OpenFile(string(path), os.O_RDONLY, 0644)
w.Header().Set("Content-Type", "application/octet-stream")
}
if err != nil {
@@ -93,7 +101,15 @@ func (sm *StorageMinerAPI) remoteGetSector(w http.ResponseWriter, r *http.Reques
func (sm *StorageMinerAPI) remotePutSector(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
path, err := sm.SectorBuilder.GetPath(vars["type"], vars["sname"])
id, err := strconv.ParseUint(vars["id"], 10, 64)
if err != nil {
log.Error("parsing sector id: ", err)
w.WriteHeader(500)
return
}
// This is going to get better with worker-to-worker transfers
path, err := sm.SectorBuilder.AllocSectorPath(fs.DataType(vars["type"]), id, true)
if err != nil {
log.Error(err)
w.WriteHeader(500)
@@ -107,7 +123,7 @@ func (sm *StorageMinerAPI) remotePutSector(w http.ResponseWriter, r *http.Reques
return
}
if err := os.RemoveAll(path); err != nil {
if err := os.RemoveAll(string(path)); err != nil {
log.Error(err)
w.WriteHeader(500)
return
@@ -115,13 +131,13 @@ func (sm *StorageMinerAPI) remotePutSector(w http.ResponseWriter, r *http.Reques
switch mediatype {
case "application/x-tar":
if err := tarutil.ExtractTar(r.Body, path); err != nil {
if err := tarutil.ExtractTar(r.Body, string(path)); err != nil {
log.Error(err)
w.WriteHeader(500)
return
}
default:
if err := files.WriteTo(files.NewReaderFile(r.Body), path); err != nil {
if err := files.WriteTo(files.NewReaderFile(r.Body), string(path)); err != nil {
log.Error(err)
w.WriteHeader(500)
return
node/modules/storageminer.go
+19 -8
View File
@@ -17,6 +17,7 @@ import (
storageimpl "github.com/filecoin-project/go-fil-markets/storagemarket/impl"
paramfetch "github.com/filecoin-project/go-paramfetch"
"github.com/filecoin-project/go-sectorbuilder"
"github.com/filecoin-project/go-sectorbuilder/fs"
"github.com/filecoin-project/go-statestore"
"github.com/ipfs/go-bitswap"
"github.com/ipfs/go-bitswap/network"
@@ -57,14 +58,14 @@ func minerAddrFromDS(ds dtypes.MetadataDS) (address.Address, error) {
}
func GetParams(sbc *sectorbuilder.Config) error {
if err := paramfetch.GetParams(build.ParametersJson, sbc.SectorSize); err != nil {
if err := paramfetch.GetParams(build.ParametersJson(), sbc.SectorSize); err != nil {
return xerrors.Errorf("fetching proof parameters: %w", err)
}
return nil
}
func SectorBuilderConfig(storagePath string, threads uint, noprecommit, nocommit bool) func(dtypes.MetadataDS, api.FullNode) (*sectorbuilder.Config, error) {
func SectorBuilderConfig(storage []fs.PathConfig, threads uint, noprecommit, nocommit bool) func(dtypes.MetadataDS, api.FullNode) (*sectorbuilder.Config, error) {
return func(ds dtypes.MetadataDS, api api.FullNode) (*sectorbuilder.Config, error) {
minerAddr, err := minerAddrFromDS(ds)
if err != nil {
@@ -76,9 +77,11 @@ func SectorBuilderConfig(storagePath string, threads uint, noprecommit, nocommit
return nil, err
}
sp, err := homedir.Expand(storagePath)
if err != nil {
return nil, err
for i := range storage {
storage[i].Path, err = homedir.Expand(storage[i].Path)
if err != nil {
return nil, err
}
}
if threads > math.MaxUint8 {
@@ -93,7 +96,7 @@ func SectorBuilderConfig(storagePath string, threads uint, noprecommit, nocommit
NoPreCommit: noprecommit,
NoCommit: nocommit,
Dir: sp,
Paths: storage,
}
return sb, nil
@@ -106,15 +109,23 @@ func StorageMiner(mctx helpers.MetricsCtx, lc fx.Lifecycle, api api.FullNode, h
return nil, err
}
sm, err := storage.NewMiner(api, maddr, h, ds, sb, tktFn)
ctx := helpers.LifecycleCtx(mctx, lc)
worker, err := api.StateMinerWorker(ctx, maddr, nil)
if err != nil {
return nil, err
}
ctx := helpers.LifecycleCtx(mctx, lc)
fps := storage.NewFPoStScheduler(api, sb, maddr, worker)
sm, err := storage.NewMiner(api, maddr, worker, h, ds, sb, tktFn)
if err != nil {
return nil, err
}
lc.Append(fx.Hook{
OnStart: func(context.Context) error {
go fps.Run(ctx)
return sm.Run(ctx)
},
OnStop: sm.Stop,
node/node_test.go
+1 -1
View File
@@ -232,7 +232,7 @@ func builder(t *testing.T, nFull int, storage []int) ([]test.TestNode, []test.Te
SectorSize: 1024,
WorkerThreads: 2,
Miner: genMiner,
Dir: psd,
Paths: sectorbuilder.SimplePath(psd),
}, namespace.Wrap(mds, datastore.NewKey("/sectorbuilder")))
if err != nil {
t.Fatal(err)