lotus/node/modules/storageminer.go

245 lines
6.8 KiB
Go
Raw Normal View History

2019-08-01 14:19:53 +00:00
package modules
import (
"context"
"math"
2019-08-08 04:24:49 +00:00
"path/filepath"
"reflect"
2019-08-08 04:24:49 +00:00
2019-08-06 22:04:21 +00:00
"github.com/ipfs/go-bitswap"
"github.com/ipfs/go-bitswap/network"
"github.com/ipfs/go-blockservice"
2019-08-01 14:19:53 +00:00
"github.com/ipfs/go-datastore"
"github.com/ipfs/go-datastore/namespace"
2019-08-06 22:04:21 +00:00
blockstore "github.com/ipfs/go-ipfs-blockstore"
"github.com/ipfs/go-merkledag"
2019-08-01 14:19:53 +00:00
"github.com/libp2p/go-libp2p-core/host"
2019-08-29 15:09:34 +00:00
"github.com/libp2p/go-libp2p-core/routing"
2019-08-01 14:19:53 +00:00
"github.com/mitchellh/go-homedir"
"go.uber.org/fx"
2019-10-27 08:56:53 +00:00
"golang.org/x/xerrors"
2019-08-01 14:19:53 +00:00
"github.com/filecoin-project/lotus/api"
2019-10-27 08:56:53 +00:00
"github.com/filecoin-project/lotus/build"
"github.com/filecoin-project/lotus/chain/address"
"github.com/filecoin-project/lotus/chain/deals"
2019-11-25 04:45:13 +00:00
"github.com/filecoin-project/lotus/chain/gen"
"github.com/filecoin-project/lotus/datatransfer"
"github.com/filecoin-project/lotus/lib/sectorbuilder"
"github.com/filecoin-project/lotus/lib/statestore"
2019-11-25 04:45:13 +00:00
"github.com/filecoin-project/lotus/miner"
"github.com/filecoin-project/lotus/node/modules/dtypes"
"github.com/filecoin-project/lotus/node/modules/helpers"
"github.com/filecoin-project/lotus/node/repo"
"github.com/filecoin-project/lotus/retrieval"
"github.com/filecoin-project/lotus/storage"
2019-08-01 14:19:53 +00:00
)
func minerAddrFromDS(ds dtypes.MetadataDS) (address.Address, error) {
maddrb, err := ds.Get(datastore.NewKey("miner-address"))
if err != nil {
return address.Undef, err
}
return address.NewFromBytes(maddrb)
}
2019-12-04 19:44:15 +00:00
func GetParams(sbc *sectorbuilder.Config) error {
if err := build.GetParams(sbc.SectorSize); err != nil {
return xerrors.Errorf("fetching proof parameters: %w", err)
}
return nil
}
func SectorBuilderConfig(storagePath string, threads uint) func(dtypes.MetadataDS, api.FullNode) (*sectorbuilder.Config, error) {
return func(ds dtypes.MetadataDS, api api.FullNode) (*sectorbuilder.Config, error) {
minerAddr, err := minerAddrFromDS(ds)
if err != nil {
return nil, err
}
ssize, err := api.StateMinerSectorSize(context.TODO(), minerAddr, nil)
if err != nil {
return nil, err
}
2019-08-01 14:19:53 +00:00
sp, err := homedir.Expand(storagePath)
if err != nil {
return nil, err
}
if threads > math.MaxUint8 {
return nil, xerrors.Errorf("too many sectorbuilder threads specified: %d, max allowed: %d", threads, math.MaxUint8)
}
2019-10-30 18:10:29 +00:00
cache := filepath.Join(sp, "cache")
2019-12-01 17:58:31 +00:00
unsealed := filepath.Join(sp, "unsealed")
2019-08-01 14:19:53 +00:00
sealed := filepath.Join(sp, "sealed")
staging := filepath.Join(sp, "staging")
sb := &sectorbuilder.Config{
Miner: minerAddr,
SectorSize: ssize,
WorkerThreads: uint8(threads),
CacheDir: cache,
2019-12-01 17:58:31 +00:00
UnsealedDir: unsealed,
2019-08-01 14:19:53 +00:00
SealedDir: sealed,
StagedDir: staging,
}
return sb, nil
}
}
2019-11-07 18:22:59 +00:00
func StorageMiner(mctx helpers.MetricsCtx, lc fx.Lifecycle, api api.FullNode, h host.Host, ds dtypes.MetadataDS, sb *sectorbuilder.SectorBuilder, tktFn storage.TicketFn) (*storage.Miner, error) {
maddr, err := minerAddrFromDS(ds)
2019-08-01 14:19:53 +00:00
if err != nil {
return nil, err
}
2019-11-07 18:22:59 +00:00
sm, err := storage.NewMiner(api, maddr, h, ds, sb, tktFn)
2019-08-01 14:19:53 +00:00
if err != nil {
return nil, err
}
ctx := helpers.LifecycleCtx(mctx, lc)
lc.Append(fx.Hook{
OnStart: func(context.Context) error {
return sm.Run(ctx)
},
2019-11-01 13:58:48 +00:00
OnStop: sm.Stop,
2019-08-01 14:19:53 +00:00
})
return sm, nil
}
2019-08-02 16:25:10 +00:00
2019-08-26 13:45:36 +00:00
func HandleRetrieval(host host.Host, lc fx.Lifecycle, m *retrieval.Miner) {
lc.Append(fx.Hook{
OnStart: func(context.Context) error {
2019-08-27 18:45:21 +00:00
host.SetStreamHandler(retrieval.QueryProtocolID, m.HandleQueryStream)
host.SetStreamHandler(retrieval.ProtocolID, m.HandleDealStream)
2019-08-26 13:45:36 +00:00
return nil
},
})
}
func HandleDeals(mctx helpers.MetricsCtx, lc fx.Lifecycle, host host.Host, h *deals.Provider) {
2019-08-06 23:08:34 +00:00
ctx := helpers.LifecycleCtx(mctx, lc)
lc.Append(fx.Hook{
OnStart: func(context.Context) error {
h.Run(ctx)
host.SetStreamHandler(deals.DealProtocolID, h.HandleStream)
2019-09-13 21:00:36 +00:00
host.SetStreamHandler(deals.AskProtocolID, h.HandleAskStream)
2019-08-06 23:08:34 +00:00
return nil
},
OnStop: func(context.Context) error {
h.Stop()
return nil
},
})
2019-08-02 16:25:10 +00:00
}
2019-08-06 22:04:21 +00:00
// RegisterProviderValidator is an initialization hook that registers the provider
// request validator with the data transfer module as the validator for
// StorageDataTransferVoucher types
func RegisterProviderValidator(mrv *deals.ProviderRequestValidator, dtm dtypes.ProviderDataTransfer) {
2019-12-05 05:02:14 +00:00
if err := dtm.RegisterVoucherType(reflect.TypeOf(deals.StorageDataTransferVoucher{}), mrv); err != nil {
panic(err)
}
}
// NewProviderDAGServiceDataTransfer returns a data transfer manager that just
// uses the provider's Staging DAG service for transfers
func NewProviderDAGServiceDataTransfer(dag dtypes.StagingDAG) dtypes.ProviderDataTransfer {
return datatransfer.NewDAGServiceDataTransfer(dag)
}
// NewProviderDealStore creates a statestore for the client to store its deals
func NewProviderDealStore(ds dtypes.MetadataDS) dtypes.ProviderDealStore {
return statestore.New(namespace.Wrap(ds, datastore.NewKey("/deals/client")))
}
2019-08-06 22:04:21 +00:00
func StagingDAG(mctx helpers.MetricsCtx, lc fx.Lifecycle, r repo.LockedRepo, rt routing.Routing, h host.Host) (dtypes.StagingDAG, error) {
stagingds, err := r.Datastore("/staging")
if err != nil {
return nil, err
}
bs := blockstore.NewBlockstore(stagingds)
ibs := blockstore.NewIdStore(bs)
bitswapNetwork := network.NewFromIpfsHost(h, rt)
exch := bitswap.New(helpers.LifecycleCtx(mctx, lc), bitswapNetwork, bs)
bsvc := blockservice.New(ibs, exch)
dag := merkledag.NewDAGService(bsvc)
lc.Append(fx.Hook{
OnStop: func(_ context.Context) error {
return bsvc.Close()
},
})
return dag, nil
}
2019-11-25 04:45:13 +00:00
func SetupBlockProducer(lc fx.Lifecycle, ds dtypes.MetadataDS, api api.FullNode, epp gen.ElectionPoStProver) (*miner.Miner, error) {
minerAddr, err := minerAddrFromDS(ds)
if err != nil {
2019-11-25 04:45:13 +00:00
return nil, err
}
2019-11-25 04:45:13 +00:00
m := miner.NewMiner(api, epp)
lc.Append(fx.Hook{
OnStart: func(ctx context.Context) error {
2019-11-25 04:45:13 +00:00
if err := m.Register(minerAddr); err != nil {
return err
}
return nil
},
2019-09-17 14:23:08 +00:00
OnStop: func(ctx context.Context) error {
2019-11-25 04:45:13 +00:00
return m.Unregister(ctx, minerAddr)
2019-09-17 14:23:08 +00:00
},
})
2019-11-25 04:45:13 +00:00
return m, nil
}
2019-10-27 08:56:53 +00:00
func SectorBuilder(cfg *sectorbuilder.Config, ds dtypes.MetadataDS) (*sectorbuilder.SectorBuilder, error) {
sb, err := sectorbuilder.New(cfg, ds)
if err != nil {
return nil, err
}
return sb, nil
}
2019-11-07 18:22:59 +00:00
func SealTicketGen(api api.FullNode) storage.TicketFn {
2019-10-27 08:56:53 +00:00
return func(ctx context.Context) (*sectorbuilder.SealTicket, error) {
ts, err := api.ChainHead(ctx)
if err != nil {
return nil, xerrors.Errorf("getting head ts for SealTicket failed: %w", err)
}
r, err := api.ChainGetRandomness(ctx, ts.Key(), int64(ts.Height())-build.SealRandomnessLookback)
2019-10-27 08:56:53 +00:00
if err != nil {
return nil, xerrors.Errorf("getting randomness for SealTicket failed: %w", err)
}
var tkt [sectorbuilder.CommLen]byte
if n := copy(tkt[:], r); n != sectorbuilder.CommLen {
return nil, xerrors.Errorf("unexpected randomness len: %d (expected %d)", n, sectorbuilder.CommLen)
}
return &sectorbuilder.SealTicket{
BlockHeight: ts.Height(),
2019-10-27 08:56:53 +00:00
TicketBytes: tkt,
}, nil
}
}